From 02810c7aa89b8100b90b7b0f5e96bc55aafd3d0a Mon Sep 17 00:00:00 2001 From: Oliver Simons Date: Tue, 16 Jun 2026 11:52:38 +0200 Subject: [PATCH 001/292] Fix and restrict NVFP4 edge-cases in llama-graph (#24331) * Move post-GEMM MUL required for dequant b4 lora and bias add see https://github.com/ggml-org/llama.cpp/pull/23484 : 1. For lora, I would presume we want fully dequantized values before doing the residuals, but this depends on how the LORAs were generated. Literature tells me LORA happens post-mul but pre-bias add https://github.com/ggml-org/llama.cpp/pull/8332 2. For ModelOPT, bias-add should happen on [fully-dequantized values](https://github.com/NVIDIA/Model-Optimizer/blob/b49f9b9e2d747af992d78a3aa7f10efe5a8847e1/modelopt/torch/quantization/backends/nvfp4_gemm.py#L59-L64) * Restrict build_ffn for NVFP4 to supported combinations --- src/llama-graph.cpp | 103 +++++++++++++++++++++++++------------------- src/llama-graph.h | 5 ++- 2 files changed, 61 insertions(+), 47 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 7468bd9b7..68c9e606c 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1088,6 +1088,10 @@ ggml_tensor * llm_graph_context::build_lora_mm( ggml_tensor * w_s) const { ggml_tensor * res = ggml_mul_mat(ctx0, w, cur); + if (w_s) { + res = ggml_mul(ctx0, res, w_s); + } + for (const auto & lora : *loras) { llama_adapter_lora_weight * lw = lora.first->get_weight(w); if (lw == nullptr) { @@ -1106,18 +1110,24 @@ ggml_tensor * llm_graph_context::build_lora_mm( res = ggml_add(ctx0, res, ab_cur); } - if (w_s) { - res = ggml_mul(ctx0, res, w_s); - } - return res; } ggml_tensor * llm_graph_context::build_lora_mm_id( ggml_tensor * w, // ggml_tensor * as ggml_tensor * cur, // ggml_tensor * b - ggml_tensor * ids) const { + ggml_tensor * ids, + ggml_tensor * w_s) const { ggml_tensor * res = ggml_mul_mat_id(ctx0, w, cur, ids); + + if (w_s) { + const int64_t n_expert = w_s->ne[0]; + const int64_t n_tokens = cur->ne[2]; + ggml_tensor * s = ggml_reshape_3d(ctx0, w_s, 1, n_expert, 1); + s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1); + s = ggml_get_rows(ctx0, s, ids); + res = ggml_mul(ctx0, res, s); + } for (const auto & lora : *loras) { llama_adapter_lora_weight * lw = lora.first->get_weight(w); if (lw == nullptr) { @@ -1269,6 +1279,29 @@ ggml_tensor * llm_graph_context::build_ffn( llm_ffn_op_type type_op, llm_ffn_gate_type type_gate, int il) const { + // NVFP4 support is currently restricted to + // 1) LORA absence (*_s would be applied after LORA residual, which is incorrect) + // 2) bias absense (*_s would be applied after bias addition, which is incorrect) + // TODO: disambiguate LLM-architectural scales (which use *_s) from NVFP4 scale_2 (which also uses *_s currently) + auto has_lora = [this](ggml_tensor * w) { + if (!w) { + return false; + } + for (const auto & lora : *loras) { + if (lora.first->get_weight(w) != nullptr) { + return true; + } + } + return false; + }; + + GGML_ASSERT(!up_s || !up_b || !up || up->type != GGML_TYPE_NVFP4); + GGML_ASSERT(!gate_s || !gate_b || !gate || gate->type != GGML_TYPE_NVFP4); + GGML_ASSERT(!down_s || !down_b || !down || down->type != GGML_TYPE_NVFP4); + GGML_ASSERT(!up_s || !up || up->type != GGML_TYPE_NVFP4 || !has_lora(up)); + GGML_ASSERT(!gate_s || !gate || gate->type != GGML_TYPE_NVFP4 || !has_lora(gate)); + GGML_ASSERT(!down_s || !down || down->type != GGML_TYPE_NVFP4 || !has_lora(down)); + ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur; cb(tmp, "ffn_up", il); @@ -1627,23 +1660,18 @@ ggml_tensor * llm_graph_context::build_moe_ffn( if (gate_up_exps) { // merged gate_up path: one mul_mat_id, then split into gate and up views - ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts); // [n_ff*2, n_expert_used, n_tokens] + ggml_tensor * gate_up = build_lora_mm_id(gate_up_exps, cur, selected_experts, up_exps_s); // [n_ff*2, n_expert_used, n_tokens] cb(gate_up, "ffn_moe_gate_up", il); + if (up_exps_s) { + cb(gate_up, "ffn_moe_gate_up_scaled", il); + } + if (gate_up_exps_b) { gate_up = ggml_add_id(ctx0, gate_up, gate_up_exps_b, selected_experts); cb(gate_up, "ffn_moe_gate_up_biased", il); } - // apply per-expert scale2 to merged gate_up (use up_exps_s since gate and up are fused) - if (up_exps_s) { - ggml_tensor * s = ggml_reshape_3d(ctx0, up_exps_s, 1, n_expert, 1); - s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1); - s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens] - gate_up = ggml_mul(ctx0, gate_up, s); - cb(gate_up, "ffn_moe_gate_up_scaled", il); - } - const int64_t n_ff = gate_up->ne[0] / 2; cur = ggml_view_3d(ctx0, gate_up, n_ff, gate_up->ne[1], gate_up->ne[2], gate_up->nb[1], gate_up->nb[2], 0); cb(cur, "ffn_moe_gate", il); @@ -1651,43 +1679,33 @@ ggml_tensor * llm_graph_context::build_moe_ffn( cb(up, "ffn_moe_up", il); } else { // separate gate and up path - up = build_lora_mm_id(up_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens] + up = build_lora_mm_id(up_exps, cur, selected_experts, up_exps_s); // [n_ff, n_expert_used, n_tokens] cb(up, "ffn_moe_up", il); + if (up_exps_s) { + cb(up, "ffn_moe_up_scaled", il); + } + if (up_exps_b) { up = ggml_add_id(ctx0, up, up_exps_b, selected_experts); cb(up, "ffn_moe_up_biased", il); } - // apply per-expert scale2 to up - if (up_exps_s) { - ggml_tensor * s = ggml_reshape_3d(ctx0, up_exps_s, 1, n_expert, 1); - s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1); - s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens] - up = ggml_mul(ctx0, up, s); - cb(up, "ffn_moe_up_scaled", il); - } - if (gate_exps) { - cur = build_lora_mm_id(gate_exps, cur, selected_experts); // [n_ff, n_expert_used, n_tokens] + cur = build_lora_mm_id(gate_exps, cur, selected_experts, gate_exps_s); // [n_ff, n_expert_used, n_tokens] cb(cur, "ffn_moe_gate", il); } else { cur = up; } + if (gate_exps_s) { + cb(cur, "ffn_moe_gate_scaled", il); + } + if (gate_exps_b) { cur = ggml_add_id(ctx0, cur, gate_exps_b, selected_experts); cb(cur, "ffn_moe_gate_biased", il); } - - // apply per-expert scale2 to gate - if (gate_exps_s) { - ggml_tensor * s = ggml_reshape_3d(ctx0, gate_exps_s, 1, n_expert, 1); - s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1); - s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens] - cur = ggml_mul(ctx0, cur, s); - cb(cur, "ffn_moe_gate_scaled", il); - } } const bool has_gate = gate_exps || gate_up_exps; @@ -1759,23 +1777,18 @@ ggml_tensor * llm_graph_context::build_moe_ffn( GGML_ABORT("fatal error"); } - experts = build_lora_mm_id(down_exps, cur, selected_experts); // [n_embd, n_expert_used, n_tokens] + experts = build_lora_mm_id(down_exps, cur, selected_experts, down_exps_s); // [n_embd, n_expert_used, n_tokens] cb(experts, "ffn_moe_down", il); + if (down_exps_s) { + cb(experts, "ffn_moe_down_scaled", il); + } + if (down_exps_b) { experts = ggml_add_id(ctx0, experts, down_exps_b, selected_experts); cb(experts, "ffn_moe_down_biased", il); } - // apply per-expert scale2 to down - if (down_exps_s) { - ggml_tensor * s = ggml_reshape_3d(ctx0, down_exps_s, 1, n_expert, 1); - s = ggml_repeat_4d(ctx0, s, 1, n_expert, n_tokens, 1); - s = ggml_get_rows(ctx0, s, selected_experts); // [1, n_expert_used, n_tokens] - experts = ggml_mul(ctx0, experts, s); - cb(experts, "ffn_moe_down_scaled", il); - } - if (!weight_before_ffn) { experts = ggml_mul(ctx0, experts, weights); cb(experts, "ffn_moe_weighted", il); diff --git a/src/llama-graph.h b/src/llama-graph.h index cc5cfe51d..5e8a65835 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -853,11 +853,12 @@ struct llm_graph_context { ggml_tensor * cur, ggml_tensor * w_s = nullptr) const; - // do mat_mul_id, while optionally apply lora + // do mat_mul_id, while optionally apply lora and per-expert scale ggml_tensor * build_lora_mm_id( ggml_tensor * w, // ggml_tensor * as ggml_tensor * cur, // ggml_tensor * b - ggml_tensor * ids) const; + ggml_tensor * ids, + ggml_tensor * w_s = nullptr) const; ggml_tensor * build_norm( ggml_tensor * cur, From c1304d7b28e14380dbb90252c92aa2798db60185 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 16 Jun 2026 14:14:22 +0200 Subject: [PATCH 002/292] ui: add source toggle to mermaid and svg blocks (#24652) * ui: add source toggle to mermaid and svg blocks Add a toggle button next to copy and preview that switches a rendered mermaid or svg block to its source code and back. The button is shared by both block types and the rendered view stays the default. The source view reuses the code block scroll container and the highlighted code element captured at transform time, so it matches the app code blocks without highlighting again. Make tall diagrams scroll like text code blocks: safe centering keeps the diagram centered when it fits and falls back to start alignment when it overflows, so the top stays reachable instead of clipping above. Keep the block header opaque and layered above the scrolled diagram, and ignore header clicks in the zoom handler, so a button click never falls through to the zoom dialog. * ui: transparent diagram block header, address review from @allozaur --- .../MarkdownContent/MarkdownContent.svelte | 29 +++++++- .../MarkdownContent/markdown-content.css | 48 +++++++++++-- .../plugins/rehype/code-block-utils.ts | 70 +++++++++++++++++-- .../plugins/rehype/enhance-mermaid-blocks.ts | 16 ++++- .../plugins/rehype/enhance-svg-blocks.ts | 24 +++++-- .../plugins/rehype/pre-transform.ts | 14 +++- tools/ui/src/lib/constants/diagram-blocks.ts | 9 +++ tools/ui/src/lib/constants/icons.ts | 2 + tools/ui/src/lib/constants/index.ts | 1 + 9 files changed, 192 insertions(+), 21 deletions(-) create mode 100644 tools/ui/src/lib/constants/diagram-blocks.ts diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte index 7139d2e63..8ac7f9448 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/MarkdownContent.svelte @@ -41,6 +41,7 @@ DATA_ERROR_HANDLED_ATTR, BOOL_TRUE_STRING, SETTINGS_KEYS, + CODE_BLOCK_HEADER_CLASS, MERMAID_WRAPPER_CLASS, MERMAID_BLOCK_CLASS, MERMAID_LANGUAGE, @@ -53,7 +54,11 @@ SVG_TAG_PREFIX, SVG_SOURCE_ATTR, SVG_RENDERED_ATTR, - SVG_INLINE_SHADOW_STYLE + SVG_INLINE_SHADOW_STYLE, + TOGGLE_SOURCE_BTN_CLASS, + DIAGRAM_VIEW_MODE_ATTR, + DIAGRAM_VIEW_RENDERED, + DIAGRAM_VIEW_SOURCE } from '$lib/constants'; import { ColorMode, UrlProtocol } from '$lib/enums'; import { FileTypeText } from '$lib/enums/files.enums'; @@ -501,6 +506,23 @@ async function handleMermaidClick(event: MouseEvent) { const target = event.target as HTMLElement; + // Toggle a diagram block between its rendered view and its source view. + // Shared by mermaid and svg, css drives the visibility from the wrapper mode. + const toggleBtn = target.closest(`.${TOGGLE_SOURCE_BTN_CLASS}`); + if (toggleBtn) { + event.preventDefault(); + event.stopPropagation(); + + const wrapper = toggleBtn.closest(`.${MERMAID_WRAPPER_CLASS}, .${SVG_WRAPPER_CLASS}`); + if (!wrapper) return; + + const isSource = wrapper.getAttribute(DIAGRAM_VIEW_MODE_ATTR) === DIAGRAM_VIEW_SOURCE; + const next = isSource ? DIAGRAM_VIEW_RENDERED : DIAGRAM_VIEW_SOURCE; + wrapper.setAttribute(DIAGRAM_VIEW_MODE_ATTR, next); + toggleBtn.setAttribute('aria-pressed', String(!isSource)); + return; + } + // Check if clicking on copy or preview button in mermaid block const copyBtn = target.closest(`.${MERMAID_WRAPPER_CLASS} .copy-code-btn`); const previewBtn = target.closest(`.${MERMAID_WRAPPER_CLASS} .preview-code-btn`); @@ -573,6 +595,11 @@ } } + // A click on the header chrome targets the action buttons, never the + // diagram. Guard so a header click can not fall through to the click to + // zoom branches below, whatever the scroll position or stacking. + if (target.closest(`.${CODE_BLOCK_HEADER_CLASS}`)) return; + // Open preview when clicking the svg block itself. A final block carries its // source, a streaming block does not and is mirrored live into the dialog. const svgEl = target.closest(`.${SVG_BLOCK_CLASS}`); diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css index 072db383d..b0e04ca62 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/markdown-content.css @@ -300,7 +300,8 @@ div.markdown-user-content :global(.table-wrapper) { } .markdown-content :global(.copy-code-btn), -.markdown-content :global(.preview-code-btn) { +.markdown-content :global(.preview-code-btn), +.markdown-content :global(.toggle-source-btn) { display: flex; align-items: center; justify-content: center; @@ -312,15 +313,22 @@ div.markdown-user-content :global(.table-wrapper) { } .markdown-content :global(.copy-code-btn:hover), -.markdown-content :global(.preview-code-btn:hover) { +.markdown-content :global(.preview-code-btn:hover), +.markdown-content :global(.toggle-source-btn:hover) { transform: scale(1.05); } .markdown-content :global(.copy-code-btn:active), -.markdown-content :global(.preview-code-btn:active) { +.markdown-content :global(.preview-code-btn:active), +.markdown-content :global(.toggle-source-btn:active) { transform: scale(0.95); } +/* Pressed state marks the source view as active */ +.markdown-content :global(.toggle-source-btn[aria-pressed='true']) { + color: var(--primary); +} + .markdown-content :global(.code-block-wrapper pre) { background: transparent; margin: 0; @@ -629,8 +637,8 @@ div.markdown-user-content :global(.table-wrapper) { overflow-y: auto; overflow-x: auto; display: flex; - align-items: center; - justify-content: center; + align-items: safe center; + justify-content: safe center; padding: 3rem 1rem 1rem; } @@ -645,7 +653,9 @@ div.markdown-user-content :global(.table-wrapper) { overflow-y: visible; } -/* Diagram block uses same header styling as code blocks */ +/* Diagram block uses same header styling as code blocks. The header floats over + scrollable diagram content and stays transparent, so the overflow shows up to + the box edge. It keeps a z-index so it stays the click target above content. */ .markdown-content :global(.mermaid-block-wrapper .code-block-header), .markdown-content :global(.svg-block-wrapper .code-block-header) { display: flex; @@ -657,6 +667,7 @@ div.markdown-user-content :global(.table-wrapper) { top: 0; left: 0; right: 0; + z-index: 2; } .markdown-content :global(.mermaid-block-wrapper .code-block-actions), @@ -683,6 +694,31 @@ div.markdown-user-content :global(.table-wrapper) { padding: 3rem 1rem; } +/* Source view stays hidden while the block renders, css swaps the two views + from the wrapper mode so the click handler only flips one attribute. The view + reuses the code block scroll container, so it matches the app code blocks. */ +.markdown-content :global(.diagram-source) { + display: none; + text-align: left; +} + +.markdown-content :global(.diagram-source pre) { + background: transparent; + margin: 0; + border-radius: 0; + border: none; + font-size: 0.875rem; +} + +.markdown-content :global([data-view-mode='source'] .mermaid-scroll-container), +.markdown-content :global([data-view-mode='source'] .svg-scroll-container) { + display: none; +} + +.markdown-content :global([data-view-mode='source'] .diagram-source) { + display: block; +} + /* Streaming mermaid block - empty preview box */ .mermaid-streaming-block { min-height: 300px; diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/code-block-utils.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/code-block-utils.ts index 732315464..f1dd867e8 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/code-block-utils.ts +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/code-block-utils.ts @@ -7,12 +7,16 @@ import type { Element, ElementContent } from 'hast'; import { CODE_BLOCK_HEADER_CLASS, CODE_BLOCK_ACTIONS_CLASS, + CODE_BLOCK_SCROLL_CONTAINER_CLASS, CODE_LANGUAGE_CLASS, COPY_CODE_BTN_CLASS, PREVIEW_CODE_BTN_CLASS, + TOGGLE_SOURCE_BTN_CLASS, + DIAGRAM_SOURCE_CLASS, RELATIVE_CLASS, COPY_ICON_SVG, - PREVIEW_ICON_SVG + PREVIEW_ICON_SVG, + CODE_ICON_SVG } from '$lib/constants'; export interface BlockIdGenerator { @@ -32,14 +36,16 @@ export function createIconElement(svg: string): Element { } /** - * Creates a button element with icon. + * Creates a button element with icon. Extra properties merge onto the button, + * which lets a stateful button carry attributes like aria-pressed. */ export function createButton( className: string, title: string, iconSvg: string, id: string, - idAttribute: string + idAttribute: string, + extraProperties: Record = {} ): Element { return { type: 'element', @@ -48,7 +54,8 @@ export function createButton( className: [className], [idAttribute]: id, title, - type: 'button' + type: 'button', + ...extraProperties }, children: [createIconElement(iconSvg)] }; @@ -72,6 +79,52 @@ export function createPreviewButton( return createButton(PREVIEW_CODE_BTN_CLASS, title, PREVIEW_ICON_SVG, id, idAttribute); } +/** + * Creates a button that toggles a diagram block between its rendered view and + * its source view. aria-pressed starts false, the rendered view is the default. + */ +export function createToggleSourceButton( + id: string, + idAttribute: string, + title: string = 'Toggle source' +): Element { + return createButton(TOGGLE_SOURCE_BTN_CLASS, title, CODE_ICON_SVG, id, idAttribute, { + 'aria-pressed': 'false' + }); +} + +/** + * Creates a source view for a diagram block. It reuses the code block scroll + * container so it matches the app code blocks, and wraps the highlighted code + * element captured at transform time. A missing code element falls back to a + * plain code node built from the raw source. + */ +export function createSourceView( + codeElement: Element | undefined, + source: string, + language: string +): Element { + const code: Element = codeElement ?? { + type: 'element', + tagName: 'code', + properties: { className: ['hljs', `language-${language}`] }, + children: [{ type: 'text', value: source }] + }; + return { + type: 'element', + tagName: 'div', + properties: { className: [DIAGRAM_SOURCE_CLASS, CODE_BLOCK_SCROLL_CONTAINER_CLASS] }, + children: [ + { + type: 'element', + tagName: 'pre', + properties: {}, + children: [code] + } + ] + }; +} + /** * Creates a block header with language label and action buttons. */ @@ -116,14 +169,17 @@ export function createScrollContainer(preElement: Element, scrollContainerClass: } /** - * Creates a wrapper element with header and scroll container. + * Creates a wrapper element with header and scroll container. Extra children + * append after the scroll container, which lets a block carry a source view + * alongside its rendered output. */ export function createWrapper( header: Element, preElement: Element, wrapperClass: string, scrollContainerClass: string, - additionalAttributes?: Record + additionalAttributes?: Record, + extraChildren: Element[] = [] ): Element { return { type: 'element', @@ -132,7 +188,7 @@ export function createWrapper( className: [wrapperClass, RELATIVE_CLASS], ...additionalAttributes } as Element['properties'], - children: [header, createScrollContainer(preElement, scrollContainerClass)] + children: [header, createScrollContainer(preElement, scrollContainerClass), ...extraChildren] }; } diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-mermaid-blocks.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-mermaid-blocks.ts index f9decf206..4007c20a1 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-mermaid-blocks.ts +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-mermaid-blocks.ts @@ -19,12 +19,17 @@ import { MERMAID_BLOCK_CLASS, MERMAID_LANGUAGE, MERMAID_SYNTAX_ATTR, - MERMAID_ID_ATTR + MERMAID_ID_ATTR, + DIAGRAM_VIEW_MODE_ATTR, + DIAGRAM_VIEW_RENDERED } from '$lib/constants'; +import type { DiagramPreData } from './pre-transform'; import { createBlockHeader, createCopyButton, createPreviewButton, + createToggleSourceButton, + createSourceView, createWrapper, generateBlockId } from './code-block-utils'; @@ -75,16 +80,23 @@ export const rehypeEnhanceMermaidBlocks: Plugin<[], Root> = () => { const actions = [ createCopyButton(mermaidId, MERMAID_ID_ATTR, 'Copy mermaid syntax'), + createToggleSourceButton(mermaidId, MERMAID_ID_ATTR, 'Toggle mermaid source'), createPreviewButton(mermaidId, MERMAID_ID_ATTR, 'Preview diagram') ]; const header = createBlockHeader(MERMAID_LANGUAGE, mermaidId, MERMAID_ID_ATTR, actions); + const preservedCode = (node.data as DiagramPreData | undefined)?.sourceCode; + const sourceView = createSourceView(preservedCode, diagramText, MERMAID_LANGUAGE); const wrapper = createWrapper( header, node, MERMAID_WRAPPER_CLASS, MERMAID_SCROLL_CONTAINER_CLASS, - { [MERMAID_ID_ATTR]: mermaidId } + { + [MERMAID_ID_ATTR]: mermaidId, + [DIAGRAM_VIEW_MODE_ATTR]: DIAGRAM_VIEW_RENDERED + }, + [sourceView] ); // Replace pre with wrapper in parent diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-svg-blocks.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-svg-blocks.ts index e5e7514ed..55bcb6065 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-svg-blocks.ts +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/enhance-svg-blocks.ts @@ -18,12 +18,17 @@ import { SVG_BLOCK_CLASS, SVG_LANGUAGE, SVG_SOURCE_ATTR, - SVG_ID_ATTR + SVG_ID_ATTR, + DIAGRAM_VIEW_MODE_ATTR, + DIAGRAM_VIEW_RENDERED } from '$lib/constants'; +import type { DiagramPreData } from './pre-transform'; import { createBlockHeader, createCopyButton, createPreviewButton, + createToggleSourceButton, + createSourceView, createWrapper, generateBlockId } from './code-block-utils'; @@ -65,13 +70,24 @@ export const rehypeEnhanceSvgBlocks: Plugin<[], Root> = () => { const actions = [ createCopyButton(svgId, SVG_ID_ATTR, 'Copy svg source'), + createToggleSourceButton(svgId, SVG_ID_ATTR, 'Toggle svg source'), createPreviewButton(svgId, SVG_ID_ATTR, 'Preview svg') ]; const header = createBlockHeader(SVG_LANGUAGE, svgId, SVG_ID_ATTR, actions); - const wrapper = createWrapper(header, node, SVG_WRAPPER_CLASS, SVG_SCROLL_CONTAINER_CLASS, { - [SVG_ID_ATTR]: svgId - }); + const preservedCode = (node.data as DiagramPreData | undefined)?.sourceCode; + const sourceView = createSourceView(preservedCode, svgSource, SVG_LANGUAGE); + const wrapper = createWrapper( + header, + node, + SVG_WRAPPER_CLASS, + SVG_SCROLL_CONTAINER_CLASS, + { + [SVG_ID_ATTR]: svgId, + [DIAGRAM_VIEW_MODE_ATTR]: DIAGRAM_VIEW_RENDERED + }, + [sourceView] + ); // Replace pre with wrapper in parent (parent.children as ElementContent[])[index] = wrapper; diff --git a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/pre-transform.ts b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/pre-transform.ts index 06848eb26..7aa967bb8 100644 --- a/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/pre-transform.ts +++ b/tools/ui/src/lib/components/app/content/MarkdownContent/plugins/rehype/pre-transform.ts @@ -2,6 +2,15 @@ import type { Plugin } from 'unified'; import type { Root, Element, ElementContent, Text } from 'hast'; import { visit } from 'unist-util-visit'; +/** + * Metadata a diagram pre carries on its unist data field. The source code holds + * the highlighted code element captured before the pre became a render target, + * which the enhancer reuses to build a matching source view. + */ +export interface DiagramPreData { + sourceCode: Element; +} + /** * Recursively extracts all text content from a HAST node. * Handles nested elements (e.g., span wrappers from syntax highlighting). @@ -69,7 +78,10 @@ export function createPreTransform( properties: { className: [targetClass] }, - children: [{ type: 'text', value: text } as Text] + children: [{ type: 'text', value: text } as Text], + // Keep the highlighted code element so the block can offer a source + // view that matches the app code blocks without re highlighting. + data: { sourceCode: codeElement } satisfies DiagramPreData }; (parent.children as ElementContent[])[index] = pre; diff --git a/tools/ui/src/lib/constants/diagram-blocks.ts b/tools/ui/src/lib/constants/diagram-blocks.ts new file mode 100644 index 000000000..caeb6b5b3 --- /dev/null +++ b/tools/ui/src/lib/constants/diagram-blocks.ts @@ -0,0 +1,9 @@ +// Shared constants for diagram blocks (mermaid and svg) that toggle between a +// rendered view and a source view. The wrapper carries the active mode, css +// drives the visibility, the click handler only flips the attribute. + +export const DIAGRAM_VIEW_MODE_ATTR = 'data-view-mode'; +export const DIAGRAM_VIEW_RENDERED = 'rendered'; +export const DIAGRAM_VIEW_SOURCE = 'source'; +export const DIAGRAM_SOURCE_CLASS = 'diagram-source'; +export const TOGGLE_SOURCE_BTN_CLASS = 'toggle-source-btn'; diff --git a/tools/ui/src/lib/constants/icons.ts b/tools/ui/src/lib/constants/icons.ts index a9448c2a6..6ef02c4cb 100644 --- a/tools/ui/src/lib/constants/icons.ts +++ b/tools/ui/src/lib/constants/icons.ts @@ -39,3 +39,5 @@ export const MODALITY_LABELS = { export const COPY_ICON_SVG = ``; export const PREVIEW_ICON_SVG = ``; + +export const CODE_ICON_SVG = ``; diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index 07e441112..c51d84cdc 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -30,6 +30,7 @@ export * from './literal-html'; export * from './markdown'; export * from './mermaid-blocks'; export * from './svg-blocks'; +export * from './diagram-blocks'; export * from './max-bundle-size'; export * from './mcp'; export * from './mcp-form'; From 94653a9be4c66cf1263514cc96466075e7673584 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Tue, 16 Jun 2026 23:23:36 +0800 Subject: [PATCH 003/292] change test prompt for macos --- .github/workflows/kcpp-build-release-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kcpp-build-release-macos.yaml b/.github/workflows/kcpp-build-release-macos.yaml index bfed75028..61e7ea90d 100644 --- a/.github/workflows/kcpp-build-release-macos.yaml +++ b/.github/workflows/kcpp-build-release-macos.yaml @@ -43,7 +43,7 @@ jobs: id: test run: | wget https://huggingface.co/concedo/koboldcpp/resolve/main/baby_llama.gguf - dist/koboldcpp-mac-arm64 --model baby_llama.gguf --gpulayers 99 --benchmark --prompt 'Hi, my name is' + dist/koboldcpp-mac-arm64 --model baby_llama.gguf --gpulayers 99 --benchmark --prompt 'Once upon a' - name: Save artifact uses: actions/upload-artifact@v6 From 74ade52741203e5c8f81eaf06a96cb1cfe15f2a3 Mon Sep 17 00:00:00 2001 From: "Alessandro de Oliveira Faria (A.K.A.CABELO)" Date: Tue, 16 Jun 2026 15:24:28 -0300 Subject: [PATCH 004/292] vendor : update BoringSSL to 0.20260616.0 (#24693) --- vendor/cpp-httplib/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/cpp-httplib/CMakeLists.txt b/vendor/cpp-httplib/CMakeLists.txt index 5fb3cf8d5..408e06151 100644 --- a/vendor/cpp-httplib/CMakeLists.txt +++ b/vendor/cpp-httplib/CMakeLists.txt @@ -41,7 +41,7 @@ if (LLAMA_BUILD_BORINGSSL) set(FIPS OFF CACHE BOOL "Enable FIPS (BoringSSL)") set(BORINGSSL_GIT "https://boringssl.googlesource.com/boringssl" CACHE STRING "BoringSSL git repository") - set(BORINGSSL_VERSION "0.20260526.0" CACHE STRING "BoringSSL version") + set(BORINGSSL_VERSION "0.20260616.0" CACHE STRING "BoringSSL version") message(STATUS "Fetching BoringSSL version ${BORINGSSL_VERSION}") From 9b260fc9efff574394bd4948ffaf533ef1d00082 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 17 Jun 2026 07:54:21 +0200 Subject: [PATCH 005/292] sycl: Add optional USM system allocations (#22526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This introduces an optional feature to allocate large GPU buffers (≥ 1GB) using USM system allocations if supported by the device. It allows using buffers from the system allocator then letting the system manage memory migrations between host and device as necessary. This feature is disabled by default and requires the GGML_SYCL_USM_SYSTEM environment variable to enable. If USM system allocations are not supported by the device or the system, we fallback to regular allocations. This feature can allow VRAM overcommit. For example, the test below fails on B580 due to lack of memory for allocation, but it passes when enabling USM system allocations: ./examples/sycl/test.sh -m Qwen3.5-27B-Q3_K_M.gguf -lv 4 Signed-off-by: Francois Dugast --- docs/backend/SYCL.md | 1 + ggml/src/ggml-sycl/common.hpp | 1 + ggml/src/ggml-sycl/ggml-sycl.cpp | 92 +++++++++++++++++++++++--------- 3 files changed, 69 insertions(+), 25 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 18307d170..97d4b5216 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -720,6 +720,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | | UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS | 0 (default) or 1 | Allow SYCL/Unified Runtime Level Zero device allocations larger than 4 GiB. llama.cpp's direct Level Zero allocation path requests the relaxed maximum-size limit itself when GGML_SYCL_ENABLE_LEVEL_ZERO=1. | +| GGML_SYCL_USM_SYSTEM | 0 (default) or 1 | Enable experimental support for [USM system allocations](https://github.khronos.org/SYCL_Reference/iface/usm_basic_concept.html#system-allocations) for large GPU buffers. This requires enough host memory for model weights and caches, an Intel Xe2+ GPU such as BMG or newer and supported on Linux only, with CONFIG_DRM_XE_GPUSVM enabled. | ## Compile-time Flags diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 5fb1a1d6b..9ec94464b 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -230,6 +230,7 @@ struct sycl_device_info { size_t total_vram; sycl_hw_info hw_info; optimize_feature opt_feature; + bool usm_system_support; // support for USM system allocations }; diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 0900fade6..f029f6325 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -72,6 +72,9 @@ #include "ggml-sycl/gated_delta_net.hpp" #include "ggml-sycl/pool.hpp" +#define MEM_SIZE_2M 0x00200000 +#define MEM_SIZE_1G 0x40000000 + static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; int g_ggml_sycl_disable_optimize = 0; @@ -83,7 +86,7 @@ int g_ggml_sycl_use_async_mem_op = 0; int g_ggml_sycl_use_async_mem_op_requested = 1; int g_ggml_sycl_enable_level_zero = 0; int g_ggml_sycl_enable_flash_attention = 1; - +int g_ggml_sycl_usm_system = 0; static ggml_sycl_device_info ggml_sycl_init() { ggml_sycl_device_info info = {}; @@ -137,6 +140,7 @@ static ggml_sycl_device_info ggml_sycl_init() { info.devices[i].opt_feature.reorder = device.ext_oneapi_architecture_is(syclex::arch_category::intel_gpu); info.devices[i].smpbo = prop.get_local_mem_size(); info.devices[i].warp_size = WARP_SIZE; + info.devices[i].usm_system_support = device.has(sycl::aspect::usm_system_allocations); info.max_work_group_sizes[i] = prop.get_max_work_group_size(); info.devices[i].max_wg_per_cu = info.max_work_group_sizes[i] / prop.get_max_compute_units(); @@ -274,6 +278,8 @@ static void ggml_check_sycl() try { g_ggml_sycl_enable_flash_attention = 0; #endif + g_ggml_sycl_usm_system = ggml_sycl_get_env("GGML_SYCL_USM_SYSTEM", 0); + GGML_SYCL_DEBUG("[SYCL] call ggml_check_sycl\n"); GGML_LOG_INFO("Build with Macros:\n"); @@ -342,6 +348,8 @@ static void ggml_check_sycl() try { g_ggml_sycl_enable_flash_attention); #endif + GGML_LOG_INFO(" GGML_SYCL_USM_SYSTEM: %d\n", g_ggml_sycl_usm_system); + /* NOT REMOVE, keep it for next optimize for XMX. #if defined(SYCL_USE_XMX) fprintf(stderr, "%s: SYCL_USE_XMX: yes\n", __func__); @@ -417,6 +425,14 @@ catch (sycl::exception const &exc) { std::exit(1); } +inline void free_aligned_mem_host(void * memblock) { +#ifdef _WIN32 + _aligned_free(memblock); +#else + free(memblock); +#endif +} + // sycl buffer struct ggml_backend_sycl_buffer_context { @@ -426,9 +442,10 @@ struct ggml_backend_sycl_buffer_context { std::string name; optimize_feature opt_feature; std::vector tensor_extras; + bool is_usm_system; - ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream) : - device(device), dev_ptr(dev_ptr), stream(stream) { + ggml_backend_sycl_buffer_context(int device, void * dev_ptr, queue_ptr stream, bool is_usm_system) : + device(device), dev_ptr(dev_ptr), stream(stream), is_usm_system(is_usm_system) { check_allow_gpu_index(device); name = (GGML_SYCL_NAME + std::to_string(device)); opt_feature = ggml_sycl_info().devices[device].opt_feature; @@ -437,7 +454,10 @@ struct ggml_backend_sycl_buffer_context { ~ggml_backend_sycl_buffer_context() { if (dev_ptr != nullptr) { ggml_sycl_set_device(device); - SYCL_CHECK(CHECK_TRY_ERROR(ggml_sycl_free_device(dev_ptr, *stream))); + if (is_usm_system) + free_aligned_mem_host(dev_ptr); + else + SYCL_CHECK(CHECK_TRY_ERROR(ggml_sycl_free_device(dev_ptr, *stream))); } //release extra used by tensors @@ -759,21 +779,59 @@ static const char * ggml_backend_sycl_buffer_type_get_name(ggml_backend_buffer_t return ctx->name.c_str(); } +static bool check_usm_system(int device, size_t size) { + bool use_usm_system = g_ggml_sycl_usm_system && size >= MEM_SIZE_1G; + + if (use_usm_system && !ggml_sycl_info().devices[device].usm_system_support) { + GGML_LOG_INFO("Device does not support USM system allocations\n"); + use_usm_system = false; + } + + return use_usm_system; +} + +inline void * aligned_malloc_host(size_t alignment, size_t size) { +#ifdef _WIN32 + return _aligned_malloc(size, alignment); +#else + return aligned_alloc(alignment, size); +#endif +} + static ggml_backend_buffer_t ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) try { + ggml_check_sycl(); + ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; ggml_sycl_set_device(buft_ctx->device); const queue_ptr stream = buft_ctx->stream; size = std::max(size, (size_t)1); // syclMalloc returns null for size 0 + /* + Alignment below ensures best performance. While in theory it could lead to + wasting memory, this is acceptable because in practice only few buffers are + allocated and even less exceed the minimum size accepted here for USM system + allocations. + */ + size_t alignment = MEM_SIZE_2M; + size_t aligned_size = ((size + alignment - 1) / alignment) * alignment; + bool use_usm_system = check_usm_system(buft_ctx->device, aligned_size); void * dev_ptr; - SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream))); - if (!dev_ptr) { - GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, size); - return nullptr; + if (use_usm_system) { + dev_ptr = (void *)aligned_malloc_host(alignment, aligned_size); + if (!dev_ptr) { + GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on host\n", __func__, size); + return nullptr; + } + } else { + SYCL_CHECK(CHECK_TRY_ERROR(dev_ptr = (void *)ggml_sycl_malloc_device(size, *stream))); + if (!dev_ptr) { + GGML_LOG_ERROR("%s: can't allocate %lu Bytes of memory on device\n", __func__, size); + return nullptr; + } } - ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(buft_ctx->device, dev_ptr, buft_ctx->stream); + ggml_backend_sycl_buffer_context * ctx = new ggml_backend_sycl_buffer_context(buft_ctx->device, dev_ptr, buft_ctx->stream, use_usm_system); return ggml_backend_buffer_init(buft, ggml_backend_sycl_buffer_interface, ctx, size); } catch (sycl::exception const &exc) { @@ -1300,22 +1358,6 @@ static const char * ggml_backend_sycl_host_buffer_type_name(ggml_backend_buffer_ GGML_UNUSED(buft); } -inline void * aligned_malloc_host(size_t alignment, size_t size) { -#ifdef _WIN32 - return _aligned_malloc(size, alignment); -#else - return aligned_alloc(alignment, size); -#endif -} - -inline void free_aligned_mem_host(void * memblock) { -#ifdef _WIN32 - _aligned_free(memblock); -#else - free(memblock); -#endif -} - static void ggml_backend_sycl_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { free_aligned_mem_host((void *)buffer->context); } From ebbc1e51c136093f887bfc36e337520da352ee22 Mon Sep 17 00:00:00 2001 From: Alexey Kopytko Date: Wed, 17 Jun 2026 14:57:29 +0900 Subject: [PATCH 006/292] SYCL: fix use-after-free bug with async memcpy in MoE prefill (#24676) * SYCL: fix a bug with async memcpy * make mmid_row_mapping_host persistent * comment on stream->wait * Apply suggestion from @sanmai * Apply suggestion from @sanmai * Apply suggestion from @sanmai --- ggml/src/ggml-sycl/common.hpp | 7 +++++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 9 +++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 9ec94464b..96586ea46 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -324,6 +324,11 @@ void ggml_sycl_free_device(void *ptr, sycl::queue &q); void release_extra_gpu(ggml_tensor_extra_gpu * extra, std::vector streams={}); +struct mmid_row_mapping { + int32_t i1; + int32_t i2; +}; + namespace sycl_ex = sycl::ext::oneapi::experimental; struct ggml_backend_sycl_context { int device; @@ -421,6 +426,8 @@ struct ggml_backend_sycl_context { std::unique_ptr host_pools[GGML_SYCL_MAX_DEVICES]; + std::vector mmid_row_mapping_host; + static std::unique_ptr new_pool_for_device(queue_ptr qptr, int device); static std::unique_ptr new_pool_for_host(queue_ptr qptr, int device); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index f029f6325..376d4376f 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4224,11 +4224,6 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor } -struct mmid_row_mapping { - int32_t i1; - int32_t i2; -}; - __dpct_inline__ static void k_copy_src1_to_contiguous( const char *__restrict__ src1_original, char *__restrict__ src1_contiguous, const mmid_row_mapping *__restrict__ row_mapping, @@ -4399,6 +4394,8 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, SYCL_CHECK(CHECK_TRY_ERROR( stream->memcpy(ids_host.data(), ids_dev, ggml_nbytes(ids)))); + + // also ensures ctx.mmid_row_mapping_host is drained before we use it again SYCL_CHECK(CHECK_TRY_ERROR(stream->wait())); ggml_tensor src0_row = *src0; @@ -4456,7 +4453,7 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx, // where each expert's slice starts and the previous ends (row indices, right-exclusive) std::vector expert_row_offsets; // the sources (slot/token pairs) of contiguous rows to guide k_copy_src1_to_contiguous - std::vector routed_row_src; + std::vector & routed_row_src = ctx.mmid_row_mapping_host; mmid_counting_sort_rows(ids, ids_host.data(), n_ids, n_as, n_routed_rows, expert_row_counts, expert_row_offsets, routed_row_src); From 58728bdbf0421f3f6d4407436ccbadf0ee90ab53 Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Wed, 17 Jun 2026 13:58:03 +0800 Subject: [PATCH 007/292] sycl : Enable to support fp16 by OPs: SQR, SQRT, LOG, SIN, COS, CLAMP (#24692) --- docs/ops.md | 12 ++++----- docs/ops/SYCL.csv | 44 ++++++++++++++++---------------- ggml/src/ggml-sycl/ggml-sycl.cpp | 5 ---- 3 files changed, 28 insertions(+), 33 deletions(-) diff --git a/docs/ops.md b/docs/ops.md index fe74df805..d46f9b731 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -23,7 +23,7 @@ Legend: | ARGMAX | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | ARGSORT | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ✅ | ❌ | ❌ | | CEIL | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | -| CLAMP | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | ❌ | ❌ | +| CLAMP | ❌ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ❌ | ❌ | | COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | CONCAT | ❌ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | CONT | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ | @@ -32,7 +32,7 @@ Legend: | CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | +| COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | | COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | | CROSS_ENTROPY_LOSS | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | @@ -65,7 +65,7 @@ Legend: | IM2COL_3D | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | L2_NORM | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | LEAKY_RELU | ❌ | ✅ | ✅ | ✅ | 🟡 | ❌ | ✅ | 🟡 | ❌ | ❌ | ❌ | -| LOG | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | +| LOG | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | MEAN | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | | MUL | ❌ | ✅ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | MUL_MAT | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | @@ -99,13 +99,13 @@ Legend: | SIGMOID | ❌ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | SILU | ❌ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | SILU_BACK | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| SIN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | +| SIN | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | | SOFTPLUS | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | SOFT_MAX | ❌ | 🟡 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | SOFT_MAX_BACK | ❌ | ❌ | 🟡 | 🟡 | ❌ | ❌ | 🟡 | ✅ | ❌ | ❌ | ❌ | | SOLVE_TRI | ❌ | ❌ | ✅ | 🟡 | ✅ | ❌ | 🟡 | ✅ | ✅ | ❌ | ❌ | -| SQR | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ❌ | -| SQRT | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | ❌ | ❌ | +| SQR | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | +| SQRT | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | 🟡 | ✅ | ❌ | ❌ | | SSM_CONV | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | SSM_SCAN | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | 🟡 | 🟡 | ✅ | ❌ | ❌ | | STEP | ❌ | ✅ | ✅ | 🟡 | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | diff --git a/docs/ops/SYCL.csv b/docs/ops/SYCL.csv index 9ff664a5b..19a5346f8 100644 --- a/docs/ops/SYCL.csv +++ b/docs/ops/SYCL.csv @@ -9743,29 +9743,29 @@ "SYCL0","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=1","support","1","yes","SYCL" "SYCL0","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=32","support","1","yes","SYCL" "SYCL0","ADD_ID","type_a=f32,type_b=f32,n_embd=129,n_experts=8,n_experts_used=4,n_token=129","support","1","yes","SYCL" -"SYCL0","SQR","type=f16,ne=[10,5,4,3]","support","0","no","SYCL" -"SYCL0","SQRT","type=f16,ne=[10,3,3,2]","support","0","no","SYCL" -"SYCL0","LOG","type=f16,ne=[10,5,4,3]","support","0","no","SYCL" -"SYCL0","SIN","type=f16,ne=[10,2,2,2]","support","0","no","SYCL" -"SYCL0","COS","type=f16,ne=[10,2,2,2]","support","0","no","SYCL" -"SYCL0","CLAMP","type=f16,ne=[10,5,4,3],min=-0.500000,max=0.500000","support","0","no","SYCL" +"SYCL0","SQR","type=f16,ne=[10,5,4,3]","support","1","yes","SYCL" +"SYCL0","SQRT","type=f16,ne=[10,3,3,2]","support","1","yes","SYCL" +"SYCL0","LOG","type=f16,ne=[10,5,4,3]","support","1","yes","SYCL" +"SYCL0","SIN","type=f16,ne=[10,2,2,2]","support","1","yes","SYCL" +"SYCL0","COS","type=f16,ne=[10,2,2,2]","support","1","yes","SYCL" +"SYCL0","CLAMP","type=f16,ne=[10,5,4,3],min=-0.500000,max=0.500000","support","1","yes","SYCL" "SYCL0","LEAKY_RELU","type=f16,ne_a=[10,5,4,3],negative_slope=0.100000","support","1","yes","SYCL" "SYCL0","FLOOR","type=f16,ne=[10,2,2,2]","support","1","yes","SYCL" "SYCL0","CEIL","type=f16,ne=[10,2,2,2]","support","1","yes","SYCL" "SYCL0","ROUND","type=f16,ne=[10,2,2,2]","support","1","yes","SYCL" "SYCL0","TRUNC","type=f16,ne=[10,2,2,2]","support","1","yes","SYCL" -"SYCL0","SQR","type=f16,ne=[7,1,5,3]","support","0","no","SYCL" -"SYCL0","SQR","type=f16,ne=[1024,1024,1,1]","support","0","no","SYCL" -"SYCL0","SQRT","type=f16,ne=[7,1,5,3]","support","0","no","SYCL" -"SYCL0","SQRT","type=f16,ne=[1024,1024,1,1]","support","0","no","SYCL" -"SYCL0","LOG","type=f16,ne=[7,1,5,3]","support","0","no","SYCL" -"SYCL0","LOG","type=f16,ne=[1024,1024,1,1]","support","0","no","SYCL" -"SYCL0","SIN","type=f16,ne=[7,1,5,3]","support","0","no","SYCL" -"SYCL0","SIN","type=f16,ne=[1024,1024,1,1]","support","0","no","SYCL" -"SYCL0","COS","type=f16,ne=[7,1,5,3]","support","0","no","SYCL" -"SYCL0","COS","type=f16,ne=[1024,1024,1,1]","support","0","no","SYCL" -"SYCL0","CLAMP","type=f16,ne=[7,1,5,3],min=-0.500000,max=0.500000","support","0","no","SYCL" -"SYCL0","CLAMP","type=f16,ne=[1024,1024,1,1],min=-0.500000,max=0.500000","support","0","no","SYCL" +"SYCL0","SQR","type=f16,ne=[7,1,5,3]","support","1","yes","SYCL" +"SYCL0","SQR","type=f16,ne=[1024,1024,1,1]","support","1","yes","SYCL" +"SYCL0","SQRT","type=f16,ne=[7,1,5,3]","support","1","yes","SYCL" +"SYCL0","SQRT","type=f16,ne=[1024,1024,1,1]","support","1","yes","SYCL" +"SYCL0","LOG","type=f16,ne=[7,1,5,3]","support","1","yes","SYCL" +"SYCL0","LOG","type=f16,ne=[1024,1024,1,1]","support","1","yes","SYCL" +"SYCL0","SIN","type=f16,ne=[7,1,5,3]","support","1","yes","SYCL" +"SYCL0","SIN","type=f16,ne=[1024,1024,1,1]","support","1","yes","SYCL" +"SYCL0","COS","type=f16,ne=[7,1,5,3]","support","1","yes","SYCL" +"SYCL0","COS","type=f16,ne=[1024,1024,1,1]","support","1","yes","SYCL" +"SYCL0","CLAMP","type=f16,ne=[7,1,5,3],min=-0.500000,max=0.500000","support","1","yes","SYCL" +"SYCL0","CLAMP","type=f16,ne=[1024,1024,1,1],min=-0.500000,max=0.500000","support","1","yes","SYCL" "SYCL0","LEAKY_RELU","type=f16,ne_a=[7,1,5,3],negative_slope=0.100000","support","1","yes","SYCL" "SYCL0","LEAKY_RELU","type=f16,ne_a=[1024,1024,1,1],negative_slope=0.100000","support","1","yes","SYCL" "SYCL0","FLOOR","type=f16,ne=[7,1,5,3]","support","1","yes","SYCL" @@ -11046,8 +11046,8 @@ "SYCL0","ARGSORT","type=f32,ne=[8192,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16383,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16384,1,1,1],order=0","support","1","yes","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","0","no","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","0","no","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","1","yes","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65535,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65536,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[131071,1,1,1],order=0","support","0","no","SYCL" @@ -11095,8 +11095,8 @@ "SYCL0","ARGSORT","type=f32,ne=[8192,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16383,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16384,1,1,1],order=0","support","1","yes","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","0","no","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","0","no","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","1","yes","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65535,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65536,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[131071,1,1,1],order=0","support","0","no","SYCL" diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 376d4376f..057e0dccd 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -5582,11 +5582,6 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_COS: case GGML_OP_CLAMP: case GGML_OP_LOG: -#if defined (GGML_SYCL_F16) - return ((op->type == GGML_TYPE_F32 || op->type == GGML_SYCL_F16) && (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_SYCL_F16) && (op->type == op->src[0]->type)); -#else - return (op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32) && (op->type == op->src[0]->type); -#endif case GGML_OP_NORM: case GGML_OP_L2_NORM: case GGML_OP_GROUP_NORM: From 890f1a27ed4810d9655fc3f748d9b6bc483932fb Mon Sep 17 00:00:00 2001 From: Zijun Yu Date: Wed, 17 Jun 2026 14:11:21 +0800 Subject: [PATCH 008/292] openvino: OV 2026.2, context-shift, Q5_1 support, gemma4 dense/embedding, and -fa off (#24503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add interface is_model_splitted() to check the c-graph is splited or not * Infer and propagate dynamic-dimension indices for all tensors in the GGML graph in api compute_model_outputs() * Only do this for fallback sub graph * Move dynamic dims compute in graph missmatch * ggml-openvino: fix tensor data handling for PERMUTE/VIEW ops in split models * ggml-openvino:add comments * ggml-openvino: override VIEW op_case to 0 for split model inputs * openvino backend: Handle unsupported VIEW shape-mismatch in OpenVINO backend * Enable additional mul_mat tests and add tensor data saving function (#81) * ggml-openvino: fix CONT/TRANSPOSE mapping and improve dynamic-dimension handling * OpenVINO: add NORM/TANH support and rework SOFT_MAX translation * ggml-openvino: extend VIEW handling * Enable -fa off (#118) * Enable --context-shift * Fix llm param compute error for normal softmax not the softmax in attention * OpenVINO backend: fix error for attention size compute in llm param * use tensor->extra in infer_request i/o * OpenVINO backend: refacter the compute_llm_params() func add get_attention_pattern_case to easy extand * OpenVINO backend: clean unused code * 1to1 match op update (#146) * added translate_1to1_match_1_input function and updated gelu and tanh translations * Remove unused translation function calls --------- Co-authored-by: Mustafa Cavus * initial gemma4 support * removed hardcoded names for kv cache slicing * OpenVINO backend: Add new attention pattern for llm parameters compute * flash attn Q shape static conversion * Remove slice in permute translation when n_seq is 1 * return optional in extract_layer_from_name * OpenVINO backend: refactor VIEW related operation (#148) * OpenVINO backend: refactor VIEW related operation * Enable VIEW handling in following ops * OpenVINO backend does not support GGML_OP_NORM & GGML_OP_L2_NORM with VIEW input accuracy issue from OpenVINO * OpenVINO backend: Add ops l2_norm & pad * OpenVINO backend does not support CPY with non-contiguous data or mismatched types * add op SSM_CONV GATED_DELTA_NET * OpenVINO backend: fix error for bf16 in OV gpu plugin * reverted static Q input shape for attention layer * OpenVINO backend: remove hardcode name inp_tokens, which ignore some leaf case * Disable remote tensor due to bug in ov gpu * Disable n_token > 1 GATED_DELTA_NET on gpu * OpenVINO backend: fix the view op dynamic handling issue in gemma4 & enable view + get_row * OpenVINO backend: clean code * OpenVINO backend: enable view + norm/rms_norm * OpenVINO backend: concat op * OpenVINO backend: argsort op * OpenVINO backend: enable unary + view & GGML_UNARY_OP_SOFTPLUS * Fix issue for test-backend-ops in TOPK_MOE, which compare VIEW ops result, VIEW node in OpenVINO no need compare, the whole graph result is correct * OpenVINO backend: enable sum_rows * OpenVINO backend: enable clamp * OpenVINO backend: enable DIV * OpenVINO backend: enable GGML_OP_MUL_MAT_ID * OpenVINO backend: disable MUL_MAT_ID_FUSION case with large mem needed * OpenVINO backend: Disable GGML_OP_ARGSORT, cause test_backend-ops failed * OpenVINO backend: fix issue in mul_mat_id * OpenVINO backend: Disable DIV with broadcast on GPU * OpenVINO backend: update DIV * use ov internal op GatedDeltaNet * OpenVINO backend: enable llama erch test qwen3next * OpenVINO backend: enable RMS_NORM + VIEW & remove op_case 2 for rope * OpenVINO backend: fix error * suggested changes, need review * suggested changes, need review * OpenVINO backend: clean unused code & fix build warning * OpenVINO backend: enable minicpm3 for arch test * Disable GDN op (#177) * disable gated_delta_net * update stateful_kv_size correctly in mismatch case * OpenVINO backend: enable arch test for qwen3vl * OpenVINO backend: enable cohere2 for arch test * OpenVINO backend: enable t5 for arch test * OpenVINO backend: enable jamba for arch test * OpenVINO backend: remove warning for tmp * OpenVINO backend: enable kimi-linear for arch test * Remove unused * Fix gpt-oss accuracy issue * OpenVINO backend: enable arctic for arch test * OpenVINO backend: enable grok for arch test * Gemma4 initial npu support (#179) * Initiall gemma4 npu support * temp. fix for gemma4 accuracy bug on npu * Remove hardcoded names for npu-fold handling * revert static n tokens for cont translation as it is not needed * removed unused variable * ggml-openvino: add GGML_OPENVINO_ENABLE_CACHE env var to control decoder cache. Add environment variable GGML_OPENVINO_ENABLE_CACHE (default: YES). When set to NO, the decoder_cache is bypassed and models are rebuilt from the cgraph on every inference call in both dynamic and static compute paths. This is useful for debugging and verifying correctness without caching interference. * Revert "Gemma4 initial npu support (#179)" This reverts commit 0d29a9c4a52dc2c8aa52990f1a3854cfb01768ad. * OpenVINO backend: disable debug log print * Update TBB discovery. Delegated to OpenVINOs own config. * OpenVINO backend: GGML_OPENVINO_ENABLE_CACHE YES -> 1 * OpenVINO backend: fallback FLASH_ATTN_EXT in gemma3n to CPU backend * Add raw ov infer profiling metric * Add OV raw infer time metric to static compute path Co-authored-by: virajwad <84867530+virajwad@users.noreply.github.com> * Modify precision of static profiling * update to OV 2026.2, add OV windows CI * fix editorconfig-checks * Initiall gemma4 npu support * temp. fix for gemma4 accuracy bug on npu * Remove hardcoded names for npu-fold handling * revert static n tokens for cont translation as it is not needed * removed unused variable * test-llama-archs fix * Fix gemma4 flash_attn fallback * support im2col * fix code style * disable add_rope_sin_cos optimization * stateless boradcast and rope optimizations * Enable manual gqa attn by default for stateless gpu * manual gqa: fixed static batch * gemma4 llama-bench ctx update fix * Update OV win CI * stateful rope fusion temp. fix * OpenVINO backend: Conslolidate supported ops * Exclude unsupported GGML_OP_SUB cases * Exclude unsupported TOPK_MOE cases * OpenVINO Backend: MUL_MAT enhancements * Update OV CI * support f16 mask input for npu * Make GGML_OPENVINO_* env vars usage uniform Standardize all GGML_OPENVINO_* env flags: positive integers >0 to enable. Unset, empty, =0, or non-numeric values to disable. This fixes cases where text values or empty strings enabled features. * OpenVINO backend: Enhance envvar handling * more cleanup * move ggml_openvino_env_flag to appropriate place * OpenVINO backend: add REPEAT translator, Q5_1 weights, and GLU view-input fix * ggml-openvino: fix -Werror=cast-qual in extract_q5_1_data * Update openvino.Dockerfile Use BuildKit cache mounts for faster Docker rebuilds. Use apt instead of dpkg, remove unused .ddeb downloads, add DLLAMA_BUILD_TESTS=OFF. * ggml-openvino: centralize env var access via *getenv_str/getenv_int helpers Replace getenv and legacy flags with _str and _int helpers.Minor cleanup, doc updates. * OpenVINO backend: Enable GGML_OP_ADD_ID * Uptade openvino backend clamg-format * clang-format * Update OPENVINO.md (#211) * OpenVINO backend: fix accuracy issue for op CONCAT with i64 precision * Remove strict concurrency for gpu-openvino-low-perf * Update openvino CI keynames; add ccache-clear * Apply suggestions from code review Co-authored-by: Sigbjørn Skjæret <1629204+CISC@users.noreply.github.com> * Fix formatting --------- Co-authored-by: Xuejun Zhai Co-authored-by: Mustafa Cavus Co-authored-by: Mustafa Cavus Co-authored-by: Xuejun Co-authored-by: Wang Yang Co-authored-by: Ravi Panchumarthy Co-authored-by: virajwad <84867530+virajwad@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Mostafa Faheem Co-authored-by: Sigbjørn Skjæret <1629204+CISC@users.noreply.github.com> --- .devops/openvino.Dockerfile | 109 ++- .../actions/windows-setup-openvino/action.yml | 24 + .github/workflows/build-cache.yml | 32 +- .github/workflows/build-openvino.yml | 89 +- .github/workflows/build-self-hosted.yml | 8 +- .github/workflows/release.yml | 110 ++- docs/backend/OPENVINO.md | 667 +++++++++++--- ggml/src/ggml-openvino/.clang-format | 5 - ggml/src/ggml-openvino/CMakeLists.txt | 6 +- ggml/src/ggml-openvino/ggml-decoder.cpp | 868 ++++++++++++++++-- ggml/src/ggml-openvino/ggml-decoder.h | 99 +- .../src/ggml-openvino/ggml-openvino-extra.cpp | 60 +- ggml/src/ggml-openvino/ggml-openvino-extra.h | 37 +- ggml/src/ggml-openvino/ggml-openvino.cpp | 376 ++++++-- ggml/src/ggml-openvino/ggml-quants.cpp | 66 ++ ggml/src/ggml-openvino/ggml-quants.h | 14 +- ggml/src/ggml-openvino/openvino/decoder.h | 72 +- ggml/src/ggml-openvino/openvino/frontend.h | 2 +- ggml/src/ggml-openvino/openvino/input_model.h | 8 +- .../src/ggml-openvino/openvino/node_context.h | 133 ++- ggml/src/ggml-openvino/openvino/op/add_id.cpp | 62 ++ .../src/ggml-openvino/openvino/op/argsort.cpp | 47 + ggml/src/ggml-openvino/openvino/op/clamp.cpp | 33 + ggml/src/ggml-openvino/openvino/op/concat.cpp | 48 + ggml/src/ggml-openvino/openvino/op/cont.cpp | 24 +- ggml/src/ggml-openvino/openvino/op/cpy.cpp | 15 +- ggml/src/ggml-openvino/openvino/op/div.cpp | 146 +++ .../openvino/op/flash_attn_ext.cpp | 129 ++- .../openvino/op/gated_delta_net.cpp | 282 ++++++ .../openvino/op/gated_delta_net.hpp | 65 ++ .../ggml-openvino/openvino/op/get_rows.cpp | 11 +- .../ggml-openvino/openvino/op/glu_geglu.cpp | 28 +- .../ggml-openvino/openvino/op/glu_swiglu.cpp | 17 +- ggml/src/ggml-openvino/openvino/op/im2col.cpp | 120 +++ .../src/ggml-openvino/openvino/op/l2_norm.cpp | 44 + .../ggml-openvino/openvino/op/mul_mat_id.cpp | 108 +++ ggml/src/ggml-openvino/openvino/op/mulmat.cpp | 28 +- ggml/src/ggml-openvino/openvino/op/norm.cpp | 58 ++ ggml/src/ggml-openvino/openvino/op/pad.cpp | 95 ++ .../src/ggml-openvino/openvino/op/permute.cpp | 71 +- ggml/src/ggml-openvino/openvino/op/repeat.cpp | 74 ++ .../src/ggml-openvino/openvino/op/reshape.cpp | 19 +- .../ggml-openvino/openvino/op/rms_norm.cpp | 2 +- ggml/src/ggml-openvino/openvino/op/rope.cpp | 166 +++- .../ggml-openvino/openvino/op/set_rows.cpp | 6 +- .../src/ggml-openvino/openvino/op/softmax.cpp | 121 +-- .../ggml-openvino/openvino/op/ssm_conv.cpp | 59 ++ .../ggml-openvino/openvino/op/sum_rows.cpp | 27 + .../ggml-openvino/openvino/op/transpose.cpp | 33 +- .../ggml-openvino/openvino/op/unary_gelu.cpp | 25 - .../ggml-openvino/openvino/op/unary_silu.cpp | 2 +- .../openvino/op/unary_softplus.cpp | 38 + ggml/src/ggml-openvino/openvino/op/view.cpp | 117 ++- ggml/src/ggml-openvino/openvino/op_table.cpp | 63 +- ggml/src/ggml-openvino/openvino/op_table.h | 22 +- ...k_decompression_convert_constant_folding.h | 2 +- .../openvino/translate_session.cpp | 83 +- .../openvino/translate_session.h | 9 +- ggml/src/ggml-openvino/openvino/utils.cpp | 551 ++++++++++- ggml/src/ggml-openvino/openvino/utils.h | 54 +- ggml/src/ggml-openvino/utils.cpp | 481 ++++++++-- ggml/src/ggml-openvino/utils.h | 19 +- 62 files changed, 5280 insertions(+), 909 deletions(-) create mode 100644 .github/actions/windows-setup-openvino/action.yml create mode 100644 ggml/src/ggml-openvino/openvino/op/add_id.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/argsort.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/clamp.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/concat.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/div.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp create mode 100644 ggml/src/ggml-openvino/openvino/op/im2col.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/l2_norm.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/norm.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/pad.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/repeat.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/sum_rows.cpp delete mode 100644 ggml/src/ggml-openvino/openvino/op/unary_gelu.cpp create mode 100644 ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp diff --git a/.devops/openvino.Dockerfile b/.devops/openvino.Dockerfile index e85c585e6..9e96244ce 100644 --- a/.devops/openvino.Dockerfile +++ b/.devops/openvino.Dockerfile @@ -1,17 +1,17 @@ -ARG OPENVINO_VERSION_MAJOR=2026.0 -ARG OPENVINO_VERSION_FULL=2026.0.0.20965.c6d6a13a886 +ARG OPENVINO_VERSION_MAJOR=2026.2 +ARG OPENVINO_VERSION_FULL=2026.2.0.21903.52ddc073857 ARG UBUNTU_VERSION=24.04 # Intel GPU driver versions. https://github.com/intel/compute-runtime/releases -ARG IGC_VERSION=v2.30.1 -ARG IGC_VERSION_FULL=2_2.30.1+20950 -ARG COMPUTE_RUNTIME_VERSION=26.09.37435.1 -ARG COMPUTE_RUNTIME_VERSION_FULL=26.09.37435.1-0 -ARG IGDGMM_VERSION=22.9.0 +ARG IGC_VERSION=v2.34.4 +ARG IGC_VERSION_FULL=2_2.34.4+21428 +ARG COMPUTE_RUNTIME_VERSION=26.18.38308.1 +ARG COMPUTE_RUNTIME_VERSION_FULL=26.18.38308.1-0 +ARG IGDGMM_VERSION=22.10.0 # Intel NPU driver versions. https://github.com/intel/linux-npu-driver/releases -ARG NPU_DRIVER_VERSION=v1.32.0 -ARG NPU_DRIVER_FULL=v1.32.0.20260402-23905121947 +ARG NPU_DRIVER_VERSION=v1.33.0 +ARG NPU_DRIVER_FULL=v1.33.0.20260529-26625960453 ARG LIBZE1_VERSION=1.27.0-1~24.04~ppa2 # Optional proxy build arguments @@ -46,13 +46,18 @@ RUN apt-get update && \ intel-opencl-icd && \ rm -rf /var/lib/apt/lists/* -# Install OpenVINO for Ubuntu 24.04 +# OpenVINO toolkit and GPU/NPU drivers are cached via BuildKit cache mounts to avoid re-downloading on rebuilds. +# Install OpenVINO for Ubuntu 24.04. ARG OPENVINO_VERSION_MAJOR ARG OPENVINO_VERSION_FULL -RUN mkdir -p /opt/intel && \ - wget https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \ - tar -xf openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \ - mv openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64 /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \ +RUN --mount=type=cache,target=/var/cache/openvino,sharing=locked \ + mkdir -p /opt/intel && \ + TGZ=/var/cache/openvino/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz && \ + if [ ! -f "$TGZ" ]; then \ + wget -O "$TGZ" https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz; \ + fi && \ + tar -xf "$TGZ" -C /opt/intel/ && \ + mv /opt/intel/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64 /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \ cd /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} && \ echo "Y" | ./install_dependencies/install_openvino_dependencies.sh && \ cd - && \ @@ -68,14 +73,14 @@ COPY . . RUN bash -c "source ${OpenVINO_DIR}/setupvars.sh && \ cmake -B build/ReleaseOV -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ + -DLLAMA_BUILD_TESTS=OFF \ -DGGML_OPENVINO=ON && \ - cmake --build build/ReleaseOV -j$(nproc)" + cmake --build build/ReleaseOV --parallel " -# Copy all necessary libraries +# Copy all necessary libraries (build outputs + OpenVINO runtime libs) RUN mkdir -p /app/lib && \ - find build/ReleaseOV -name '*.so*' -exec cp {} /app/lib \; && \ - find ${OpenVINO_DIR}/runtime/lib/intel64 -name '*.so*' -exec cp -P {} /app/lib \; 2>/dev/null || \ - find ${OpenVINO_DIR}/lib/intel64 -name '*.so*' -exec cp -P {} /app/lib \; + find build/ReleaseOV -name '*.so*' -exec cp -P {} /app/lib \; && \ + find "${OpenVINO_DIR}/runtime/lib/intel64" -name '*.so*' -exec cp -P {} /app/lib \; # Create runtime directories and copy binaries RUN mkdir -p /app/full \ @@ -120,33 +125,41 @@ ARG IGC_VERSION_FULL ARG COMPUTE_RUNTIME_VERSION ARG COMPUTE_RUNTIME_VERSION_FULL ARG IGDGMM_VERSION -RUN mkdir /tmp/neo/ && cd /tmp/neo/ \ - && wget https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \ - && wget https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc-dbgsym_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.ddeb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd-dbgsym_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.ddeb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libigdgmm12_${IGDGMM_VERSION}_amd64.deb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1-dbgsym_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.ddeb \ - && wget https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \ - && dpkg --install *.deb \ - && rm -rf /tmp/neo/ +RUN --mount=type=cache,target=/var/cache/intel-gpu,sharing=locked \ + set -eux; \ + cd /var/cache/intel-gpu; \ + for url in \ + https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-core-${IGC_VERSION_FULL}_amd64.deb \ + https://github.com/intel/intel-graphics-compiler/releases/download/${IGC_VERSION}/intel-igc-opencl-${IGC_VERSION_FULL}_amd64.deb \ + https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-ocloc_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \ + https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/intel-opencl-icd_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb \ + https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libigdgmm12_${IGDGMM_VERSION}_amd64.deb \ + https://github.com/intel/compute-runtime/releases/download/${COMPUTE_RUNTIME_VERSION}/libze-intel-gpu1_${COMPUTE_RUNTIME_VERSION_FULL}_amd64.deb ; do \ + f=$(basename "$url"); \ + [ -f "$f" ] || wget -q -O "$f" "$url"; \ + done; \ + apt-get update; \ + apt-get install -y --no-install-recommends ./*.deb; \ + rm -rf /var/lib/apt/lists/* # Install NPU drivers ARG NPU_DRIVER_VERSION ARG NPU_DRIVER_FULL ARG LIBZE1_VERSION -RUN mkdir /tmp/npu/ && cd /tmp/npu/ \ - && wget https://github.com/intel/linux-npu-driver/releases/download/${NPU_DRIVER_VERSION}/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz \ - && tar -xf linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz \ - && dpkg --install *.deb \ - && rm -rf /tmp/npu/ - -RUN cd /tmp \ - && wget https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260324T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb \ - && dpkg --install libze1_${LIBZE1_VERSION}_amd64.deb \ - && rm libze1_${LIBZE1_VERSION}_amd64.deb +RUN --mount=type=cache,target=/var/cache/intel-npu,sharing=locked \ + set -eux; \ + TGZ=/var/cache/intel-npu/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz; \ + if [ ! -f "$TGZ" ]; then \ + wget -q -O "$TGZ" https://github.com/intel/linux-npu-driver/releases/download/${NPU_DRIVER_VERSION}/linux-npu-driver-${NPU_DRIVER_FULL}-ubuntu2404.tar.gz; \ + fi; \ + DEB=/var/cache/intel-npu/libze1_${LIBZE1_VERSION}_amd64.deb; \ + if [ ! -f "$DEB" ]; then \ + wget -q -O "$DEB" https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260324T100000Z/pool/main/l/level-zero-loader/libze1_${LIBZE1_VERSION}_amd64.deb; \ + fi; \ + mkdir /tmp/npu/ && cd /tmp/npu/ && tar -xf "$TGZ" && cp "$DEB" .; \ + apt-get update; \ + apt-get install -y --no-install-recommends ./*.deb; \ + rm -rf /tmp/npu/ /var/lib/apt/lists/* COPY --from=build /app/lib/ /app/ @@ -166,22 +179,26 @@ RUN apt-get update && \ python3 \ python3-venv \ python3-pip && \ - python3 -m venv /ov-venv && \ - /ov-venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \ - /ov-venv/bin/pip install --no-cache-dir -r requirements.txt && \ + python3 -m venv /openvino-venv && \ + /openvino-venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \ + /openvino-venv/bin/pip install --no-cache-dir -r requirements.txt && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /tmp/* /var/tmp/* && \ find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete && \ find /var/cache -type f -delete -ENTRYPOINT ["/bin/bash", "-c", "source /ov-venv/bin/activate && exec /app/tools.sh \"$@\"", "--"] +# Activate the venv +ENV VIRTUAL_ENV=/openvino-venv \ + PATH=/openvino-venv/bin:$PATH + +ENTRYPOINT ["/app/tools.sh"] ### Light, CLI only FROM base AS light -COPY --from=build /app/full/llama-cli /app/ +COPY --from=build /app/full/llama-cli /app/full/llama-completion /app/ WORKDIR /app diff --git a/.github/actions/windows-setup-openvino/action.yml b/.github/actions/windows-setup-openvino/action.yml new file mode 100644 index 000000000..f983df560 --- /dev/null +++ b/.github/actions/windows-setup-openvino/action.yml @@ -0,0 +1,24 @@ +name: "Windows - Setup OpenVINO Toolkit" +description: "Setup OpenVINO Toolkit for Windows" +inputs: + path: + description: "Installation path" + required: true + version_major: + description: "OpenVINO major version (e.g., 2026.2)" + required: true + version_full: + description: "OpenVINO full version" + required: true + +runs: + using: "composite" + steps: + - name: Download and extract OpenVINO Runtime + shell: powershell + run: | + $url = "https://storage.openvinotoolkit.org/repositories/openvino/packages/${{ inputs.version_major }}/windows/openvino_toolkit_windows_${{ inputs.version_full }}_x86_64.zip" + $out = "openvino.zip" + Invoke-WebRequest -Uri $url -OutFile $out + Expand-Archive -Path $out -DestinationPath ${{ inputs.path }} -Force + Remove-Item $out diff --git a/.github/workflows/build-cache.yml b/.github/workflows/build-cache.yml index 53d65f376..b36c6e1ea 100644 --- a/.github/workflows/build-cache.yml +++ b/.github/workflows/build-cache.yml @@ -68,8 +68,8 @@ jobs: env: # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.0" - OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886" + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" steps: - name: Clone @@ -91,6 +91,34 @@ jobs: version_major: ${{ env.OPENVINO_VERSION_MAJOR }} version_full: ${{ env.OPENVINO_VERSION_FULL }} + windows-2022-openvino-cache: + runs-on: windows-2022 + + env: + # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + windows-2022-rocm-cache: runs-on: windows-2022 diff --git a/.github/workflows/build-openvino.yml b/.github/workflows/build-openvino.yml index ddcbc6697..49ab13695 100644 --- a/.github/workflows/build-openvino.yml +++ b/.github/workflows/build-openvino.yml @@ -37,14 +37,10 @@ jobs: ubuntu-24-openvino: runs-on: [self-hosted, Linux, Intel, OpenVINO] - concurrency: - group: openvino-gpu-${{ github.head_ref || github.ref }} - cancel-in-progress: false - env: # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.0" - OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886" + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" steps: - name: Clone @@ -78,7 +74,7 @@ jobs: cmake -B build/ReleaseOV -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DGGML_OPENVINO=ON - time cmake --build build/ReleaseOV --config Release -j $(nproc) + time cmake --build build/ReleaseOV --config Release --parallel - name: Test (CPU) id: cmake_test_cpu @@ -93,4 +89,81 @@ jobs: run: | cd ${{ github.workspace }} export GGML_OPENVINO_DEVICE=GPU - ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 2000 + ctest --test-dir build/ReleaseOV -L main -E "test-llama-archs" --verbose --timeout 3000 + + openvino-windows-2022: + runs-on: windows-2022 + + env: + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + + steps: + - name: Clone + id: checkout + uses: actions/checkout@v6 + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: openvino-windows-2022 + variant: ccache + evict-old-files: 1d + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenCL using vcpkg + shell: powershell + run: | + git clone https://github.com/microsoft/vcpkg C:\vcpkg + C:\vcpkg\bootstrap-vcpkg.bat + C:\vcpkg\vcpkg install opencl + + - name: Build + id: cmake_build + shell: cmd + run: | + REM Find extracted OpenVINO folder dynamically + for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i + + if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" ( + echo ERROR: OpenVINOConfig.cmake not found + exit /b 1 + ) + + call "%OPENVINO_ROOT%\setupvars.bat" + + cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^ + -A x64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_OPENVINO=ON ^ + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake + + cmake --build build\ReleaseOV --config Release -- /m + + - name: Test (CPU) + id: cmake_test_cpu + shell: cmd + # TODO: fix and re-enable the `test-llama-archs` test below + run: | + REM Find extracted OpenVINO folder dynamically + for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i + call "%OPENVINO_ROOT%\setupvars.bat" + + cd build + ctest --test-dir ReleaseOV -L main -E "test-llama-archs" -C Release --verbose --timeout 3000 diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 436100c8a..c4366ece3 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -264,14 +264,10 @@ jobs: gpu-openvino-low-perf: runs-on: [self-hosted, Linux, Intel, OpenVINO] - concurrency: - group: openvino-gpu-${{ github.head_ref || github.ref }} - cancel-in-progress: false - env: # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.0" - OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886" + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" steps: - name: Clone diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9c8fda51b..d5bcb4d4e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -443,9 +443,9 @@ jobs: openvino_version: ${{ steps.openvino_version.outputs.value }} env: - # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile - OPENVINO_VERSION_MAJOR: "2026.0" - OPENVINO_VERSION_FULL: "2026.0.0.20965.c6d6a13a886" + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" steps: - name: Set OpenVINO version output @@ -528,6 +528,108 @@ jobs: path: llama-${{ steps.tag.outputs.name }}-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz name: llama-bin-ubuntu-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.tar.gz + windows-openvino: + runs-on: windows-2022 + + outputs: + openvino_version: ${{ steps.openvino_version.outputs.value }} + + env: + # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile + OPENVINO_VERSION_MAJOR: "2026.2" + OPENVINO_VERSION_FULL: "2026.2.0.21903.52ddc073857" + + steps: + - name: Set OpenVINO version output + id: openvino_version + run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT + + - name: Clone + id: checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: "npm" + cache-dependency-path: "tools/ui/package-lock.json" + + - name: ccache + uses: ggml-org/ccache-action@v1.2.21 + with: + key: release-windows-2022-openvino + variant: ccache + evict-old-files: 1d + + - name: Setup Cache + uses: actions/cache@v5 + id: cache-openvino + with: + path: ./openvino_toolkit + key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }} + + - name: Setup OpenVINO Toolkit + if: steps.cache-openvino.outputs.cache-hit != 'true' + uses: ./.github/actions/windows-setup-openvino + with: + path: ./openvino_toolkit + version_major: ${{ env.OPENVINO_VERSION_MAJOR }} + version_full: ${{ env.OPENVINO_VERSION_FULL }} + + - name: Install OpenCL using vcpkg + shell: powershell + run: | + git clone https://github.com/microsoft/vcpkg C:\vcpkg + C:\vcpkg\bootstrap-vcpkg.bat + C:\vcpkg\vcpkg install opencl + + - name: Build + id: cmake_build + shell: cmd + run: | + REM Find extracted OpenVINO folder dynamically + for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i + + if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" ( + echo ERROR: OpenVINOConfig.cmake not found + exit /b 1 + ) + + call "%OPENVINO_ROOT%\setupvars.bat" + + cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^ + -A x64 ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_OPENVINO=ON ^ + -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake + + cmake --build build\ReleaseOV --config Release -- /m + + - name: ccache-clear + uses: ./.github/actions/ccache-clear + with: + key: release-windows-2022-openvino + + - name: Determine tag name + id: tag + uses: ./.github/actions/get-tag-name + + - name: Pack artifacts + id: pack_artifacts + shell: powershell + run: | + Copy-Item LICENSE .\build\ReleaseOV\bin\ + 7z a -snl llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip .\build\ReleaseOV\bin\* + + - name: Upload artifacts + uses: actions/upload-artifact@v6 + with: + path: llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip + name: llama-bin-win-openvino-${{ env.OPENVINO_VERSION_MAJOR }}-x64.zip + windows-cpu: needs: [check-release] if: ${{ needs.check-release.outputs.should_release == 'true' }} @@ -1403,6 +1505,7 @@ jobs: - windows-cuda #- windows-sycl - windows-hip + - windows-openvino - ubuntu-22-rocm - ubuntu-cpu - ubuntu-vulkan @@ -1524,6 +1627,7 @@ jobs: - [Windows x64 (CUDA 12)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-12.4-x64.zip) - [CUDA 12.4 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-12.4-x64.zip) - [Windows x64 (CUDA 13)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-cuda-13.3-x64.zip) - [CUDA 13.3 DLLs](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/cudart-llama-bin-win-cuda-13.3-x64.zip) - [Windows x64 (Vulkan)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-vulkan-x64.zip) + - [Windows x64 (OpenVINO)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-openvino-${{ needs.windows-openvino.outputs.openvino_version }}-x64.zip) - [Windows x64 (SYCL)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-sycl-x64.zip) - [Windows x64 (HIP)](https://github.com/ggml-org/llama.cpp/releases/download/${{ steps.tag.outputs.name }}/llama-${{ steps.tag.outputs.name }}-bin-win-hip-radeon-x64.zip) diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index b0e19abb0..631d4bc3b 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -12,6 +12,25 @@ The OpenVINO backend is implemented in `ggml/src/ggml-openvino` and provides a t - Compiles and caches the model for the target device. - Binds GGML tensor memory to OpenVINO inference tensors and runs inference. +## Contents + +- [Supported Devices](#supported-devices) +- [Supported Model Precisions](#supported-model-precisions) +- [Supported Llama.cpp Tools](#supported-llamacpp-tools) +- [Validated Models](#validated-models) +- [Build Instructions](#build-instructions) + - [0. Prerequisites](#0-prerequisites) + - [1. Install OpenVINO Runtime](#1-install-openvino-runtime) + - [2. Build llama.cpp with OpenVINO Backend](#2-build-llamacpp-with-openvino-backend) + - [Automated Ubuntu Build Script](#automated-ubuntu-build-script) + - [Automated Windows Build Script](#automated-windows-build-script) + - [3. Download Sample Model](#3-download-sample-model) + - [4. Run Inference with OpenVINO Backend](#4-run-inference-with-openvino-backend) + - [5. Docker Build](#5-docker-build) +- [GGML OpenVINO Backend Runtime Configurations](#ggml-openvino-backend-runtime-configurations) +- [Known Limitations](#known-limitations) +- [Work in Progress](#work-in-progress) + ## Supported Devices OpenVINO backend supports the following hardware: @@ -31,55 +50,102 @@ Although OpenVINO supports a wide range of [Intel hardware](https://docs.openvin - `Q4_1` - `Q4_K` - `Q4_K_M` -- `Q5_K` (converted to Q8_0_C at runtime) -- `Q6_K` (converted to Q8_0_C at runtime) +- `Q5_K` (converted to `Q8_0_C` at runtime) +- `Q6_K` (converted to `Q8_0_C` at runtime) > [!NOTE] > Accuracy validation and performance optimizations for quantized models are a work in progress. -## Quantization Support Details - -### CPU and GPU - -- **`Q4_0`, `Q4_1`, `Q4_K_M`, `Q6_K` models are supported** +**CPU and GPU Quantization Details:** - `Q5_K` and `Q6_K` tensors are converted to `Q8_0_C` -### NPU - -- **Primary supported quantization scheme is `Q4_0`** +**NPU Quantization Details:** +- Primary supported quantization scheme is `Q4_0` - `Q6_K` tensors are requantized to `Q4_0_128` in general. For embedding weights, `Q6_K` tensors are requantized to `Q8_0_C` except for the token embedding matrix which is dequantized to fp16 -### Additional Notes - +**Additional Notes:** - Both `Q4_0` and `Q4_1` models use `Q6_K` for the token embedding tensor and the final matmul weight tensor (often the same tensor) - `Q4_0` models may produce some `Q4_1` tensors if an imatrix is provided during quantization using `llama-quantize` - `Q4_K_M` models may include both `Q6_K` and `Q5_K` tensors (observed in Phi-3) +- `Q5_1` tensors are dequantized natively (weights, scales, and zero-points extracted directly) + +## Supported Llama.cpp Tools + +The OpenVINO backend integrates with the standard llama.cpp tools listed below. +However, all the tools coverage across all devices is not uniform and exhaustive validation is work in progress. + +- llama-bench +- llama-cli +- llama-completion +- llama-embedding +- llama-perplexity +- llama-run +- llama-server +- llama-simple ## Validated Models -The following models were validated on Intel® Core™ Ultra Series 2. While our testing was limited, the OpenVINO backend is expected to work across a broad range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html). -- Use `GGML_OPENVINO_STATEFUL_EXECUTION=1` when using GPU device. -- `-fa 1` is required when running llama-bench with the OpenVINO backend. -- Additional model support, quantization formats and validations are work in progress. +Although, the validated models below were tested with `llama-cli` using the `Q4_K_M` quantization format on Intel® Core™ Ultra Series 2 (Lunar Lake), the OpenVINO backend is expected to work across a broader range of [Intel hardware](https://docs.openvino.ai/2026/about-openvino/release-notes-openvino/system-requirements.html), [supported model precisions](#supported-model-precisions), [supported llama.cpp tools](#supported-llamacpp-tools) and additional model architectures. -| Model | Validated | Known Issues | -| :------| :---------- | :-------------| -| [Llama-3.2-1B-Instruct](https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — | -| [Meta-Llama-3.1-8B-Instruct](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | `Q8_0`, `Q4_K_M` on CPU/GPU/NPU | `Q4_0_8_8`, `Q4_0_4_8`, `Q4_0_4_4` fail | -| [Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf) | `FP16`, `Q4` on CPU/NPU | GPU unsupported for `FP16` and `Q4` (`llama-cli`, `llama-bench`) | -| [Qwen2.5-1.5B-Instruct](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — | -| [Qwen3-8B-Instruct](https://huggingface.co/Qwen/Qwen3-8B-GGUF) | `FP16`, `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/NPU; GPU works via `llama-bench` | GPU `llama-cli` unsupported for all quantizations | -| [MiniCPM-V-2_6-GGUF](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | `Q4_0` on CPU/GPU/NPU | — | -| [DeepSeek-R1-Distill-Llama-8B](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M` on CPU/GPU/NPU | — | -| [Hunyuan-7B-Instruct](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | CPU: `Q8_0`, `Q4_0`, `Q4_1`, `Q4_K_M`; GPU: `Q8_0`, `Q4_0`, `Q4_1`; NPU (`llama-bench` only): `Q4_0`, `Q4_1`, `Q4_K_M` | GPU `Q4_K_M` unsupported; NPU `llama-cli` unsupported | -| [Mistral-7B-Instruct-v0.3](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF/) | CPU/GPU: `Q8_0`, `Q4_K_M`; NPU: `Q8_0`, `Q4_K_M` (via `llama-bench`) | NPU `llama-cli` unsupported for `Q8_0`, `Q4_K_M` | +> [!NOTE] +> Extensive accuracy validation, performance optimizations, and broader architecture coverage are work in progress. + +**Legend & Test Configuration:** +- **Status:** ✓ = Passed | ✗ = Failed or Unsupported +- **Execution Modes:** + - **SL** = Stateless (`GGML_OPENVINO_STATEFUL_EXECUTION=0`) + - **SF** = Stateful (`GGML_OPENVINO_STATEFUL_EXECUTION=1`) + - Note: The NPU operates in stateless mode only. +- **Validation system:** Intel® Core™ Ultra 5 238V (Lunar Lake) | 32 GB RAM | Ubuntu 24.04 | Intel OpenCL GPU Driver 26.18.38308.1 | Intel NPU Driver 1.33.0. +- See [Known Limitations](#known-limitations) for context on observed failures. + +| Model | CPU (SL / SF) | GPU (SL / SF) | NPU (SL) | +| :--- | :---: | :---: | :---: | +| [bartowski/Llama-3.2-1B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [bartowski/Llama-3.2-3B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Llama-3.2-3B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [bartowski/Meta-Llama-3.1-8B-Instruct-Q4_K_M](https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| | | | | +| [Qwen/qwen2.5-1.5b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-1.5B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [Qwen/qwen2.5-coder-7b-instruct-q4_k_m](https://huggingface.co/Qwen/Qwen2.5-Coder-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [bartowski/Qwen_Qwen3-0.6B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-0.6B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ | +| | | | | +| [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ | +| [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✓ | +| [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✗ | ✓ / ✗ | ✗ | +| | | | | +| [bartowski/Phi-3-mini-4k-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3-mini-4k-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [bartowski/Phi-3.5-mini-instruct-Q4_K_M](https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| | | | | +| [bartowski/Mistral-7B-Instruct-v0.3-Q4_K_M](https://huggingface.co/bartowski/Mistral-7B-Instruct-v0.3-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [QuantFactory/Ministral-3b-instruct.Q4_K_M](https://huggingface.co/QuantFactory/Ministral-3b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [bartowski/Ministral-8B-Instruct-2410-Q4_K_M](https://huggingface.co/bartowski/Ministral-8B-Instruct-2410-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| | | | | +| [bartowski/DeepSeek-R1-Distill-Llama-8B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Llama-8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [bartowski/DeepSeek-R1-Distill-Qwen-7B-Q4_K_M](https://huggingface.co/bartowski/DeepSeek-R1-Distill-Qwen-7B-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| | | | | +| [ibm-granite/granite-4.0-350m-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-350m-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ | +| [ibm-granite/granite-4.0-micro-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-micro-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [ibm-granite/granite-4.0-1b-Q4_K_M](https://huggingface.co/ibm-granite/granite-4.0-1b-GGUF) | ✓ / ✓ | ✗ / ✗ | ✗ | +| [ibm-research/granite-3.2-8b-instruct-Q4_K_M](https://huggingface.co/ibm-research/granite-3.2-8b-instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| | | | | +| [HuggingFaceTB/smollm2-1.7b-instruct-q4_k_m](https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | +| [openbmb/MiniCPM-V-2_6-Q4_K_M](https://huggingface.co/openbmb/MiniCPM-V-2_6-gguf) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [bartowski/tencent_Hunyuan-7B-Instruct-Q4_K_M](https://huggingface.co/bartowski/tencent_Hunyuan-7B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-Q4_K_M](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-7.8B-Instruct-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| [bartowski/prism-ml_Bonsai-8B-unpacked-Q4_K_M](https://huggingface.co/bartowski/prism-ml_Bonsai-8B-unpacked-GGUF) | ✓ / ✓ | ✓ / ✗ | ✓ | +| | | | | +| [gpustack/bge-m3-Q4_K_M.gguf](https://huggingface.co/gpustack/bge-m3-GGUF) | ✓ | ✗ | ✗ | ## Build Instructions -### Prerequisites +### 0. Prerequisites - Linux or Windows system with Intel hardware (CPU, GPU, or NPU) -- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2025/get-started/install-openvino/configurations.html). +- **For Intel GPU or NPU Usage**: Install the appropriate hardware drivers for your Intel GPU or NPU. For detailed instructions, see: [Additional Configurations for Hardware Acceleration](https://docs.openvino.ai/2026/get-started/install-openvino/configurations.html). - **Linux:** - Git, CMake, and Ninja software tools are needed for building. @@ -119,28 +185,14 @@ The following models were validated on Intel® Core™ Ultra Series 2. While our - Follow the guide to install OpenVINO Runtime from an archive file: [Linux](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-linux.html) | [Windows](https://docs.openvino.ai/2026/get-started/install-openvino/install-openvino-archive-windows.html) -- **Linux:** - -
- 📦 Click to expand OpenVINO installation from an archive file on Ubuntu -
- - ```bash - wget https://raw.githubusercontent.com/ravi9/misc-scripts/main/openvino/ov-archive-install/install-openvino-from-archive.sh - chmod +x install-openvino-from-archive.sh - ./install-openvino-from-archive.sh - ``` - - Verify OpenVINO is initialized properly: - ```bash - echo $OpenVINO_DIR - ``` -
- +- Verify OpenVINO is initialized properly: + ```bash + echo $OpenVINO_DIR + ``` ### 2. Build llama.cpp with OpenVINO Backend -Clone the OpenVINO-enabled llama.cpp fork and build it: +Clone llama.cpp repo and build : ```bash git clone https://github.com/ggml-org/llama.cpp @@ -148,39 +200,375 @@ cd llama.cpp ``` - **Linux:** - ```bash - source /opt/intel/openvino/setupvars.sh - cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON - cmake --build build/ReleaseOV --parallel - ``` +```bash +source /opt/intel/openvino/setupvars.sh +cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON +cmake --build build/ReleaseOV --parallel +``` + +- **Windows:** Open a **Developer Command Prompt for VS 2022** (so the MSVC toolchain is on `PATH`), then run: + +```cmd +C:\Intel\openvino\setupvars.bat +cmake -B build\ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake +cmake --build build\ReleaseOV --parallel +``` -- **Windows:** - ```cmd - # x64 Native Tools Command Prompt for VS 2022 - "C:\Program Files (x86)\Intel\openvino_2026.0\setupvars.bat" - cmake -B build\ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON -DLLAMA_CURL=OFF -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake - cmake --build build\ReleaseOV --parallel - ``` > [!NOTE] -> Use `x64 Native Tools Command Prompt` for Windows build. After building, you could use either `cmd` or `PowerShell` to run the OpenVINO backend. +> The Windows install path is `C:\Intel\openvino` (no spaces) to avoid quoting problems some CMake/Ninja toolchains have with `C:\Program Files (x86)\...`. Adjust to wherever you installed OpenVINO Runtime. From `cmd`, run `C:\Intel\openvino\setupvars.bat`; from PowerShell, run `& "C:\Intel\openvino\setupvars.ps1"` instead. Once the build is finished you can launch the binaries from any `cmd` or `PowerShell` window after sourcing the matching `setupvars` script for that shell. + +#### Automated Ubuntu Build Script + +For Ubuntu24 users, the following shell script automates the prerequisite installs (build tools, OpenCL ICD), the OpenVINO Runtime download/extract/setup, and the Ninja-based llama.cpp build. +Save the following as `ubuntu-llamacpp-ov-install.sh` next to where you want the `llama.cpp` folder to land, then run it: + +```bash +chmod +x ubuntu-llamacpp-ov-install.sh +./ubuntu-llamacpp-ov-install.sh +``` + +
+Click to expand ubuntu-llamacpp-ov-install.sh + +```bash +#!/usr/bin/env bash +# ============================================ +# llama.cpp OpenVINO Build Script (Ninja) +# ============================================ +set -euo pipefail + +OPENVINO_VERSION_MAJOR="2026.2" +OPENVINO_VERSION_FULL="2026.2.0.21903.52ddc073857" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OPENVINO_INSTALL_DIR="/opt/intel/openvino_${OPENVINO_VERSION_MAJOR}" +OPENVINO_LINK_DIR="/opt/intel/openvino" +OPENVINO_TGZ="${SCRIPT_DIR}/openvino.tgz" +OPENVINO_URL="https://storage.openvinotoolkit.org/repositories/openvino/packages/${OPENVINO_VERSION_MAJOR}/linux/openvino_toolkit_ubuntu24_${OPENVINO_VERSION_FULL}_x86_64.tgz" + +echo "============================================" +echo "Installing prerequisites (apt)..." +echo "============================================" +sudo apt-get update +sudo apt-get install -y \ + build-essential libcurl4-openssl-dev libtbb12 \ + cmake ninja-build python3-pip \ + curl wget tar git + +echo "============================================" +echo "Installing OpenCL runtime + headers..." +echo "============================================" +sudo apt-get install -y \ + ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd + +cd "${SCRIPT_DIR}" + +# ============================================ +# Clone llama.cpp if missing +# ============================================ +if [[ ! -f "llama.cpp/CMakeLists.txt" ]]; then + echo "Cloning llama.cpp..." + git clone https://github.com/ggml-org/llama.cpp +fi + +# ============================================ +# Setup OpenVINO: download & extract to /opt/intel/openvino_${OPENVINO_VERSION_MAJOR}, +# then point /opt/intel/openvino at it via symlink so the active version is swappable. +# ============================================ +if [[ -f "${OPENVINO_INSTALL_DIR}/setupvars.sh" ]]; then + echo "OpenVINO ${OPENVINO_VERSION_MAJOR} already installed at ${OPENVINO_INSTALL_DIR}. Skipping download." +else + echo "OpenVINO not found at ${OPENVINO_INSTALL_DIR}. Starting download..." + curl -L -o "${OPENVINO_TGZ}" "${OPENVINO_URL}" + + echo "Extracting OpenVINO to ${OPENVINO_INSTALL_DIR}..." + sudo mkdir -p "${OPENVINO_INSTALL_DIR}" + sudo tar -xzf "${OPENVINO_TGZ}" -C "${OPENVINO_INSTALL_DIR}" --strip-components=1 + rm -f "${OPENVINO_TGZ}" +fi + +# Refresh symlink: /opt/intel/openvino -> /opt/intel/openvino_${OPENVINO_VERSION_MAJOR} +sudo ln -sfn "${OPENVINO_INSTALL_DIR}" "${OPENVINO_LINK_DIR}" + +OPENVINO_ROOT="${OPENVINO_LINK_DIR}" +echo "OpenVINO Ready: ${OPENVINO_ROOT} -> ${OPENVINO_INSTALL_DIR}" + +# Install OpenVINO's own runtime dependencies (one-time per system). +if [[ -x "${OPENVINO_ROOT}/install_dependencies/install_openvino_dependencies.sh" ]]; then + echo "============================================" + echo "Installing OpenVINO runtime dependencies..." + echo "============================================" + echo "Y" | sudo -E "${OPENVINO_ROOT}/install_dependencies/install_openvino_dependencies.sh" +fi + +# ============================================ +# Clean old build cache +# ============================================ +cd "${SCRIPT_DIR}/llama.cpp" +if [[ -d "build/ReleaseOV" ]]; then + echo "Removing old build directory..." + rm -rf "build/ReleaseOV" +fi + +echo "============================================" +echo "Configuring with CMake..." +echo "============================================" +# shellcheck disable=SC1091 +source "${OPENVINO_ROOT}/setupvars.sh" + +cmake -B build/ReleaseOV -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_OPENVINO=ON + +cmake --build build/ReleaseOV --parallel + +echo "============================================" +echo "Build completed successfully!" +echo "============================================" +echo "Binaries: $(pwd)/build/ReleaseOV/bin" +echo +echo "NOTE: To run, source setupvars.sh and pick a device:" +echo " source /opt/intel/openvino/setupvars.sh" +echo " export GGML_OPENVINO_DEVICE=CPU # or GPU / NPU" +echo " ./build/ReleaseOV/bin/llama-cli -m model.gguf" +``` + +> [!NOTE] +> The script pins OpenVINO `2026.2` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. + +
+ +#### Automated Windows Build Script + +For Windows users, the following `.bat` script automates the prerequisite installs (Git, Ninja, CMake, Visual Studio 2022 Build Tools, vcpkg + OpenCL), the OpenVINO Runtime download/extract, and the Ninja-based llama.cpp build. +Save the following as `windows-llamacpp-ov-install.bat` next to where you want the `llama.cpp` to land, then run it from either **Command Prompt** or **PowerShell**: + +```cmd +:: Command Prompt +windows-llamacpp-ov-install.bat +``` + +```powershell +# PowerShell +.\windows-llamacpp-ov-install.bat +``` + +
+Click to expand windows-llamacpp-ov-install.bat + +```bat +@echo off +setlocal enabledelayedexpansion + +REM ============================================ +REM llama.cpp OpenVINO Build Script (Ninja) +REM ============================================ + +set "OPENVINO_VERSION_MAJOR=2026.2" +set "OPENVINO_VERSION_FULL=2026.2.0.21903.52ddc073857" + +set "SCRIPT_DIR=%~dp0" +set "VCPKG_DIR=C:\vcpkg" +set "OPENVINO_INSTALL_DIR=C:\Intel\openvino_%OPENVINO_VERSION_MAJOR%" +set "OPENVINO_LINK_DIR=C:\Intel\openvino" +set "OPENVINO_ZIP=%SCRIPT_DIR%openvino.zip" +set "OPENVINO_EXTRACT_TMP=%SCRIPT_DIR%openvino_extract_tmp" +set "OPENVINO_URL=https://storage.openvinotoolkit.org/repositories/openvino/packages/%OPENVINO_VERSION_MAJOR%/windows/openvino_toolkit_windows_%OPENVINO_VERSION_FULL%_x86_64.zip" + +echo ============================================ +echo Installing prerequisites... +echo ============================================ +winget install --id Git.Git -e --accept-source-agreements --accept-package-agreements 2>nul +winget install --id Ninja-build.Ninja -e --accept-source-agreements --accept-package-agreements 2>nul +winget install --id Kitware.CMake -e --accept-source-agreements --accept-package-agreements 2>nul + +REM Ensure Visual Studio Build Tools are installed. +echo Checking for Visual Studio Build Tools... +set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +set "VS_INSTALLED=" +if exist "%VSWHERE%" ( + for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath 2^>nul`) do ( + set "VS_INSTALLED=%%i" + ) +) +if defined VS_INSTALLED ( + echo Visual Studio with VC++ x86/x64 tools already present at "!VS_INSTALLED!". Skipping winget install. +) else ( + winget install --id Microsoft.VisualStudio.2022.BuildTools -e --override "--wait --passive --add Microsoft.VisualStudio.Workload.VCTools --includeRecommended" --accept-source-agreements --accept-package-agreements + if errorlevel 1 ( + echo WARNING: winget could not install Visual Studio Build Tools automatically. + echo Install manually from https://aka.ms/vs/17/release/vs_BuildTools.exe ^(select the "Desktop development with C++" workload^) + echo and re-run this script from a "Developer Command Prompt for VS 2022". + ) +) + +echo ============================================ +echo Installing OpenCL via vcpkg... +echo ============================================ +if not exist "%VCPKG_DIR%" ( + git clone https://github.com/microsoft/vcpkg "%VCPKG_DIR%" + cd /d "%VCPKG_DIR%" + call bootstrap-vcpkg.bat + call vcpkg integrate install +) +cd /d "%VCPKG_DIR%" +call vcpkg install opencl + +cd /d "%SCRIPT_DIR%" + +REM ============================================ +REM Clone llama.cpp if missing +REM ============================================ +if not exist "llama.cpp\CMakeLists.txt" ( + echo Cloning llama.cpp... + git clone https://github.com/ggml-org/llama.cpp +) + +cd /d "llama.cpp" +set "SCRIPT_DIR=%CD%" + +REM ============================================ +REM Setup OpenVINO: download & extract to C:\Intel\openvino_%OPENVINO_VERSION_MAJOR%, +REM then point C:\Intel\openvino at it via a directory junction (mklink /J). +REM ============================================ + +if exist "%OPENVINO_INSTALL_DIR%\setupvars.bat" ( + echo OpenVINO %OPENVINO_VERSION_MAJOR% already installed at "%OPENVINO_INSTALL_DIR%". Skipping download. +) else ( + echo OpenVINO not found at "%OPENVINO_INSTALL_DIR%". Starting download... + + curl -L -o "%OPENVINO_ZIP%" "%OPENVINO_URL%" + if errorlevel 1 ( + echo ERROR: Download failed. + exit /b 1 + ) + + echo Extracting OpenVINO... + if exist "%OPENVINO_EXTRACT_TMP%" rmdir /s /q "%OPENVINO_EXTRACT_TMP%" + mkdir "%OPENVINO_EXTRACT_TMP%" + tar -xf "%OPENVINO_ZIP%" -C "%OPENVINO_EXTRACT_TMP%" + if errorlevel 1 ( + echo ERROR: Extraction failed. + exit /b 1 + ) + + REM Move the single top-level folder contents into the versioned install dir. + REM NOTE: delayed expansion (!VAR!) is required because the surrounding else( ... ) + REM block is parsed once up-front, so %OPENVINO_EXTRACTED% would expand to "" here + REM and xcopy would then treat "\*" as C:\* and fail with "Cannot perform a cyclic copy". + set "OPENVINO_EXTRACTED=" + for /d %%i in ("%OPENVINO_EXTRACT_TMP%\*") do set "OPENVINO_EXTRACTED=%%i" + if not defined OPENVINO_EXTRACTED ( + echo ERROR: Could not locate extracted OpenVINO folder under "%OPENVINO_EXTRACT_TMP%". + exit /b 1 + ) + if not exist "%OPENVINO_INSTALL_DIR%" mkdir "%OPENVINO_INSTALL_DIR%" + xcopy /e /i /y /q "!OPENVINO_EXTRACTED!\*" "%OPENVINO_INSTALL_DIR%\" >nul + if errorlevel 1 ( + echo ERROR: Failed to copy OpenVINO from "!OPENVINO_EXTRACTED!" to "%OPENVINO_INSTALL_DIR%". + echo Re-run this script from an elevated Command Prompt ^(Run as administrator^) if access is denied. + exit /b 1 + ) + + rmdir /s /q "%OPENVINO_EXTRACT_TMP%" + del "%OPENVINO_ZIP%" +) + +REM Refresh junction: C:\Intel\openvino -> C:\Intel\openvino_. +REM `mklink /J` creates a directory junction (no admin / Developer Mode required). +if exist "%OPENVINO_LINK_DIR%" rmdir "%OPENVINO_LINK_DIR%" +mklink /J "%OPENVINO_LINK_DIR%" "%OPENVINO_INSTALL_DIR%" >nul +if errorlevel 1 ( + echo ERROR: Failed to create junction "%OPENVINO_LINK_DIR%" -^> "%OPENVINO_INSTALL_DIR%". + echo If "%OPENVINO_LINK_DIR%" already exists as a regular non-empty folder, remove it manually and re-run. + exit /b 1 +) + +set "OPENVINO_ROOT=%OPENVINO_LINK_DIR%" +echo OpenVINO Ready: %OPENVINO_ROOT% -^> %OPENVINO_INSTALL_DIR% + + +echo ============================================ +echo Setting up compiler environment... +echo ============================================ +REM Locate Visual Studio Build Tools vcvars64.bat +set "VSWHERE=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" +if exist "%VSWHERE%" ( + for /f "usebackq tokens=*" %%i in (`"%VSWHERE%" -latest -products Microsoft.VisualStudio.Product.BuildTools -property installationPath`) do ( + set "VS_PATH=%%i" + ) +) +if defined VS_PATH ( + call "%VS_PATH%\VC\Auxiliary\Build\vcvars64.bat" >nul +) else ( + echo WARNING: Visual Studio Build Tools not found. Compiler may be missing. +) + +REM ============================================ +REM Clean old build cache +REM ============================================ +if exist "build\ReleaseOV" ( + echo Removing old build directory ... + rmdir /s /q "build\ReleaseOV" +) + +echo ============================================ +echo Configuring with CMake... +echo ============================================ +call "%OPENVINO_ROOT%\setupvars.bat" >nul 2>nul + +cmake -B build\ReleaseOV -G Ninja ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DGGML_OPENVINO=ON ^ + -DCMAKE_TOOLCHAIN_FILE="%VCPKG_DIR%\scripts\buildsystems\vcpkg.cmake" + +if errorlevel 1 ( + echo If you continue to face CMAKE errors, make sure to install: + echo winget install Microsoft.VisualStudio.2022.BuildTools + echo Then run the "Developer Command Prompt for VS 2022" and launch this script from there. + exit /b 1 +) + +cmake --build build\ReleaseOV --config Release +if errorlevel 1 exit /b 1 + +echo ============================================ +echo Build completed successfully! +echo ============================================ +echo Binaries: %CD%\build\ReleaseOV\bin +echo. +echo NOTE: To run, source setupvars.bat and pick a device: +echo call "C:\Intel\openvino\setupvars.bat" +echo set GGML_OPENVINO_DEVICE=CPU ^&^& REM or GPU / NPU +echo build\ReleaseOV\bin\llama-cli.exe -m model.gguf +echo. + +endlocal +``` + +> [!NOTE] +> The script pins OpenVINO `2026.2` via the `OPENVINO_VERSION_MAJOR` / `OPENVINO_VERSION_FULL` variables at the top — edit them to track a different release. From any new shell, source the matching `setupvars` script via the junction — `call "C:\Intel\openvino\setupvars.bat"` from `cmd`, or `& "C:\Intel\openvino\setupvars.ps1"` from PowerShell. If `winget` cannot register Visual Studio Build Tools on first run, install them once manually and re-run the script from an elevated **Developer Command Prompt for VS 2022**. + +
+ ### 3. Download Sample Model -Download models for testing: +Download sample model for testing. ```bash # Linux mkdir -p ~/models/ -wget https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf \ - -O ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf +wget https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf \ + -O ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf # Windows PowerShell mkdir C:\models -Invoke-WebRequest -Uri https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf +Invoke-WebRequest -Uri https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -OutFile C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf # Windows Command Line mkdir C:\models -curl -L https://huggingface.co/unsloth/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf +curl -L https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_K_M.gguf -o C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf ``` ### 4. Run Inference with OpenVINO Backend @@ -196,65 +584,45 @@ When using the OpenVINO backend, the first inference token may have slightly hig # Linux export GGML_OPENVINO_DEVICE=GPU -# Enable stateful execution with GPU device to avoid known stateless execution failures. +# Optional: enable stateful execution for improved GPU performance (recommended). export GGML_OPENVINO_STATEFUL_EXECUTION=1 # To run llama-simple: -./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -n 50 "The story of AI is " +./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is " # To run in chat mode: -./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -c 1024 +./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024 # To run llama-bench, -fa 1 is needed -GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -fa 1 +GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./build/ReleaseOV/bin/llama-bench -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -fa 1 # NPU: keep context small to avoid failures from very large model context windows. export GGML_OPENVINO_DEVICE=NPU -./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -c 512 +./build/ReleaseOV/bin/llama-cli -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 512 # Windows Command Line set GGML_OPENVINO_DEVICE=GPU -# Enable stateful execution with GPU device to avoid known stateless execution failures. +# Optional: enable stateful execution for improved GPU performance (recommended). set GGML_OPENVINO_STATEFUL_EXECUTION=1 # Windows PowerShell $env:GGML_OPENVINO_DEVICE = "GPU" $env:GGML_OPENVINO_STATEFUL_EXECUTION = "1" # To run llama-simple -build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -n 50 "The story of AI is " +build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is " # To run in chat mode: -build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -c 1024 +build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 1024 # To run llama-bench, -fa 1 is needed -build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -fa 1 +build\ReleaseOV\bin\llama-bench.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -fa 1 # NPU: keep context small to avoid failures from very large model context windows. # Windows Command Line set GGML_OPENVINO_DEVICE=NPU # Windows PowerShell $env:GGML_OPENVINO_DEVICE = "NPU" -build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -c 512 +build\ReleaseOV\bin\llama-cli.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -c 512 ``` > [!NOTE] > On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html) for more details. -### Known Issues and Current Workarounds - -- GPU stateless execution is currently affected by a known issue. - - Workaround: set `GGML_OPENVINO_STATEFUL_EXECUTION=1` when using GPU device. -- NPU failures can happen when context size is too large. Recent llama.cpp behavior may resolve context size to the model training context (for example, 131072 for Llama 3.2 1B), which is too large for current NPU usage and can also stress laptop CPU/GPU on larger models. To inspect the selected context size, run `llama-cli` or `llama-server` with `-lv 3`. - - Workaround: explicitly set context size, for ex. `-c 1024` for NPU runs. Performance will be better with lower context size. -- Additional NPU limitations: - - Model caching is not yet supported. - - `llama-server -np > 1` (multiple parallel sequences) is not supported. - - `llama-perplexity` is only supported with `-b 512` or smaller. -- `--context-shift` with `llama-cli` is currently not supported with OpenVINO backend across CPU, GPU, and NPU devices. -- Encoder models (embedding, reranking) are not supported with the current OpenVINO backend implementation. -- `-fa 1` is required when running llama-bench with the OpenVINO backend. - - `GGML_OPENVINO_STATEFUL_EXECUTION=1 GGML_OPENVINO_DEVICE=GPU ./llama-bench -fa 1` -- `llama-server` with OpenVINO backend supports only one chat session/thread, when `GGML_OPENVINO_STATEFUL_EXECUTION=1` is enabled. - -> [!NOTE] -> The OpenVINO backend is actively under development. Fixes are underway, and this document will continue to be updated as issues are resolved. - - -### Docker Build +### 5. Docker Build You can build and run llama.cpp with OpenVINO backend using Docker. @@ -272,7 +640,7 @@ docker build --target=light -t llama-openvino:light -f .devops/openvino.Dockerfi docker build --target=server -t llama-openvino:server -f .devops/openvino.Dockerfile . # If you are behind a proxy: -docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=light -t llama-openvino:light -f .devops/openvino.Dockerfile . +docker build --build-arg http_proxy=$http_proxy --build-arg https_proxy=$https_proxy --target=server -t llama-openvino:server -f .devops/openvino.Dockerfile . ``` Run llama.cpp with OpenVINO backend Docker container. @@ -281,19 +649,19 @@ Save sample models in `~/models` as [shown above](#3-download-sample-model). It ```bash # Run Docker container -docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf +docker run --rm -it -v ~/models:/models llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf # With Intel GPU access (iGPU or dGPU) docker run --rm -it -v ~/models:/models \ --device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ --env=GGML_OPENVINO_DEVICE=GPU --env=GGML_OPENVINO_STATEFUL_EXECUTION=1 \ -llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf +llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf # With Intel NPU access docker run --rm -it -v ~/models:/models \ --device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ --env=GGML_OPENVINO_DEVICE=NPU \ -llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf +llama-openvino:light --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf ``` Run Llama.cpp Server with OpenVINO Backend. @@ -301,17 +669,30 @@ Run Llama.cpp Server with OpenVINO Backend. > `llama-server` with OpenVINO backend supports only one chat session/thread, when `GGML_OPENVINO_STATEFUL_EXECUTION=1` is enabled. ```bash -# Run the Server Docker container -docker run --rm -it -p 8080:8080 -v ~/models:/models llama-openvino:server --no-warmup -m /models/Llama-3.2-1B-Instruct-Q4_0.gguf -c 1024 -# Or Using llama-server executable -./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf --port 8080 -c 1024 +# Run the llama-openvino:server Docker container (CPU) +docker run --rm -it -p 8080:8080 -v ~/models:/models llama-openvino:server --no-warmup -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -c 1024 --host 0.0.0.0 -# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost -export NO_PROXY=localhost,127.0.0.1 +# Run the llama-openvino:server Docker container with Intel GPU access (iGPU or dGPU) +docker run --rm -it -v ~/models:/models \ +--device=/dev/dri --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ +-p 8080:8080 --env=GGML_OPENVINO_DEVICE=GPU \ +llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0 + +# Run the llama-openvino:server Docker container with Intel NPU access +docker run --rm -it -v ~/models:/models \ +--device=/dev/accel --group-add=$(stat -c "%g" /dev/dri/render* | head -n 1) -u $(id -u):$(id -g) \ +-p 8080:8080 --env=GGML_OPENVINO_DEVICE=NPU \ +llama-openvino:server --no-warmup -c 1024 -m /models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --host 0.0.0.0 + +# Or Using llama-server executable +./build/ReleaseOV/bin/llama-server -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf --port 8080 -c 1024 # Option 1: Open your browser to http://localhost:8080 to access the web UI for the llama.cpp server. # Option 2: In a NEW terminal, test the server with curl +# If you are behind a proxy, make sure to set NO_PROXY to avoid proxy for localhost +export NO_PROXY=localhost,127.0.0.1 + # Test health endpoint curl -f http://localhost:8080/health @@ -320,24 +701,26 @@ curl -X POST "http://localhost:8080/v1/chat/completions" -H "Content-Type: appli -d '{"messages":[{"role":"user","content":"Write a poem about OpenVINO"}],"max_tokens":100}' | jq . ``` -## Runtime Configuration +## GGML OpenVINO Backend Runtime Configurations The OpenVINO backend can be configured using the following environment variables at runtime to control device selection, caching, debugging, and profiling behavior. +Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `1`) to enable; unset, empty, `0`, negative, or non-numeric values are treated as disabled. -### Configuration Options - -| Variable | Default | Description | -|-----------------------------------|------------|-------------------------------------------------------------------------------------------------------------| -| `GGML_OPENVINO_DEVICE` | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. | -| `GGML_OPENVINO_CACHE_DIR` | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** | -| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| `256` | Token chunk size for **NPU** prefill. | -| `GGML_OPENVINO_STATEFUL_EXECUTION`| `0` | Enable stateful KV cache on for better performance. Recommended on CPU, GPU. | -| `GGML_OPENVINO_PROFILING` | `0` | Enable execution-time profiling. | -| `GGML_OPENVINO_DUMP_CGRAPH` | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. | -| `GGML_OPENVINO_DUMP_IR` | `0` | Serialize OpenVINO IR files with timestamps. | -| `GGML_OPENVINO_DEBUG_INPUT` | `0` | Enable input debugging and print input tensor info. | -| `GGML_OPENVINO_DEBUG_OUTPUT` | `0` | Enable output debugging and print output tensor info. | -| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | `0` | Print tensor address map once. | +| Variable | Type | Default | Description | +|-----------------------------------|-----------|------------|-------------------------------------------------------------------------------------------------------------| +| `GGML_OPENVINO_DEVICE` | String | `CPU` | Specify the target device (CPU, GPU, NPU). On systems with multiple GPUs, use `GPU.0` or `GPU.1` to explicitly target specific GPU. See [OpenVINO GPU Device](https://docs.openvino.ai/2026/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html). When set to **NPU**, static compilation mode is enabled for optimal performance. | +| `GGML_OPENVINO_CACHE_DIR` | String | `not set` | Directory for OpenVINO model caching (recommended: `/tmp/ov_cache`). Enables model caching when set. **Not supported on NPU devices.** | +| `GGML_OPENVINO_PREFILL_CHUNK_SIZE`| Integer | `256` | Token chunk size for **NPU** prefill (NPU-only; ignored on CPU/GPU). Must be a positive integer; otherwise the default is used. | +| `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Enable stateful KV cache for better performance. Recommended on CPU, GPU. | +| `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | +| `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | +| `GGML_OPENVINO_PROFILING` | Boolean | `0` | Enable execution-time profiling. | +| `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. | +| `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. | +| `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. | +| `GGML_OPENVINO_DEBUG_OUTPUT` | Boolean | `0` | Enable output debugging and print output tensor info. | +| `GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS` | Boolean | `0` | Print tensor address map once. | > [!NOTE] >`GGML_OPENVINO_STATEFUL_EXECUTION` is an **Experimental** feature to allow stateful execution for managing the KV cache internally inside the OpenVINO model, improving performance on CPUs and GPUs. Stateful execution is not effective on NPUs, and not all models currently support this feature. This feature is experimental and has been validated only with the llama-simple, llama-cli, llama-bench, and llama-run applications and is recommended to enable for the best performance. Other applications, such as llama-server and llama-perplexity, are not yet supported. @@ -355,7 +738,7 @@ export GGML_OPENVINO_PROFILING=1 export GGML_OPENVINO_DEVICE=GPU export GGML_OPENVINO_STATEFUL_EXECUTION=1 -./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_0.gguf -n 50 "The story of AI is " +./build/ReleaseOV/bin/llama-simple -m ~/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf -n 50 "The story of AI is " # Windows Command Line set GGML_OPENVINO_CACHE_DIR=C:\tmp\ov_cache @@ -369,19 +752,39 @@ $env:GGML_OPENVINO_PROFILING = "1" $env:GGML_OPENVINO_DEVICE = "GPU" $env:GGML_OPENVINO_STATEFUL_EXECUTION = "1" -build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_0.gguf" -n 50 "The story of AI is " +build\ReleaseOV\bin\llama-simple.exe -m "C:\models\Llama-3.2-1B-Instruct-Q4_K_M.gguf" -n 50 "The story of AI is " ``` -## Llama.cpp Tools +## Known Limitations -The following tools work with the OpenVINO backend on CPU, GPU, NPU: -- llama-bench -- llama-cli -- llama-completion -- llama-perplexity -- llama-server -- llama-simple +**General (all devices)** + +- Llama.cpp OpenVINO backend currently supports a subset of GGML ops and text-only models. Unsupported ops or unsupported op shapes/cases fail during OpenVINO translation. +- Multimodal features (audio/image/video) are a work in progress. +- Limited Embedding and Reranking model support. +- Llama.cpp tool coverage across CPU/GPU/NPU is not uniform. + +**Tool-specific** + +- `llama-bench`: requires `-fa 1` (flash-attention). +- `llama-cli --context-shift`: stateless only (`GGML_OPENVINO_STATEFUL_EXECUTION=0`). In stateful mode the KV cache is owned by the OpenVINO model and cannot be shifted externally. +- `llama-server`: only one chat session/thread when `GGML_OPENVINO_STATEFUL_EXECUTION=1`. + +**GPU-specific** + +- `llama-server -np > 1`: concurrent requests are batched together, which may slightly reduce per-request throughput. + +**NPU-specific** + +- Default context resolves to the model's training context (e.g. 131072 for Llama 3.2 1B), which can OOM or fail or degrade performance on NPU. Inspect the resolved value with `-lv 3`. + - **Workaround:** Pass an explicit `-c `, e.g. `-c 1024`. +- NPU device uses a static graph with a fixed prefill chunk size (defaults to 256), configurable with `GGML_OPENVINO_PREFILL_CHUNK_SIZE`. Large prefill/batch settings may need tuning. +- `llama-server -np > 1` (multiple parallel sequences) is not supported. +- `llama-perplexity`: requires `-b 512` or smaller. + +> [!NOTE] +> The OpenVINO backend is actively under development. Fixes and improvements are underway, and this document will continue to be updated. ## Work in Progress diff --git a/ggml/src/ggml-openvino/.clang-format b/ggml/src/ggml-openvino/.clang-format index a2a24d7d3..4a5c7c208 100644 --- a/ggml/src/ggml-openvino/.clang-format +++ b/ggml/src/ggml-openvino/.clang-format @@ -2,12 +2,7 @@ # Override root .clang-format AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false -Cpp11BracedListStyle: true -SpacesInContainerLiterals: false -BreakBeforeBraces: Attach AccessModifierOffset: -4 -IndentCaseBlocks: false -IndentCaseLabels: false Language: Cpp AlignAfterOpenBracket: Align diff --git a/ggml/src/ggml-openvino/CMakeLists.txt b/ggml/src/ggml-openvino/CMakeLists.txt index 175b58566..cc089b721 100644 --- a/ggml/src/ggml-openvino/CMakeLists.txt +++ b/ggml/src/ggml-openvino/CMakeLists.txt @@ -1,8 +1,6 @@ -find_package(OpenVINO REQUIRED) +find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading) find_package(OpenCL REQUIRED) -include("${OpenVINO_DIR}/../3rdparty/tbb/lib/cmake/TBB/TBBConfig.cmake") - file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp") file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp") @@ -11,7 +9,7 @@ ggml_add_backend_library(ggml-openvino ${GGML_HEADERS_OPENVINO} ) -target_link_libraries(ggml-openvino PRIVATE openvino::runtime TBB::tbb OpenCL::OpenCL) +target_link_libraries(ggml-openvino PRIVATE openvino::runtime openvino::threading OpenCL::OpenCL) if (GGML_OPENVINO) if (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 5095e7998..b6df4f0fb 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1,20 +1,17 @@ #include "ggml-decoder.h" -#include "ggml-backend-impl.h" -#include "ggml-backend.h" +#include "ggml-impl.h" #include "ggml-openvino-extra.h" #include "ggml-openvino.h" #include "ggml-quants.h" - -#include -#include +#include "ggml.h" +#include "utils.h" #include #include #include #include #include -#include #include #include #include @@ -30,12 +27,10 @@ #include #include #include -#include #include #include #include #include -#include #include GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, @@ -44,6 +39,7 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map> & model_weights, bool is_static, bool is_stateful, + bool model_is_splitted, bool is_prefill, int prefill_chunk_size) : m_is_static(is_static), @@ -51,22 +47,23 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, m_is_prefill(is_prefill), m_naive(false), m_prefill_chunk_size(prefill_chunk_size), + m_model_is_splitted(model_is_splitted), m_cgraph(cgraph), m_model_weights(model_weights), m_model_params(model_params), m_compute_params(compute_params) { - if (auto * env = getenv("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS"); env && std::string(env) != "0") { -#ifdef _WIN32 - _putenv_s("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS", ""); -#else - unsetenv("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS"); -#endif - print_tensor_address_map(cgraph); + static bool printed_address_map = false; + if (!printed_address_map) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS")) { + printed_address_map = true; + print_tensor_address_map(cgraph); + } } validate_cgraph(); set_input_output(); + compute_node_dynamic_dims(); compute_model_inputs(); compute_model_outputs(); @@ -136,6 +133,29 @@ void GgmlOvDecoder::set_input_output() { } current_node_info.node_inputs[src_name] = src; current_node_info.node_inputs_names.push_back(src_name); + + if (src->op == GGML_OP_VIEW) { + // Traverse upward through nested VIEW operations + std::remove_reference_t view_chain; + auto current = src; + + while (current != nullptr) { + auto current_name = std::string(current->name); + if (current->flags & GGML_TENSOR_FLAG_INPUT) { + current_name = get_graph_input_ov_name(current, node); + } + view_chain.emplace_back(current_name, current); + // If current src is also a VIEW, continue traversing + if (current->src[0] != nullptr && current->src[0]->op == GGML_OP_VIEW) { + current = current->src[0]; + } else { + break; + } + } + + // Assign all collected view inputs to node_inputs_views + current_node_info.node_inputs_views[src_name] = view_chain; + } } m_node_info_list.push_back(current_node_info); @@ -156,20 +176,13 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (src->ne[2] * src->ne[3] == node->ne[1]) { op_case = 5; } - } else if (src->ne[0] * src->ne[1] == node->ne[1]) { + } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 3; } else if (src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 6; } - break; - } - case GGML_OP_CONT: { - if (node->src[0]->op == GGML_OP_PERMUTE) { - op_case = 1; - } else if (node->src[0]->op == GGML_OP_TRANSPOSE) { - op_case = 2; - } else if (node->src[0]->op == GGML_OP_VIEW) { - op_case = 3; + if (op_case == 0 && ggml_nelements(node) == ggml_nelements(src)) { + op_case = 6; } break; } @@ -179,23 +192,41 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { } else if (node->src[0]->src[0]->op == GGML_OP_NONE) { // kv cache tensor std::string src_name(node->view_src->name); - int layer = extract_layer_from_name(src_name); - if (!is_swa_layer(layer)) { - op_case = 2; + int layer = extract_layer_from_name(src_name).value(); + if (ggml_is_contiguous(node->src[0])) { + // - 19: [ 64, 8, 256, 1] VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] + // [ 512, 1024, 1, 1] 0: NONE cache_k_l0 [ 2, 1024, 1048576, 1048576] + // - 20: [ 64, 256, 8, 1] PERMUTE cache_k_l0 (view) (permuted) [ 2, 1024, 128, 1048576] + // [ 64, 8, 256, 1] 0: VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] + if (!is_swa_layer(layer)) { + op_case = 3; + } else { + op_case = 4; + } } else { - op_case = 3; + // special case of cache v when `-fa off` + // - 17: [ 256, 8, 64, 1] VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] + // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 [ 2, 1024, 1048576, 1048576] + // - 18: [ 256, 64, 8, 1] PERMUTE cache_v_l0 (view) (permuted) [ 2, 2048, 131072, 1048576] + // [ 256, 8, 64, 1] 0: VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] + if (!is_swa_layer(layer)) { + op_case = 5; + } else { + op_case = 6; + } } } else { // rope'ed query tensor - op_case = 4; + op_case = 2; } break; } case GGML_OP_MUL_MAT: { - if (node->src[0]->op == GGML_OP_CONT && node->src[0]->src[0]->op == GGML_OP_TRANSPOSE) { - op_case = 2; - } else if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) { + if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) { op_case = 3; + } else if (node->src[1]->op == GGML_OP_SOFT_MAX) { + // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v + op_case = 2; } break; } @@ -208,43 +239,57 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { case GGML_OP_ROPE: { const int mode = node->op_params[2]; switch (mode) { - case GGML_ROPE_TYPE_NEOX: { - op_case = 0x00010000; + case GGML_ROPE_TYPE_NEOX: { + op_case = 1; break; } - case GGML_ROPE_TYPE_IMROPE: { - op_case = 0x00020000; + case GGML_ROPE_TYPE_IMROPE: { + op_case = 2; break; } default: - op_case = 0x00000000; + op_case = 0; break; } - if (node->src[0]->op == GGML_OP_VIEW) { - op_case = (op_case | 0x00000002); - } break; } case GGML_OP_VIEW: { if (node->src[0]->op == GGML_OP_VIEW) { auto * src = node->src[0]; if (ggml_nelements(node) != ggml_nelements(src)) { - throw std::runtime_error("Unsupported VIEW case"); + // throw std::runtime_error("Unsupported VIEW case"); + } + op_case = 0; + if (m_model_is_splitted && m_model_inputs.find(std::string(src->name)) != m_model_inputs.end()) { + op_case = 0; } - op_case = 2; } { auto * src = node->src[0]; - if ((ggml_nelements(node) != ggml_nelements(src)) && m_naive) { - // Compare each dimension of node and src, if only one dimension differs then op_case=3 + if (ggml_nelements(node) != ggml_nelements(src)) { + // Case 4: select one slice on src dim1 (via view offset), keep src dim2 as output dim1. + // Typical pattern: + // src: ne=[N, M, K, 1], nb=[b0, b1, b2, b3] + // dst: ne=[N, K, 1, 1], nb=[b0, b2, b3, b3] + if (node->ne[0] == src->ne[0] && node->ne[1] == src->ne[2] && node->ne[2] == 1 && + node->nb[0] == src->nb[0] && node->nb[1] == src->nb[2] && src->ne[1] > 1) { + op_case = 0; + break; + } + + // General case 3: shape differs from source (one or more dims) and is handled as VIEW slicing. int diff_count = 0; for (int i = 0; i < GGML_MAX_DIMS; i++) { if (node->ne[i] != src->ne[i]) { diff_count++; } + // if node ne[i] > src ne[i], case = 0 + if (node->ne[i] > src->ne[i]) { + return 0; + } } - if (diff_count == 1) { - op_case = 3; + if (diff_count >= 1) { + op_case = 0; } } } @@ -256,9 +301,11 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { return op_case; } -int extract_layer_from_name(const std::string & name) { +std::optional extract_layer_from_name(const std::string & name) { size_t pos1 = name.find("_l"); - assert(pos1 != std::string::npos); + if (pos1 == std::string::npos) { + return std::nullopt; + } pos1 += 2; size_t pos2 = name.find(' ', pos1); if (pos2 == std::string::npos) { @@ -272,26 +319,101 @@ int extract_layer_from_name(const std::string & name) { std::pair GgmlOvDecoder::compute_llm_params(ggml_cgraph * cgraph, bool is_static) { ModelParams model_params; ComputeParams compute_params; + auto get_attention_pattern_case = [](const ggml_tensor * node) -> int { + if (node == nullptr) { + return -1; + } + + switch (node->op) { + case GGML_OP_FLASH_ATTN_EXT: + if (node->src[0] == nullptr || node->src[1] == nullptr || node->src[3] == nullptr) { + return -1; + } + switch (node->src[1]->op) { + case GGML_OP_PERMUTE: + // case 0: node op is FLASH_ATTN_EXT, src 1 not null & op is PERMUTE & the permuted tensor src is the view of cache k + if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_VIEW) { + return 0; + } + break; + case GGML_OP_CPY: + // case 1: node op is FLASH_ATTN_EXT, src 1 not null & op is CPY & the copied tensor src is PERMUTE & the permuted tensor src is the view of cache k + if (node->src[1]->src[0] != nullptr && node->src[1]->src[0]->op == GGML_OP_PERMUTE && + node->src[1]->src[0]->src[0] != nullptr && node->src[1]->src[0]->src[0]->op == GGML_OP_VIEW) { + return 1; + } + break; + default: + break; + } + break; + case GGML_OP_SOFT_MAX: + // case 2: node op is SOFT_MAX, src 0 not null & op is MUL_MAT & the src 0 of MUL_MAT is PERMUTE & the permuted tensor src is the view of cache k + if (node->src[0] != nullptr && node->src[1] != nullptr && node->src[0]->op == GGML_OP_MUL_MAT && + node->src[0]->src[0] != nullptr && node->src[0]->src[1] != nullptr && + node->src[0]->src[0]->op == GGML_OP_PERMUTE && node->src[0]->src[0]->src[0] != nullptr && + node->src[0]->src[0]->src[0]->op == GGML_OP_VIEW) { + return 2; + } + // case 3: node op is SOFT_MAX, src 0 not null & op is ADD & the src 0 of ADD is MUL_MAT & the src 0 of MUL_MAT is PERMUTE + if (node->src[0]->op == GGML_OP_ADD && node->src[0]->src[0] != nullptr && + node->src[0]->src[0]->op == GGML_OP_MUL_MAT && node->src[0]->src[0]->src[0] != nullptr && + node->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) { + return 3; + } + break; + default: + break; + } + + return -1; + }; + + bool rope_seen = false; for (int i = 0; i < cgraph->n_nodes; i++) { auto * node = cgraph->nodes[i]; std::string name = std::string(node->name); - if (node->op == GGML_OP_FLASH_ATTN_EXT) { - model_params.n_heads = node->src[0]->ne[2]; - model_params.n_heads_kv = node->src[1]->ne[2]; - model_params.head_size = node->src[0]->ne[0]; - compute_params.input_len = node->src[0]->ne[1]; + const int attention_pattern_case = get_attention_pattern_case(node); + if (attention_pattern_case != -1) { + ggml_tensor * cache_k_permute = nullptr; + ggml_tensor * mask = nullptr; - auto * cache_k_perm = node->src[1]; - if (cache_k_perm->op == GGML_OP_CPY) { - cache_k_perm = cache_k_perm->src[0]; + switch (attention_pattern_case) { + case 0: + cache_k_permute = node->src[1]; + mask = node->src[3]; + break; + case 1: + cache_k_permute = node->src[1]->src[0]; + mask = node->src[3]; + break; + case 2: + cache_k_permute = node->src[0]->src[0]; + mask = node->src[1]; + break; + case 3: + cache_k_permute = node->src[0]->src[0]->src[0]; + mask = node->src[1]; + break; + default: + break; } - assert(cache_k_perm->op == GGML_OP_PERMUTE); - auto * cache_k_view = cache_k_perm->src[0]; - assert(cache_k_view->op == GGML_OP_VIEW); - auto * cache_k = cache_k_view->src[0]; - int layer = extract_layer_from_name(cache_k->name); - auto * mask = node->src[3]; + assert(cache_k_permute != nullptr); + + model_params.head_size = cache_k_permute->ne[0]; + model_params.n_heads_kv = cache_k_permute->ne[2]; + compute_params.input_len = node->src[0]->ne[1]; + compute_params.token_len_per_seq = node->src[0]->ne[1]; + + auto * cache_k_view = cache_k_permute->src[0]; + if (cache_k_view->op != GGML_OP_VIEW || mask == nullptr) { + continue; + } + + ggml_tensor * cache_k = cache_k_view->src[0]; + int layer = extract_layer_from_name(cache_k->name).value(); + std::string mask_name(mask->name); model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer); @@ -308,7 +430,6 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr size_t offset; memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; - compute_params.token_len_per_seq = node->ne[2]; if (mask_name.find("swa") != std::string::npos) { compute_params.attention_size_swa = mask->ne[0]; @@ -320,10 +441,40 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr compute_params.attention_size_swa = model_params.ctx_per_seq_swa; compute_params.token_len_per_seq = 1; } - break; + } + + if (node->op == GGML_OP_MUL_MAT && node->src[0]->op == GGML_OP_PERMUTE && + node->src[0]->src[0]->op == GGML_OP_VIEW && is_kvcache(node->src[0]->view_src, node->view_src)) { + if (node->src[1]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->op == GGML_OP_VIEW && + node->src[1]->src[0]->src[0]->op == GGML_OP_ROPE) { + compute_params.attention_size = node->ne[0]; + } + } + + // if the node op is TRANSPOSE and its input is PERMUTE and the source of the PERMUTE is VIEW, then get the attention size with the TRANSPOSE node ne[0] (in case no GGML_OP_FLASH_ATTN_EXT) + if (node->op == GGML_OP_TRANSPOSE && node->src[0]->op == GGML_OP_PERMUTE && + node->src[0]->src[0]->op == GGML_OP_VIEW) { + compute_params.attention_size = node->ne[0]; + if (is_static) { + compute_params.attention_size = model_params.ctx_per_seq; + } } if (node->op == GGML_OP_ROPE) { - memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15); + if (compute_params.token_len_per_seq == -1 && node->src[1] != nullptr) { + compute_params.token_len_per_seq = ggml_nelements(node->src[1]); + } + + // When multiple ROPE ops in the graph disagree on op_params (e.g. gemma4's + // mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot + // share a single precomputed rope_sin/rope_cos. Track divergence so the + // translator falls back to per-op make_sin_cos in that case. + static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 15, "rope_params size"); + if (!rope_seen) { + memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15); + rope_seen = true; + } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 15) != 0) { + model_params.mixed_rope_params = true; + } } } auto * output_tensor = cgraph->nodes[cgraph->n_nodes - 1]; @@ -333,7 +484,6 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr compute_params.output_len = 1; } model_params.ctx = model_params.ctx_per_seq * model_params.n_seq; - model_params.ctx_swa = model_params.ctx_per_seq_swa * model_params.n_seq; return {model_params, compute_params}; } @@ -343,9 +493,11 @@ void GgmlOvDecoder::validate_cgraph() const { } } -ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const { +ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, + const ggml_tensor * input, + int dynamic_dim_index) const { if (m_naive) { - return input!= nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)}; + return input != nullptr ? ov::PartialShape{get_shape(input)} : ov::PartialShape{get_shape(op)}; } auto name = std::string(input->name); ov::PartialShape input_shape; @@ -394,6 +546,15 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, co } else { input_shape = ov::PartialShape{get_shape(input)}; } + if (dynamic_dim_index != -1 && m_model_is_splitted) { + input_shape[3 - dynamic_dim_index] = -1; + } + if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && + op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) { + // for softmax input mask, the shape is [1, 1, seq_active, seq_active], where seq_active is determined by the input active sequence length instead of the kv cache sequence length + input_shape[2] = -1; + input_shape[3] = -1; + } return input_shape; } @@ -421,15 +582,19 @@ void GgmlOvDecoder::add_extra_inputs() { } }; - create_1d_input("attention_size", m_compute_params.attention_size); + if (m_compute_params.attention_size != -1) { + create_1d_input("attention_size", m_compute_params.attention_size); + } if (m_compute_params.attention_size_swa != -1) { create_1d_input("attention_size_swa", m_compute_params.attention_size_swa); } create_1d_input("n_seq_active", m_compute_params.n_seq_active); create_1d_input("seq_active_start", m_compute_params.seq_active_start); create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active); - create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq); - // create_1d_input("token_len", m_token_len_per_seq * m_n_seq_active); + if (m_compute_params.token_len_per_seq != -1) { + create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq); + } + // create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active); } bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) { @@ -455,8 +620,8 @@ void GgmlOvDecoder::compute_model_inputs() { std::string node_name(node->name); if (m_model_weights.find(node_name) == m_model_weights.end()) { m_inputs[node_name] = node; - auto param_node = - std::make_shared(get_ov_type(node), get_graph_input_shape(node, nullptr)); + auto param_node = std::make_shared( + get_ov_type(node), get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])); param_node->set_friendly_name(node_name); param_node->output(0).get_tensor().set_names({node_name}); m_model_inputs[node_name] = param_node; @@ -500,7 +665,13 @@ void GgmlOvDecoder::compute_model_inputs() { m_model_params.kv_names.push_back(src_name); } } - ov::PartialShape param_shape = get_graph_input_shape(node, src); + // Resolve nested VIEW nodes by following src[0] until the first non-VIEW tensor. + while (src->op == GGML_OP_VIEW && src->src[0] != nullptr) { + src = src->src[0]; + src_name = std::string(src->name); + } + m_inputs[src_name] = src; + ov::PartialShape param_shape = get_graph_input_shape(node, src, m_node_dynamic_dims[src]); auto param_node = std::make_shared(get_ov_type(src), param_shape); param_node->set_friendly_name(src_name); param_node->output(0).get_tensor().set_names({src_name}); @@ -515,7 +686,7 @@ void GgmlOvDecoder::compute_model_outputs() { for (int node_n = 0; node_n < m_cgraph->n_nodes; node_n++) { auto * cur_node = m_cgraph->nodes[node_n]; // if the node op is NONE means this node is not used at all, we can skip it directly without adding to model outputs. - if (cur_node->op == GGML_OP_NONE) { + if (cur_node->op == GGML_OP_NONE || cur_node->op == GGML_OP_VIEW || cur_node->op == GGML_OP_RESHAPE) { continue; } auto cur_node_use_count = m_cgraph->use_counts[ggml_hash_find(&m_cgraph->visited_hash_set, cur_node)]; @@ -644,15 +815,26 @@ std::shared_ptr GgmlOvDecoder::create_weight_node(ggml_tensor * tensor } } + // MUL_MAT_ID expert weights are 3D GGML tensors [k, m, n_expert]. + // Keep the full reversed 4D shape when materializing non-quantized constants, + // otherwise the expert dimension is collapsed and later Gather/MatMul logic + // only sees a single expert slice. + if (!ggml_is_quantized(tensor->type) && (tensor->ne[2] > 1 || tensor->ne[3] > 1)) { + auto weight_tensor = ov::Tensor(get_ov_type(tensor), get_shape(tensor), tensor->data); + auto weight_node = std::make_shared(weight_tensor); + weight_node->set_friendly_name(tensor->name); + return weight_node; + } + // There are three cases where we need to create a new weight node: // 1. weights are in openvino_host_buffer. Weight loading to host buffer will not trigger backend_buffer_set_tensor // 2. weights are in cpu/cpu_mapped buffer. On token_embd.weight goes to case 1 or 2, depending on whether mmap or direct_io is used // 3. test-backend-ops. buffers in test-backend-ops does not set USAGE_WEIGHT so backend_buffer_set_tensor will not create weight node // GGML_LOG_DEBUG("%s: creating new weight node for %s\n", __func__, tensor->name); - static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, - GGML_TYPE_Q8_0, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, - GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}; + static const std::set weight_types = {GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_Q8_0, + GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1, GGML_TYPE_Q4_K, + GGML_TYPE_Q5_K, GGML_TYPE_Q6_K}; if (weight_types.find(tensor->type) == weight_types.end()) { throw std::runtime_error("Unexpected weight tensor type: " + std::string(tensor->name) + " with type " + ggml_type_name(tensor->type)); @@ -860,6 +1042,161 @@ std::vector GgmlOvDecoder::get_input_stride(int node_idx, const std::str return get_stride(m_node_info_list[node_idx].node_inputs.at(name)); } +size_t GgmlOvDecoder::get_view_input_size(int node_idx, const std::string & name) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + return it->second.size(); + } + return 0; +} + +size_t GgmlOvDecoder::get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return it->second[view_index].second->view_offs; + } + } + return 0; +} + +size_t GgmlOvDecoder::get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return view_tensor->src[0]->view_offs; + } + } + } + return 0; +} + +std::vector GgmlOvDecoder::get_view_input_stride(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return get_stride(it->second[view_index].second); + } + } + return {}; +} + +std::vector GgmlOvDecoder::get_view_input_src_stride(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return get_stride(view_tensor->src[0]); + } + } + } + return {}; +} + +ov::Shape GgmlOvDecoder::get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return get_shape(it->second[view_index].second); + } + } + return {}; +} + +ov::Shape GgmlOvDecoder::get_view_input_src_ggml_shape(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return get_shape(view_tensor->src[0]); + } + } + } + return {}; +} + +ov::PartialShape GgmlOvDecoder::get_view_input_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * tensor = it->second[view_index].second; + ov::PartialShape shape = ov::PartialShape{get_shape(tensor)}; + + // Check if this tensor has a dynamic dimension + auto dynamic_it = m_node_dynamic_dims.find(tensor); + if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) { + int dynamic_dim_index = dynamic_it->second; + // GGML uses reverse indexing, so convert to OpenVINO indexing + shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1; + } + + return shape; + } + } + return {}; +} + +ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + auto * src_tensor = view_tensor->src[0]; + ov::PartialShape shape = ov::PartialShape{get_shape(src_tensor)}; + + // Check if this tensor has a dynamic dimension + auto dynamic_it = m_node_dynamic_dims.find(src_tensor); + if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) { + int dynamic_dim_index = dynamic_it->second; + // GGML uses reverse indexing, so convert to OpenVINO indexing + shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1; + } + + return shape; + } + } + } + return {}; +} + +std::string GgmlOvDecoder::get_view_input_name(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + return it->second[view_index].second->name; + } + } + return ""; +} + +std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const { + auto it = m_node_info_list[node_idx].node_inputs_views.find(name); + if (it != m_node_info_list[node_idx].node_inputs_views.end()) { + if (view_index < it->second.size()) { + auto * view_tensor = it->second[view_index].second; + if (view_tensor && view_tensor->src[0]) { + return view_tensor->src[0]->name; + } + } + } + return ""; +} + ov::element::Type GgmlOvDecoder::get_input_type(int node_idx, const std::string & name) const { return get_ov_type(m_node_info_list[node_idx].node_inputs.at(name)); } @@ -885,6 +1222,11 @@ ov::element::Type GgmlOvDecoder::get_output_type(const int node_idx) const { return get_ov_type(m_node_info_list[node_idx].node); } +std::vector GgmlOvDecoder::get_output_stride(int node_idx) const { + auto * ggml_tensor = m_node_info_list[node_idx].node; + return get_stride(ggml_tensor); +} + std::vector GgmlOvDecoder::get_output_names(int node_idx) const { return {m_node_info_list[node_idx].node_output_name}; } @@ -894,6 +1236,14 @@ const std::string & GgmlOvDecoder::get_op_name() const { return unknown_name; } +int32_t GgmlOvDecoder::get_op_dynamic_dim(int node_idx) const { + auto it = m_node_dynamic_dims.find(m_node_info_list[node_idx].node); + if (it == m_node_dynamic_dims.end()) { + return -1; + } + return it->second; +} + const std::string & GgmlOvDecoder::get_op_name(int node_idx) const { return m_node_info_list[node_idx].node_name; } @@ -906,6 +1256,10 @@ int32_t * GgmlOvDecoder::get_output_op_params(int node_idx) const { return m_node_info_list[node_idx].node->op_params; } +size_t GgmlOvDecoder::get_output_op_offset(int node_idx) const { + return m_node_info_list[node_idx].node->view_offs; +} + void GgmlOvDecoder::visit_subgraph(std::function, int node_idx)> node_visitor) const { for (int node_idx = 0; node_idx < m_cgraph->n_nodes; node_idx++) { if (m_cgraph->nodes[node_idx]->op == GGML_OP_NONE) { @@ -917,28 +1271,41 @@ void GgmlOvDecoder::visit_subgraph(std::function ops = { - {GGML_OP_NONE, "GGML_OP_NONE" }, - {GGML_OP_ACC, "GGML_OP_ACC" }, - {GGML_OP_ADD, "GGML_OP_ADD" }, - {GGML_OP_ADD1, "GGML_OP_ADD1" }, - {GGML_OP_CONT, "GGML_OP_CONT" }, - {GGML_OP_DIV, "GGML_OP_DIV" }, - {GGML_OP_DUP, "GGML_OP_DUP" }, - {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS" }, - {GGML_OP_MUL, "GGML_OP_MUL" }, - {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT" }, - {GGML_OP_PERMUTE, "GGML_OP_PERMUTE" }, - {GGML_OP_RESHAPE, "GGML_OP_RESHAPE" }, - {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM" }, - {GGML_OP_ROPE, "GGML_OP_ROPE" }, - {GGML_OP_SCALE, "GGML_OP_SCALE" }, - {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX" }, - {GGML_OP_SUB, "GGML_OP_SUB" }, - {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE" }, - {GGML_OP_VIEW, "GGML_OP_VIEW" }, - {GGML_OP_SET_ROWS, "GGML_OP_SET_ROWS" }, - {GGML_OP_CPY, "GGML_OP_CPY" }, - {GGML_OP_FLASH_ATTN_EXT, "GGML_OP_FLASH_ATTN_EXT"}, + {GGML_OP_NONE, "GGML_OP_NONE" }, + {GGML_OP_ACC, "GGML_OP_ACC" }, + {GGML_OP_ADD, "GGML_OP_ADD" }, + {GGML_OP_ADD1, "GGML_OP_ADD1" }, + {GGML_OP_ADD_ID, "GGML_OP_ADD_ID" }, + {GGML_OP_CONCAT, "GGML_OP_CONCAT" }, + {GGML_OP_CONT, "GGML_OP_CONT" }, + {GGML_OP_DIV, "GGML_OP_DIV" }, + {GGML_OP_DUP, "GGML_OP_DUP" }, + {GGML_OP_GET_ROWS, "GGML_OP_GET_ROWS" }, + {GGML_OP_MUL, "GGML_OP_MUL" }, + {GGML_OP_MUL_MAT, "GGML_OP_MUL_MAT" }, + {GGML_OP_MUL_MAT_ID, "GGML_OP_MUL_MAT_ID" }, + {GGML_OP_PERMUTE, "GGML_OP_PERMUTE" }, + {GGML_OP_RESHAPE, "GGML_OP_RESHAPE" }, + {GGML_OP_RMS_NORM, "GGML_OP_RMS_NORM" }, + {GGML_OP_NORM, "GGML_OP_NORM" }, + {GGML_OP_ROPE, "GGML_OP_ROPE" }, + {GGML_OP_SCALE, "GGML_OP_SCALE" }, + {GGML_OP_SOFT_MAX, "GGML_OP_SOFT_MAX" }, + {GGML_OP_SUM_ROWS, "GGML_OP_SUM_ROWS" }, + {GGML_OP_SUB, "GGML_OP_SUB" }, + {GGML_OP_TRANSPOSE, "GGML_OP_TRANSPOSE" }, + {GGML_OP_VIEW, "GGML_OP_VIEW" }, + {GGML_OP_SET_ROWS, "GGML_OP_SET_ROWS" }, + {GGML_OP_CPY, "GGML_OP_CPY" }, + {GGML_OP_FLASH_ATTN_EXT, "GGML_OP_FLASH_ATTN_EXT" }, + {GGML_OP_L2_NORM, "GGML_OP_L2_NORM" }, + {GGML_OP_CLAMP, "GGML_OP_CLAMP" }, + {GGML_OP_PAD, "GGML_OP_PAD" }, + {GGML_OP_SSM_CONV, "GGML_OP_SSM_CONV" }, + {GGML_OP_GATED_DELTA_NET, "GGML_OP_GATED_DELTA_NET"}, + {GGML_OP_ARGSORT, "GGML_OP_ARGSORT" }, + {GGML_OP_REPEAT, "GGML_OP_REPEAT" }, + {GGML_OP_IM2COL, "GGML_OP_IM2COL" } }; static const std::map unary_ops = { {GGML_UNARY_OP_ABS, "GGML_UNARY_OP_ABS" }, @@ -952,6 +1319,7 @@ std::string GgmlOvDecoder::compute_op_type(const ggml_tensor * node) { {GGML_UNARY_OP_GELU, "GGML_UNARY_OP_GELU" }, {GGML_UNARY_OP_GELU_QUICK, "GGML_UNARY_OP_GELU_QUICK" }, {GGML_UNARY_OP_SILU, "GGML_UNARY_OP_SILU" }, + {GGML_UNARY_OP_SOFTPLUS, "GGML_UNARY_OP_SOFTPLUS" }, {GGML_UNARY_OP_HARDSWISH, "GGML_UNARY_OP_HARDSWISH" }, {GGML_UNARY_OP_HARDSIGMOID, "GGML_UNARY_OP_HARDSIGMOID"}, {GGML_UNARY_OP_EXP, "GGML_UNARY_OP_EXP" }, @@ -983,3 +1351,301 @@ const std::string & GgmlOvDecoder::get_op_type() const { static const std::string unknown_op = "UNKNOWN_GGML_OP"; return unknown_op; } + +void GgmlOvDecoder::compute_node_dynamic_dims() { + auto visit_node = [&](auto && self, ggml_tensor * node) -> void { + if (!node) { + return; + } + + if (node->op == GGML_OP_CPY) { + m_node_dynamic_dims[node] = -1; + } + + if (m_node_dynamic_dims.count(node)) { + return; + } + for (int i = 0; i < GGML_MAX_SRC; i++) { + ggml_tensor * src = node->src[i]; + if (src == nullptr) { + continue; + } + struct ggml_tensor * root_src = nullptr; + // if (src->org_src) { + // root_src = src->org_src; + // } + if (root_src) { + if (is_inp_tok(root_src, node) || is_inp_pos(root_src, node) || is_output_idx(root_src, node)) { + m_node_dynamic_dims[root_src] = 0; + m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; + continue; + } + self(self, root_src); + m_node_dynamic_dims[src] = m_node_dynamic_dims[root_src]; + } else { + if (is_inp_tok(src, node) || is_inp_pos(src, node) || is_output_idx(src, node)) { + m_node_dynamic_dims[src] = 0; + continue; + } + if (node->op == GGML_OP_VIEW && src->op == GGML_OP_NONE && !is_stateful() && !m_model_is_splitted) { + m_node_dynamic_dims[src] = 1; + continue; + } + self(self, src); + } + } + switch (node->op) { + case GGML_OP_NONE: + m_node_dynamic_dims[node] = -1; + break; + case GGML_OP_GET_ROWS: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[1]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[1]]; + if (dynamic_dim_idx == 0) { + m_node_dynamic_dims[node] = 1; + } else { + auto dynamic_dim_stride = node->src[1]->nb[dynamic_dim_idx] / ggml_type_size(node->src[1]->type) * + ggml_type_size(node->src[0]->type); + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (dynamic_dim_stride == node->src[0]->nb[i]) { + m_node_dynamic_dims[node] = i; + break; + } + } + } + // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]], + // "Dynamic dim value mismatch for node: " + std::string(node->name) + + // " and its src[1]: " + std::string(node->src[1]->name)); + } + break; + case GGML_OP_MUL: + case GGML_OP_MUL_MAT: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + } + if (m_node_dynamic_dims[node->src[1]] != -1) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; + } + break; + case GGML_OP_PERMUTE: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + // auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx]; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->op_params[i] == dynamic_dim_idx) { + m_node_dynamic_dims[node] = i; + break; + } + } + // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]], + // "Dynamic dim value mismatch for node: " + std::string(node->name) + + // " and its src[0]: " + std::string(node->src[0]->name)); + } + break; + case GGML_OP_VIEW: { + // Use stride-based matching: the stride of a VIEW dimension directly + // encodes which source dimension it indexes into, so it uniquely + // identifies the dynamic dim even when two dims share the same size. + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + if (node->src[0]->op == GGML_OP_NONE) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + } + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx]; + auto dynamic_dim_stride = + node->src[0]->nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) * ggml_type_size(node->type); + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] == dynamic_dim_stride) { + m_node_dynamic_dims[node] = i; + break; + } + } + if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]]) { + m_node_dynamic_dims[node] = -1; + // std::cout << "Warning: Dynamic dim value mismatch for node: " << node->name + // << " and its src[0]: " << node->src[0]->name << std::endl; + } + } + break; + } + case GGML_OP_TRANSPOSE: + case GGML_OP_RESHAPE: { + // RESHAPE requires src[0] to be contiguous, so both src and result + // have standard compact strides: nb[i] = type_size * prod(ne[0..i-1]). + // Match src->nb[dynamic_dim] against result->nb[i] to find the output + // dimension whose flat-memory boundary aligns with the source dynamic + // boundary. This is unambiguous (result strides are strictly monotone) + // and handles merged-lower-dim cases that ne-value matching misses. + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx]; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) { + m_node_dynamic_dims[node] = i; + break; + } + } + if (m_node_dynamic_dims[node] == -1) { + // std::cout << "Cannot determine dynamic dim for RESHAPE node: " << node->name << std::endl; + } + } + break; + } + case GGML_OP_FLASH_ATTN_EXT: { + // Output shape is hard-coded in ggml_flash_attn_ext as: + // ne = { v->ne[0], q->ne[2], q->ne[1], q->ne[3] } + // i.e. output dim 0 <- v dim 0 (head_size, static) + // output dim 1 <- q dim 2 (n_heads, static) + // output dim 2 <- q dim 1 (n_tokens, potentially dynamic) + // output dim 3 <- q dim 3 (batch, static) + // Using the fixed q-dim -> output-dim mapping table. + // q is src[0]; the mapping from q's dynamic dim to the output dim is: + // q dim 1 -> output dim 2 + // q dim 2 -> output dim 1 + // q dim 3 -> output dim 3 + // q dim 0 -> output dim 0 (head_size axis, unlikely to be dynamic) + constexpr int q_to_out[GGML_MAX_DIMS] = {0, 2, 1, 3}; + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto q_dynamic_dim = m_node_dynamic_dims[node->src[0]]; + m_node_dynamic_dims[node] = q_to_out[q_dynamic_dim]; + } + break; + } + case GGML_OP_CONT: + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[0]] != -1) { + auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + if (ggml_are_same_shape(node, node->src[0])) { + m_node_dynamic_dims[node] = dynamic_dim_idx; + } else { + size_t src_logical_nb[GGML_MAX_DIMS]; + src_logical_nb[0] = ggml_type_size(node->src[0]->type); + src_logical_nb[1] = src_logical_nb[0] * (node->src[0]->ne[0] / ggml_blck_size(node->src[0]->type)); + for (int i = 2; i < GGML_MAX_DIMS; i++) { + src_logical_nb[i] = src_logical_nb[i - 1] * node->src[0]->ne[i - 1]; + } + + auto dynamic_dim_stride = src_logical_nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) * + ggml_type_size(node->type); + int matched_dim_count = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] == dynamic_dim_stride && node->ne[i] == node->src[0]->ne[dynamic_dim_idx]) { + m_node_dynamic_dims[node] = i; + matched_dim_count++; + } + } + if (matched_dim_count != 1) { + m_node_dynamic_dims[node] = -1; + // std::cout << "Warning: Cannot determine dynamic dim for CONT node: " << node->name + // << " and its src[0]: " << node->src[0]->name << std::endl; + } + } + } + break; + case GGML_OP_RMS_NORM: + case GGML_OP_NORM: + case GGML_OP_ADD: + case GGML_OP_GLU: + case GGML_OP_ROPE: + case GGML_OP_SCALE: + case GGML_OP_SOFT_MAX: + case GGML_OP_ARGSORT: + case GGML_OP_ADD_ID: + case GGML_OP_UNARY: + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; + break; + case GGML_OP_MUL_MAT_ID: + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; + break; + case GGML_OP_CPY: + case GGML_OP_SET_ROWS: + m_node_dynamic_dims[node] = -1; + break; + case GGML_OP_IM2COL: { + m_node_dynamic_dims[node] = -1; + if (m_node_dynamic_dims[node->src[1]] != -1) { + const bool is_2D = node->op_params[6] == 1; + const int src_dyn = m_node_dynamic_dims[node->src[1]]; + if (is_2D) { + if (src_dyn == 0) { + m_node_dynamic_dims[node] = 1; // IW -> OW + } else if (src_dyn == 1) { + m_node_dynamic_dims[node] = 2; // IH -> OH + } else if (src_dyn == 3) { + m_node_dynamic_dims[node] = 3; // N -> N + } + } else { + if (src_dyn == 0) { + m_node_dynamic_dims[node] = 1; // IW -> OW + } else if (src_dyn == 2) { + m_node_dynamic_dims[node] = 2; // N -> N (1D: b->ne[2] is the batch/channel dim) + } + } + if (m_node_dynamic_dims[node] != -1) { + OPENVINO_ASSERT(node->src[1]->ne[src_dyn] == node->ne[m_node_dynamic_dims[node]], + "Dynamic dim value mismatch for IM2COL node: " + std::string(node->name) + + " and its src[1]: " + std::string(node->src[1]->name)); + } + } + break; + } + default: + // std::cout << "Doesn't handle node name: " << node->name << " op: " << ggml_op_name(node->op) << std::endl; + break; + } + }; + + for (int i = 0; i < m_cgraph->n_nodes; i++) { + ggml_tensor * node = m_cgraph->nodes[i]; + visit_node(visit_node, node); + } + + // print the nodes in m_cgraph name & shape with the dynamic dim (the dynamic dim is the dimension with -1 in m_node_dynamic_dims) for debugging + if (0) { + for (int i = 0; i < m_cgraph->n_nodes; i++) { + ggml_tensor * node = m_cgraph->nodes[i]; + int dynamic_dim = m_node_dynamic_dims[node]; + std::cout << "[" << i << "] " << "node_name: " << node->name << " op: " << ggml_op_name(node->op) + << " shape: ["; + for (int j = 0; j < 4; j++) { + if (j == dynamic_dim) { + std::cout << "*"; + } else { + std::cout << node->ne[j]; + } + if (j < 3) { + std::cout << ", "; + } + } + std::cout << "]" << std::endl; + // print the src name & shape with the dynamic dim for debugging + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + if (src == nullptr) { + continue; + } + int src_dynamic_dim = m_node_dynamic_dims[src]; + std::cout << " [" << j << "] src_name: " << src->name << " ["; + for (int k = 0; k < 4; k++) { + if (k == src_dynamic_dim) { + std::cout << "*"; + } else { + std::cout << src->ne[k]; + } + if (k < 3) { + std::cout << ", "; + } + } + std::cout << "]" << std::endl; + } + std::cout << std::endl; + } + } +} diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 3ae25ddda..ae545f47e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -1,6 +1,7 @@ #pragma once -#include "ggml-quants.h" +#include "ggml-backend-impl.h" +#include "ggml-backend.h" #include "ggml.h" #include "openvino/decoder.h" @@ -14,21 +15,21 @@ struct ModelParams { int ctx = -1; - int ctx_swa = -1; int ctx_per_seq = -1; int ctx_per_seq_swa = -1; int n_seq = 1; - int n_heads = -1; int n_heads_kv = -1; int head_size = -1; int32_t rope_params[15]; + bool mixed_rope_params = false; std::vector swa_layers; std::vector kv_names; size_t kv_buffer_ctx_id = 0; bool same_rope_params(const ModelParams & other) const { - return memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0; + return mixed_rope_params == other.mixed_rope_params && + memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0; } bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); } @@ -56,12 +57,14 @@ public: std::string node_name; std::string node_op_type; std::map node_inputs; + std::map>> node_inputs_views; std::vector node_inputs_names; ggml_tensor * node_output; std::string node_output_name; int node_op_case = 0; void * data_addr; }; + // Graph decoder GgmlOvDecoder(ggml_cgraph * cgraph, ModelParams & model_params, @@ -69,6 +72,7 @@ public: std::map> & model_weights, bool is_static, bool is_stateful = false, + bool model_is_splitted = false, bool is_prefill = false, int prefill_chunk_size = 256); @@ -84,6 +88,42 @@ public: virtual std::vector get_input_stride(int node_idx, const std::string & name) const override; + virtual size_t get_view_input_size(int node_idx, const std::string & name) const override; + + virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const override; + + virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const override; + + virtual std::vector get_view_input_stride(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual std::vector get_view_input_src_stride(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::Shape get_view_input_ggml_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::Shape get_view_input_src_ggml_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::PartialShape get_view_input_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual ov::PartialShape get_view_input_src_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const override; + + virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const override; + + virtual std::string get_view_input_src_name(int node_idx, + const std::string & name, + size_t view_index) const override; + virtual ov::element::Type get_input_type(int node_idx, const std::string & name) const override; virtual size_t get_input_size() const override; @@ -106,10 +146,14 @@ public: virtual ov::element::Type get_output_type(int node_idx) const override; + virtual std::vector get_output_stride(int node_idx) const override; + virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override; virtual int32_t * get_output_op_params(int node_idx) const override; + virtual size_t get_output_op_offset(int node_idx) const override; + virtual std::vector get_output_names(int node_idx) const override; virtual const std::string & get_op_type() const override; @@ -120,7 +164,10 @@ public: virtual const std::string & get_op_name(int node_idx) const override; - virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const override; + virtual int32_t get_op_dynamic_dim(int node_idx) const override; + + virtual void visit_subgraph( + std::function, int node_idx)> node_visitor) const override; ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); } @@ -142,16 +189,12 @@ public: return m_model_weights; } - virtual std::vector get_model_output_names() const override { - return m_model_output_names; - } + virtual std::vector get_model_output_names() const override { return m_model_output_names; } const std::map & get_model_outputs() const { return m_model_outputs; } virtual int get_ctx_size() const { return m_model_params.ctx; } - virtual int get_ctx_swa_size() const { return m_model_params.ctx_swa; } - virtual int get_ctx_per_seq() const { return m_model_params.ctx_per_seq; } virtual int get_ctx_per_seq_swa() const { return m_model_params.ctx_per_seq_swa; } @@ -169,13 +212,21 @@ public: virtual int32_t * get_rope_params() const override { return const_cast(m_model_params.rope_params); } + virtual bool has_mixed_rope_params() const override { return m_model_params.mixed_rope_params; } + virtual std::map get_kv_param_res_names() const override; virtual bool is_static() const override { return m_is_static; } virtual bool is_stateful() const override { return m_is_stateful; } - ov::PartialShape get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input) const; + int get_static_n_tokens() const { return m_is_prefill ? m_prefill_chunk_size : 1; } + + virtual bool is_splited_model() const override { return m_model_is_splitted; } + + ov::PartialShape get_graph_input_shape(const ggml_tensor * op, + const ggml_tensor * input, + int dynamic_dim_index = -1) const; static void dump_cgraph(const ggml_cgraph * cgraph, std::string & filename); @@ -205,6 +256,7 @@ public: bool m_is_prefill = false; bool m_naive = false; int m_prefill_chunk_size = 0; + bool m_model_is_splitted = false; // label the cgraph is splited or not static ov::Shape get_shape(const ggml_tensor * tensor); static std::vector get_stride(const ggml_tensor * tensor); @@ -227,7 +279,8 @@ public: } inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]); + return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || + (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); } inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { @@ -235,7 +288,8 @@ public: } inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_SET_ROWS && op->src[2] == tensor; + return tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY || + (op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor); } inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { @@ -243,23 +297,18 @@ public: } inline static bool is_output_idx(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE; + return op->op == GGML_OP_GET_ROWS && tensor == op->src[1] && op->src[0]->op != GGML_OP_NONE && + op->src[1]->op == GGML_OP_NONE; } - static std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { - if (is_inp_tok(tensor, op)) { - return "inp_tokens"; - } + std::string get_graph_input_ov_name(const ggml_tensor * tensor, const ggml_tensor * op) { if (is_inp_pos(tensor, op)) { return "inp_pos"; } if (is_inp_emb(tensor, op)) { return "embd"; } - if (is_output_idx(tensor, op)) { - return "inp_out_ids"; - } - if (is_inp_mask(tensor, op)) { + if (is_stateful() && is_inp_mask(tensor, op)) { return std::string(tensor->name).find("swa") == std::string::npos ? "self_kq_mask" : "self_kq_mask_swa"; } return tensor->name; @@ -272,6 +321,9 @@ private: void compute_model_inputs(); void compute_model_outputs(); + // Infer and propagate dynamic-dimension indices for all tensors in the GGML graph. + void compute_node_dynamic_dims(); + void validate_cgraph() const; ggml_cgraph * m_cgraph = nullptr; @@ -284,6 +336,7 @@ private: std::map m_model_outputs; std::vector m_model_output_names; std::vector m_node_info_list; + std::map m_node_dynamic_dims; ModelParams m_model_params; ComputeParams m_compute_params; @@ -291,4 +344,4 @@ private: void print_tensor_address_map(const ggml_cgraph * cgraph); -int extract_layer_from_name(const std::string & name); +std::optional extract_layer_from_name(const std::string & name); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 4140136ac..d9ad7be73 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -3,6 +3,7 @@ #include "ggml-impl.h" #include "ggml.h" +#include #include #include #include @@ -22,7 +23,38 @@ void ggml_openvino_device_config::init() { if (initialized) { return; } - device_name = getenv("GGML_OPENVINO_DEVICE") ? getenv("GGML_OPENVINO_DEVICE") : "CPU"; + + // All recognized GGML_OPENVINO_* env vars. Their values are cached here + // once at backend init time and read back via ggml_openvino_getenv_str() + // (raw string) or ggml_openvino_getenv_int() (integer / boolean toggle). + static constexpr const char * env_var_names[] = { + // String values (use ggml_openvino_getenv_str) + "GGML_OPENVINO_DEVICE", + "GGML_OPENVINO_CACHE_DIR", + // Integer values (use ggml_openvino_getenv_int) + "GGML_OPENVINO_PREFILL_CHUNK_SIZE", + // Boolean toggles (treated as int flags via ggml_openvino_getenv_int) + "GGML_OPENVINO_STATEFUL_EXECUTION", + "GGML_OPENVINO_PROFILING", + "GGML_OPENVINO_DUMP_CGRAPH", + "GGML_OPENVINO_DUMP_IR", + "GGML_OPENVINO_DEBUG_INPUT", + "GGML_OPENVINO_DEBUG_OUTPUT", + "GGML_OPENVINO_PRINT_CGRAPH_TENSOR_ADDRESS", + "GGML_OPENVINO_ENABLE_CACHE", + "GGML_OPENVINO_DISABLE_CACHE", + "GGML_OPENVINO_DISABLE_KV_SLICE", + "GGML_OPENVINO_MANUAL_GQA_ATTN", + }; + + for (const char * const & env_var : env_var_names) { + auto * env = getenv(env_var); + if (env) { + environment_variables[env_var] = env; + } + } + + device_name = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE", "CPU"); auto available_devices = ov_singleton_core().get_available_devices(); if (std::find(available_devices.begin(), available_devices.end(), device_name) == available_devices.end()) { GGML_LOG_WARN("GGML OpenVINO Backend: device %s is not available, fallback to CPU\n", device_name.c_str()); @@ -30,7 +62,7 @@ void ggml_openvino_device_config::init() { } is_npu = (device_name == "NPU"); - auto * cache_dir = getenv("GGML_OPENVINO_CACHE_DIR"); + const char * cache_dir = ggml_openvino_getenv_str("GGML_OPENVINO_CACHE_DIR"); if (device_name == "NPU") { compile_config = { {"NPU_COMPILER_DYNAMIC_QUANTIZATION", "YES" }, @@ -119,6 +151,23 @@ const std::string & ggml_openvino_get_device_name() { return ggml_openvino_get_device_config().device_name; } +// Get the value of a GGML_OPENVINO_* env var as a string. Returns +// default_value when the var is unset or set to an empty string. +const char * ggml_openvino_getenv_str(const char * var, const char * default_value) { + auto & env_map = ggml_openvino_get_device_config().environment_variables; + auto it = env_map.find(var); + return (it == env_map.end() || it->second.empty()) ? default_value : it->second.c_str(); +} + +// Get the value of a GGML_OPENVINO_* env var as an int (via std::atoi). +// Returns default_value (0) when the var is unset or empty. Used for both +// integer settings (e.g. GGML_OPENVINO_PREFILL_CHUNK_SIZE) and boolean +// toggles: "0" disables, any non-zero integer enables. +int ggml_openvino_getenv_int(const char * var, int default_value) { + const char * v = ggml_openvino_getenv_str(var, nullptr); + return v ? std::atoi(v) : default_value; +} + // Check if running on NPU bool ggml_openvino_is_npu() { return ggml_openvino_get_device_config().is_npu; @@ -173,7 +222,8 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * return std::nullopt; } if (strncmp(tensor->name, "token_embd.weight", 17) == 0) { - return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); + return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : + ExtraQuantType::Q8_0_C); } if (strncmp(tensor->name, "output.weight", 13) == 0) { return ExtraQuantType::Q8_0_C; @@ -298,6 +348,10 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten layout.is_symmetric = true; break; + case GGML_TYPE_Q5_1: + // u8 weights (5-bit values), asymmetric (scale + zero point) + break; + case GGML_TYPE_Q6_K: layout.weights_per_block = 16; layout.is_symmetric = true; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index cd0baf4a6..c2654fbfa 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -64,6 +64,7 @@ struct ggml_openvino_device_config { bool initialized = false; std::optional remote_context; ov::AnyMap compile_config; + std::unordered_map environment_variables; cl_command_queue cl_queue = nullptr; void init(); @@ -79,6 +80,22 @@ void ggml_openvino_init_device_config(); // Get the device name const std::string & ggml_openvino_get_device_name(); +// Environment variable accessors. All GGML_OPENVINO_* env vars are read once +// during backend init and cached on the device config; consumers must go +// through these helpers (never call ::getenv directly) so behavior stays +// consistent and centralized. +// +// Use ggml_openvino_getenv_str() for string / path values +// (e.g. GGML_OPENVINO_DEVICE, GGML_OPENVINO_CACHE_DIR). The optional +// default_value is returned when the var is unset or empty. +// +// Use ggml_openvino_getenv_int() for boolean toggles and integer settings. +// It returns std::atoi(value) when set, otherwise default_value. For +// boolean use, `if (ggml_openvino_getenv_int(name))` is true iff the value +// is a non-zero integer (so "0" disables, "1" enables). +const char * ggml_openvino_getenv_str(const char * var, const char * default_value = nullptr); +int ggml_openvino_getenv_int(const char * var, int default_value = 0); + // Check if running on NPU bool ggml_openvino_is_npu(); @@ -115,9 +132,9 @@ struct ggml_openvino_weight_extra : public ggml_openvino_extra_base { // Extra data for quantized weight tensors - stores extracted weights/scales/zp and weight node struct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base { - ov::Tensor weights; // U4 or U8 extracted weights - ov::Tensor scales; // F16 scales - ov::Tensor zp; // U4 or U8 zero points (same type as weights) + ov::Tensor weights; // U4 or U8 extracted weights + ov::Tensor scales; // F16 scales + ov::Tensor zp; // U4 or U8 zero points (same type as weights) std::shared_ptr weight_node; // Pre-built OpenVINO weight subgraph ggml_openvino_quantized_weight_extra(ov::Tensor w, ov::Tensor s, ov::Tensor z, std::shared_ptr n) : @@ -132,8 +149,9 @@ struct ggml_openvino_quantized_weight_extra : public ggml_openvino_extra_base { struct ggml_openvino_tensor_extra : public ggml_openvino_extra_base { std::shared_ptr tensor; // For direct use with infer_request - explicit ggml_openvino_tensor_extra(std::shared_ptr t) - : ggml_openvino_extra_base(Type::TENSOR), tensor(std::move(t)) {} + explicit ggml_openvino_tensor_extra(std::shared_ptr t) : + ggml_openvino_extra_base(Type::TENSOR), + tensor(std::move(t)) {} }; // ===================================================== @@ -152,11 +170,11 @@ struct ggml_openvino_extracted_layout { size_t zp_size = 0; // Size of zero points in bytes (U4 or U8) bool is_u4; // true for U4 weights, false for U8 int64_t weights_per_block; // weights per scale/zp block - bool is_symmetric; // true for symmetric quantization + bool is_symmetric; // true for symmetric quantization // Requantization info - bool is_requant = false; // true if this tensor needs requantization - std::optional requant_type; // target requant type if is_requant + bool is_requant = false; // true if this tensor needs requantization + std::optional requant_type; // target requant type if is_requant }; // Calculate the buffer layout for extracted quantized data @@ -164,6 +182,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten ggml_openvino_tensor_extra * ggml_openvino_create_tensor_extra(const ggml_tensor * tensor, bool is_remote); +// Check if a tensor's buffer uses remote (device) memory (e.g. GPU USM) +bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor); + // Register an extra with the tensor's OpenVINO buffer context for proper lifetime management. // This sets tensor->extra and tracks the extra in the buffer context for cleanup. void ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 4f3ebf253..943aef864 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -4,13 +4,14 @@ #include "ggml-backend.h" #include "ggml-impl.h" #include "ggml-openvino-extra.h" +#include "ggml-openvino/openvino/op_table.h" #include "ggml-openvino/utils.h" #include "ggml-quants.h" #include "ggml.h" #include -#include #include +#include #include #include #include @@ -146,8 +147,7 @@ static void * ggml_backend_openvino_buffer_get_base(ggml_backend_buffer_t buffer } static bool is_stateful_enabled() { - static const auto * stateful = getenv("GGML_OPENVINO_STATEFUL_EXECUTION"); - return stateful && *stateful != '\0' && strcmp(stateful, "0") != 0; + return ggml_openvino_getenv_int("GGML_OPENVINO_STATEFUL_EXECUTION") != 0; } static enum ggml_status ggml_backend_openvino_buffer_init_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor) { @@ -367,11 +367,9 @@ static bool ggml_backend_openvino_buffer_cpy_tensor(ggml_backend_buffer_t buffer ggml_backend_openvino_buffer_context * src_ctx = (ggml_backend_openvino_buffer_context *) src->buffer->context; if (src_ctx->is_remote) { - cl_int err = - mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr); + cl_int err = mem_cpy_fn(queue, CL_TRUE, dst->data, src->data, ggml_nbytes(src), 0, nullptr, nullptr); if (err != CL_SUCCESS) { - GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__, - err); + GGML_LOG_ERROR("%s: clEnqueueMemcpyINTEL (device-to-device) failed with error %d\n", __func__, err); return false; } return true; @@ -579,6 +577,17 @@ size_t ggml_backend_openvino_buffer_get_ctx_id(ggml_backend_buffer_t buffer) { return ctx->id; } +bool ggml_openvino_buffer_is_remote(const ggml_tensor * tensor) { + if (tensor == nullptr || tensor->buffer == nullptr) { + return false; + } + if (!ggml_backend_buffer_is_openvino(tensor->buffer)) { + return false; + } + auto * ctx = static_cast(tensor->buffer->context); + return ctx->is_remote; +} + void ggml_openvino_buffer_register_extra(ggml_tensor * tensor, ggml_openvino_extra_base * extra) { GGML_ASSERT(tensor != nullptr); GGML_ASSERT(tensor->buffer != nullptr); @@ -785,6 +794,18 @@ static bool has_view_op_input(const ggml_tensor * op) { return false; } +static bool has_non_contiguous_view_input(const ggml_tensor * op) { + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (op->src[i] == nullptr) { + break; + } + if (op->src[i]->op == GGML_OP_VIEW && !ggml_is_contiguous(op->src[i])) { + return true; + } + } + return false; +} + static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { // pattern of q,k,v should be q->op==PERMUTE, q->src[0]->op==VIEW, q->src[0]->src[0]->view_src==nullptr for (int i = 0; i < 3; i++) { @@ -797,17 +818,107 @@ static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { return true; } +static bool is_gemma3n_flash_attn_pattern(const ggml_tensor * op) { + if (!is_supported_flash_attn_pattern(op)) { + return false; + } + + const ggml_tensor * q_base = + op->src[0] != nullptr && op->src[0]->src[0] != nullptr ? op->src[0]->src[0]->src[0] : nullptr; + const ggml_tensor * k_base = + op->src[1] != nullptr && op->src[1]->src[0] != nullptr ? op->src[1]->src[0]->src[0] : nullptr; + const ggml_tensor * v_base = + op->src[2] != nullptr && op->src[2]->src[0] != nullptr ? op->src[2]->src[0]->src[0] : nullptr; + + if (q_base == nullptr || q_base->op != GGML_OP_ROPE) { + return false; + } + + // gemma3n direct attention path (no KV cache): q=ROPE, k=ROPE, v=RMS_NORM + // Only match this specific pattern to avoid falsely catching other models + // (e.g. Gemma4) that also use scale=1.0 with KV-cache backed attention. + const bool is_qkv_direct = + k_base != nullptr && v_base != nullptr && k_base->op == GGML_OP_ROPE && v_base->op == GGML_OP_RMS_NORM; + + return is_qkv_direct; +} + +static bool checked_mul_size(size_t a, size_t b, size_t & out) { + if (a == 0 || b == 0) { + out = 0; + return true; + } + if (a > SIZE_MAX / b) { + return false; + } + out = a * b; + return true; +} + +static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { + const ggml_tensor * as = op->src[0]; + const ggml_tensor * ids = op->src[2]; + if (as == nullptr || ids == nullptr) { + return true; + } + + // The current OpenVINO translation materializes selected expert weights with + // shape [n_tokens, n_used, rows, k]. Skip cases that would create a very + // large temporary on GPU and let the scheduler fall back instead. + size_t tmp_elems = 1; + if (!checked_mul_size(tmp_elems, static_cast(ids->ne[1]), tmp_elems) || + !checked_mul_size(tmp_elems, static_cast(ids->ne[0]), tmp_elems) || + !checked_mul_size(tmp_elems, static_cast(as->ne[1]), tmp_elems) || + !checked_mul_size(tmp_elems, static_cast(as->ne[0]), tmp_elems)) { + return true; + } + + size_t tmp_bytes = 0; + if (!checked_mul_size(tmp_elems, sizeof(float), tmp_bytes)) { + return true; + } + + static constexpr size_t mul_mat_id_tmp_limit = 1ULL << 30; // 1 GiB + return tmp_bytes > mul_mat_id_tmp_limit; +} + static bool is_op_unsupported_case(const ggml_tensor * op) { switch (op->op) { + case GGML_OP_CONCAT: { + if (op->type == GGML_TYPE_I64) { + return true; + } + break; + } case GGML_OP_GET_ROWS: case GGML_OP_SET_ROWS: { if (op->ne[3] != 1) { return true; } + if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) { + // ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0) + // ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0) + return true; + } + + // Keep the MoE routing weights gather on CPU for GPU runs. Splitting + // only at the later SUM/CLAMP/DIV nodes still leaves this routing path + // numerically unstable for arctic-style MoE graphs. + if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { + return true; + } + break; + } + case GGML_OP_RESHAPE: { + if (strncmp(op->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0 || + strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { + return true; + } break; } case GGML_OP_ADD: - case GGML_OP_MUL: { + case GGML_OP_MUL: + case GGML_OP_SUB: { if (op->src[1]->op == GGML_OP_PERMUTE) { return true; } @@ -818,30 +929,79 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { } break; } + case GGML_OP_ADD_ID: { + // Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids. + if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 || + op->src[2]->type != GGML_TYPE_I32) { + return true; + } + break; + } + case GGML_OP_DIV: { + bool requires_broadcast = false; + for (int i = 0; i < 4; i++) { + if (op->src[0]->ne[i] == op->src[1]->ne[i]) { + continue; + } + + if (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1) { + return true; + } + + requires_broadcast = true; + } + + // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path + // and produce infs for per-channel scale vectors. Keep those DIVs on CPU + // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt) + if (requires_broadcast && ggml_openvino_get_device_name() == "GPU") { + return true; + } + + // qwen3next MoE weight normalization is numerically sensitive on the GPU + // path. Keep the normalization divide on CPU to match the reference. + if (strncmp(op->name, "ffn_moe_weights_norm", sizeof("ffn_moe_weights_norm") - 1) == 0) { + return true; + } + break; + } case GGML_OP_SOFT_MAX: { if (op->src[2] != nullptr) { // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with sinks\n"); return true; } - float scale = 1.0f; - float max_bias = 0.0f; - const auto * op_params = op->op_params; - memcpy(&scale, (const float *) op_params + 0, sizeof(float)); - memcpy(&max_bias, (const float *) op_params + 1, sizeof(float)); - if (max_bias > 0) { - // GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with max_bias > 0\n"); + + if (strncmp(op->name, "ffn_moe_probs", sizeof("ffn_moe_probs") - 1) == 0) { + return true; + } + + // GPU execution of the MoE routing weights softmax is numerically unstable + // when fused with the surrounding GET_ROWS/reshape path. Keep this softmax + // on CPU so the scheduler splits at the same boundary that restores parity. + if (op->src[0] != nullptr && op->src[0]->op == GGML_OP_RESHAPE && op->src[0]->src[0] != nullptr && + strncmp(op->src[0]->src[0]->name, "ffn_moe_weights", sizeof("ffn_moe_weights") - 1) == 0) { + return true; + } + break; + } + case GGML_OP_SUM_ROWS: { + if (strncmp(op->name, "ffn_moe_weights_sum", sizeof("ffn_moe_weights_sum") - 1) == 0) { + return true; + } + + // if the input is PERMUTE skip + if (op->src[0]->op == GGML_OP_PERMUTE) { + return true; + } + break; + } + case GGML_OP_CLAMP: { + if (strncmp(op->name, "ffn_moe_weights_sum_clamped", sizeof("ffn_moe_weights_sum_clamped") - 1) == 0) { return true; } break; } case GGML_OP_FLASH_ATTN_EXT: { - if (op->src[4] != nullptr) { - // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n"); - return true; - } - if (!is_supported_flash_attn_pattern(op)) { - return true; - } float scale = 1.0f; float max_bias = 0.0f; float logit_softcap = 0.0f; @@ -849,6 +1009,21 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { memcpy(&scale, (const float *) op_params + 0, sizeof(float)); memcpy(&max_bias, (const float *) op_params + 1, sizeof(float)); memcpy(&logit_softcap, (const float *) op_params + 2, sizeof(float)); + + // Keep gemma3n flash-attn pattern on CPU for GPU runs to avoid + // accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid + // affecting non-gemma3n models such as Llama-3.2. + if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) { + return true; + } + + if (op->src[4] != nullptr) { + // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n"); + return true; + } + if (!is_supported_flash_attn_pattern(op)) { + return true; + } if (max_bias > 0) { // GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n"); return true; @@ -868,34 +1043,44 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { break; } case GGML_OP_CPY: { - if (op->src[1] != op) { - // GGML_LOG_WARN("OpenVINO backend only supports CPY that is a cast\n"); + if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) { + // GGML_LOG_WARN("OpenVINO backend does not support CPY with non-contiguous data or bf16 types\n"); + return true; + } + // op test case with non-contiguous src or dst + if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) || + (op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) || + (op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) { return true; } break; } case GGML_OP_MUL_MAT: { - if (op->src[0]->type == GGML_TYPE_F16 && op->src[1]->type == GGML_TYPE_F16) { - // Has accuracy issue, try enabling this and see `test-backend-ops -o "MUL_MAT"` - // GGML_LOG_WARN("OpenVINO backend does not support MUL_MAT with two F16 tensors\n"); + if (ggml_openvino_get_device_name() == "GPU" && op->src[1]->op == GGML_OP_SOFT_MAX && + op->src[0]->op == GGML_OP_CONT && op->src[0]->src[0] != nullptr && + op->src[0]->src[0]->op == GGML_OP_TRANSPOSE && op->src[0]->src[0]->src[0] != nullptr && + op->src[0]->src[0]->src[0]->op == GGML_OP_PERMUTE) { return true; } if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { return true; } - if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { - return true; - } - if (ggml_is_quantized(op->src[0]->type) && op->src[0]->ne[1] == 1) { - // MUL_MAT(type_a=q4_0,type_b=f32,m=1,n=2048,k=8192,bs=[1,1],nr=[1,1],per=[0,1,2,3],k_v=0,o=1) - // triggers a bug in ov matmul_shape_inference.hpp - return true; - } if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) { return true; } break; } + case GGML_OP_MUL_MAT_ID: { + if (strncmp(op->name, "ffn_moe_gate_up", sizeof("ffn_moe_gate_up") - 1) == 0 || + strncmp(op->name, "ffn_moe_down", sizeof("ffn_moe_down") - 1) == 0) { + return true; + } + + if (mul_mat_id_requires_large_tmp(op)) { + return true; + } + break; + } case GGML_OP_ROPE: { const int32_t * op_params = op->op_params; const int n_dims = op_params[1]; @@ -909,7 +1094,7 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { // op->src[0]->ne[0]); return true; } - if (op->type != GGML_TYPE_F32) { + if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) { // GGML_LOG_WARN("OpenVINO backend does not support ROPE with type %s\n", ggml_type_name(op->type)); return true; } @@ -930,15 +1115,54 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { } break; } - default: - break; - } - if (op->op == GGML_OP_GET_ROWS) { - if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K)) { - // ERR = 0.000000306 > 0.000000100 GET_ROWS(type=q4_K,n=256,m=5,r=4,be1=1,be2=1,v=0) - // ERR = 0.000000197 > 0.000000100 GET_ROWS(type=q5_K,n=256,m=5,r=4,be1=1,be2=1,v=0) + case GGML_OP_TRANSPOSE: { + // if the type is bf16, will return true + if (op->type == GGML_TYPE_BF16) { + // GGML_LOG_WARN("OpenVINO backend does not support CONT with BF16 type\n"); return true; } + break; + } + case GGML_OP_GATED_DELTA_NET: { + // enable after https://github.com/openvinotoolkit/openvino/pull/35917 is included in OV release + return true; + // if (ggml_openvino_get_device_name() == "GPU" && op->src[0]->ne[2] > 1) { + // // CVS-186471 + // return true; + // } + if (op->src[2]->op == GGML_OP_PERMUTE) { + return true; + } + // kda (per-key-dimension gating) not supported by fused GatedDeltaNet op + if (op->src[3]->ne[0] != 1) { + return true; + } + // v_repeat > 1 (GQA): ggml uses modulo head mapping (h_q = h_v % H_k) + // but the fused op uses consecutive mapping (h_q = h_v / group_size) + if (op->src[2]->ne[1] != op->src[0]->ne[1]) { + return true; + } + // K > 1 (multiple state snapshots) not supported by fused op + if (op->src[5]->ne[1] > 1) { + return true; + } + break; + } + case GGML_OP_SSM_CONV: { + // qwen3next is numerically unstable with OpenVINO SSM_CONV. + // Keep this op on CPU until the OpenVINO implementation is fixed. + return true; + } + case GGML_OP_VIEW: { + // Skip TOPK_MOE fused tests until it is fully supported + // the argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe + if (strcmp(op->name, "selected_experts") == 0) { + return true; + } + break; + } + default: + break; } return false; } @@ -946,24 +1170,47 @@ static bool is_op_unsupported_case(const ggml_tensor * op) { static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { GGML_ASSERT(dev->reg != nullptr); - static std::set supported_types{GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, - GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, - GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; + static std::unordered_set supported_types{ + GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32, GGML_TYPE_Q4_0, + GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; - static const std::set supported_ops{GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW, - /*GGML_OP_CONT,*/ GGML_OP_RESHAPE, GGML_OP_PERMUTE, GGML_OP_TRANSPOSE, - GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, GGML_OP_SCALE, - // softmax is not updated due to replaced by flash_attn_ext - // GGML_OP_SOFT_MAX, - GGML_OP_SET_ROWS, GGML_OP_FLASH_ATTN_EXT, GGML_OP_CPY}; - static const std::set supported_unary_ops{ - GGML_UNARY_OP_GELU, - GGML_UNARY_OP_SILU, - }; - static const std::set supported_glu_ops{ - GGML_GLU_OP_SWIGLU, - GGML_GLU_OP_GEGLU, + // derive supported op sets from the op_table map, keys in + // the map use the full macro name (e.g. "GGML_OP_ADD"), while + // the ggml_*_op_name() helpers return only the trailing part (e.g. "ADD"). + // each set is built once and cached. + static const auto build_supported_sets = [] { + const auto & table = ov::frontend::ggml::get_supported_ops(); + std::unordered_set ops; + std::unordered_set unary_ops; + std::unordered_set glu_ops; + + // GGML_OP_NONE has no translator but is always safe to add to the supported set. + ops.insert(GGML_OP_NONE); + + for (int i = 0; i < GGML_OP_COUNT; ++i) { + const std::string key = std::string("GGML_OP_") + ggml_op_name(static_cast(i)); + if (table.count(key)) { + ops.insert(static_cast(i)); + } + } + for (int i = 0; i < GGML_UNARY_OP_COUNT; ++i) { + const std::string key = std::string("GGML_UNARY_OP_") + ggml_unary_op_name(static_cast(i)); + if (table.count(key)) { + unary_ops.insert(static_cast(i)); + } + } + for (int i = 0; i < GGML_GLU_OP_COUNT; ++i) { + const std::string key = std::string("GGML_GLU_OP_") + ggml_glu_op_name(static_cast(i)); + if (table.count(key)) { + glu_ops.insert(static_cast(i)); + } + } + return std::make_tuple(ops, unary_ops, glu_ops); }; + static const auto supported_sets = build_supported_sets(); + static const auto & supported_ops = std::get<0>(supported_sets); + static const auto & supported_unary_ops = std::get<1>(supported_sets); + static const auto & supported_glu_ops = std::get<2>(supported_sets); switch (op->op) { case GGML_OP_UNARY: { @@ -972,11 +1219,6 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con // GGML_LOG_WARN("OpenVINO backend does not support unary op %s\n", ggml_unary_op_name(ggml_get_unary_op(op))); return false; } - if (has_view_op_input(op)) { - // GGML_LOG_WARN("OpenVINO backend does not support unary op %s with view input\n", - // ggml_unary_op_name(ggml_get_unary_op(op))); - return false; - } break; } case GGML_OP_GLU: { @@ -1003,13 +1245,15 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con return false; } static std::set ops_not_support_view_input{ - GGML_OP_GET_ROWS, - GGML_OP_RMS_NORM, + GGML_OP_L2_NORM, }; if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) { // GGML_LOG_WARN("OpenVINO backend does not support op %s with view input\n", ggml_op_name(op->op)); return false; } + if (op->op == GGML_OP_RMS_NORM && has_non_contiguous_view_input(op)) { + return false; + } } } diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index 57d66df4f..275b95428 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -126,6 +126,68 @@ void extract_q4_1_data(const ggml_tensor * tensor, } } +// Extracts (weight, scales, zp) from Q5_1 tensors. +// Data layout is: |16 bit scale|16 bit min|32 bit qh (5th bits)|32 x 4bit low nibbles|. +// Reconstructed quant q in [0,31]: q = (low nibble) | (qh_bit << 4). Dequant: w*d + m. +// Weights are stored as u8 (5-bit values do not fit u4), matching make_int8_weights. +void extract_q5_1_data(const ggml_tensor * tensor, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + bool use_bias) { + const uint64_t bytes_per_block = 24; // 2 scale + 2 min + 4 qh + 16 (32x0.5) weights + const int qk = 32; + + auto * data = static_cast(tensor->data); + auto * weights = static_cast(weights_arr.data()); // u8 weights, one byte per weight + auto * scales = scales_arr.data::value_type>(); + + // Read a 16-bit little-endian value without aliasing/const-qual violations. + auto read_u16 = [](const uint8_t * p) { + uint16_t v; + memcpy(&v, p, sizeof(v)); + return v; + }; + + auto unpack_block = [&](const uint8_t * block, uint8_t * dst) { + uint32_t qh; + memcpy(&qh, block + 4, sizeof(uint32_t)); + const uint8_t * qs = block + 8; + for (int j = 0; j < qk / 2; ++j) { + const uint8_t lo = qs[j] & 0x0F; + const uint8_t hi = qs[j] >> 4; + const uint8_t bit_lo = (qh >> j) & 1; + const uint8_t bit_hi = (qh >> (j + qk / 2)) & 1; + dst[j] = lo | (bit_lo << 4); // first 16 weights + dst[j + qk / 2] = hi | (bit_hi << 4); // last 16 weights + } + }; + + if (use_bias) { + // Store bias (min) directly as f16: dequant w*d + m + auto * bias = zp_arr.data::value_type>(); + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + const uint8_t * block = data + i * bytes_per_block; + float scale = static_cast(ov::float16::from_bits(read_u16(block))); + float min = static_cast(ov::float16::from_bits(read_u16(block + 2))); + scales[i] = ov::float16(scale); + bias[i] = ov::float16(min); + unpack_block(block, weights + i * qk); + }); + } else { + auto * zp = static_cast(zp_arr.data()); // u8 zero points + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + const uint8_t * block = data + i * bytes_per_block; + float scale = static_cast(ov::float16::from_bits(read_u16(block))); + float min = static_cast(ov::float16::from_bits(read_u16(block + 2))); + scales[i] = ov::float16(scale); + // zp = -min / scale (dequant: (w - zp) * s == w*s + min) + zp[i] = (scale != 0.0f) ? (uint8_t) std::lround(-min / scale) : 0; + unpack_block(block, weights + i * qk); + }); + } +} + // Extracts (weight, scales, zp) from Q8_0 tensors. // Data layout is: |16 bit scale|32 x 8bit weights|. // When zp_arr is empty (symmetric), weights are stored as signed i8 directly. @@ -577,6 +639,7 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, weights_per_block = 32; break; case GGML_TYPE_Q8_0: + case GGML_TYPE_Q5_1: case GGML_TYPE_Q5_K: is_u4 = false; weights_per_block = 32; @@ -601,6 +664,9 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, case GGML_TYPE_Q4_K: extract_q4_k_data(&temp_tensor, weights, scales, zp, use_bias); break; + case GGML_TYPE_Q5_1: + extract_q5_1_data(&temp_tensor, weights, scales, zp, use_bias); + break; case GGML_TYPE_Q8_0: extract_q8_0_data(&temp_tensor, weights, scales, zp); break; diff --git a/ggml/src/ggml-openvino/ggml-quants.h b/ggml/src/ggml-openvino/ggml-quants.h index e4a02297c..28b7c1213 100644 --- a/ggml/src/ggml-openvino/ggml-quants.h +++ b/ggml/src/ggml-openvino/ggml-quants.h @@ -6,7 +6,7 @@ #include #include -void unpack_32_4(const uint8_t* data, uint8_t* dst); +void unpack_32_4(const uint8_t * data, uint8_t * dst); void extract_q4_0_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, @@ -19,12 +19,18 @@ void extract_q4_1_data(const ggml_tensor * tensor, ov::Tensor & zp_arr, bool use_bias = false); +void extract_q5_1_data(const ggml_tensor * tensor, + ov::Tensor & weights_arr, + ov::Tensor & scales_arr, + ov::Tensor & zp_arr, + bool use_bias = false); + void extract_q8_0_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, ov::Tensor & scales_arr, ov::Tensor & zp_arr); -void unpack_256_4(const uint8_t* data, uint8_t* dst); +void unpack_256_4(const uint8_t * data, uint8_t * dst); void extract_q4_k_data(const ggml_tensor * tensor, ov::Tensor & weights_arr, @@ -145,8 +151,8 @@ namespace ov { namespace op { namespace util { // From /src/common/transformations/include/transformations/utils/utils.hpp -bool get_single_value(const std::shared_ptr& const_node, - float& value, +bool get_single_value(const std::shared_ptr & const_node, + float & value, bool check_value_range = true); } // namespace util } // namespace op diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h index 3b8da2be5..9d64fe575 100644 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ b/ggml/src/ggml-openvino/openvino/decoder.h @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include #include @@ -12,22 +14,50 @@ namespace ggml { class GgmlDecoder : public DecoderBase { public: - virtual ov::Any get_attribute(const std::string& name) const = 0; + virtual ov::Any get_attribute(const std::string & name) const = 0; - virtual PartialShape get_input_shape(int node_idx, const std::string& name) const = 0; + virtual PartialShape get_input_shape(int node_idx, const std::string & name) const = 0; - virtual std::vector get_input_stride(int node_idx, const std::string& name) const = 0; + virtual std::vector get_input_stride(int node_idx, const std::string & name) const = 0; - virtual element::Type get_input_type(int node_idx, const std::string& name) const = 0; + virtual size_t get_view_input_size(int node_idx, const std::string & name) const = 0; + + virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual std::vector get_view_input_stride(int node_idx, + const std::string & name, + size_t view_index) const = 0; + + virtual std::vector get_view_input_src_stride(int node_idx, + const std::string & name, + size_t view_index) const = 0; + + virtual Shape get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual Shape get_view_input_src_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual PartialShape get_view_input_ov_shape(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual PartialShape get_view_input_src_ov_shape(int node_idx, + const std::string & name, + size_t view_index) const = 0; + + virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual std::string get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const = 0; + + virtual element::Type get_input_type(int node_idx, const std::string & name) const = 0; virtual size_t get_input_size() const = 0; virtual size_t get_input_size(int node_idx) const = 0; virtual void get_input_node(size_t input_port_idx, - std::string& producer_name, - std::string& producer_output_port_name, - size_t& producer_output_port_index) const = 0; + std::string & producer_name, + std::string & producer_output_port_name, + size_t & producer_output_port_index) const = 0; virtual std::vector get_input_names(int node_idx) const = 0; @@ -35,30 +65,36 @@ public: virtual element::Type get_output_type(const int node_idx) const = 0; - virtual int32_t* get_input_op_params(int node_idx, const std::string& name) const = 0; + virtual std::vector get_output_stride(int node_idx) const = 0; + + virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const = 0; virtual int32_t * get_output_op_params(int node_idx) const = 0; + virtual size_t get_output_op_offset(int node_idx) const = 0; + virtual std::vector get_output_names(int node_idx) const = 0; - virtual const std::string& get_op_type() const = 0; + virtual const std::string & get_op_type() const = 0; - virtual const std::string& get_op_type(int node_idx) const = 0; + virtual const std::string & get_op_type(int node_idx) const = 0; - virtual const std::string& get_op_name() const = 0; + virtual const std::string & get_op_name() const = 0; - virtual const std::string& get_op_name(int node_idx) const = 0; + virtual const std::string & get_op_name(int node_idx) const = 0; virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const = 0; virtual int get_op_case(int node_idx) const = 0; - virtual const std::map>& get_model_inputs() const = 0; - virtual const std::map>& get_model_extra_inputs() const = 0; - virtual const std::map>& get_model_weights() const = 0; + virtual const std::map> & get_model_inputs() const = 0; + virtual const std::map> & get_model_extra_inputs() const = 0; + virtual const std::map> & get_model_weights() const = 0; virtual std::vector get_model_output_names() const = 0; - virtual int32_t* get_rope_params() const = 0; + virtual int32_t * get_rope_params() const = 0; + + virtual bool has_mixed_rope_params() const = 0; virtual std::map get_kv_param_res_names() const = 0; @@ -66,7 +102,11 @@ public: virtual bool is_stateful() const = 0; + virtual bool is_splited_model() const = 0; + virtual int is_swa_layer(int layer) const = 0; + + virtual int32_t get_op_dynamic_dim(int node_idx) const = 0; }; } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/frontend.h b/ggml/src/ggml-openvino/openvino/frontend.h index f1c6f0c3e..72134a3e8 100644 --- a/ggml/src/ggml-openvino/openvino/frontend.h +++ b/ggml/src/ggml-openvino/openvino/frontend.h @@ -15,7 +15,7 @@ public: using Ptr = std::shared_ptr; FrontEnd(); - static std::shared_ptr convert(const InputModel::Ptr& model, bool naive = false); + static std::shared_ptr convert(const InputModel::Ptr & model, bool naive = false); }; } // namespace ggml diff --git a/ggml/src/ggml-openvino/openvino/input_model.h b/ggml/src/ggml-openvino/openvino/input_model.h index ce8434426..6ddcea996 100644 --- a/ggml/src/ggml-openvino/openvino/input_model.h +++ b/ggml/src/ggml-openvino/openvino/input_model.h @@ -1,9 +1,9 @@ #pragma once -#include - #include "decoder.h" +#include + namespace ov { namespace frontend { namespace ggml { @@ -16,9 +16,9 @@ class InputModel : public ov::frontend::InputModel { friend class ::ov::frontend::ggml::FrontEnd; public: - explicit InputModel(const std::shared_ptr& gdecoder); + explicit InputModel(const std::shared_ptr & gdecoder); - const std::shared_ptr& get_model_decoder() const; + const std::shared_ptr & get_model_decoder() const; private: std::shared_ptr m_decoder; diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h index aa484128a..9769c3009 100644 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ b/ggml/src/ggml-openvino/openvino/node_context.h @@ -1,11 +1,11 @@ #pragma once +#include "decoder.h" + #include #include #include -#include "decoder.h" - namespace ov { namespace frontend { namespace ggml { @@ -16,28 +16,24 @@ typedef std::map> TensorMap; class NodeContext : public frontend::NodeContext { public: - NodeContext(const std::shared_ptr& decoder, - std::shared_ptr& tensor_map, + NodeContext(const std::shared_ptr & decoder, + std::shared_ptr & tensor_map, int node_idx, - TranslateSession* translate_session = nullptr) - : ov::frontend::NodeContext(decoder->get_op_type(node_idx)), - m_decoder(decoder), - m_tensor_map(tensor_map), - m_node_idx(node_idx), - m_translate_session(translate_session) { + TranslateSession * translate_session = nullptr) : + ov::frontend::NodeContext(decoder->get_op_type(node_idx)), + m_decoder(decoder), + m_tensor_map(tensor_map), + m_node_idx(node_idx), + m_translate_session(translate_session) { m_input_names = decoder->get_input_names(m_node_idx); m_output_names = decoder->get_output_names(m_node_idx); } - TranslateSession* get_translate_session() const { - return m_translate_session; - } + TranslateSession * get_translate_session() const { return m_translate_session; } - const std::vector& get_input_names() const { return m_input_names; } + const std::vector & get_input_names() const { return m_input_names; } - size_t get_input_size() const override { - return m_decoder->get_input_size(m_node_idx); - } + size_t get_input_size() const override { return m_decoder->get_input_size(m_node_idx); } ov::element::Type get_input_type(size_t index) const { return m_decoder->get_input_type(m_node_idx, m_input_names[index]); @@ -55,42 +51,103 @@ public: PartialShape get_output_shape() const { return m_decoder->get_output_shape(m_node_idx); } - int32_t* get_input_op_params(size_t index) const { + int32_t * get_input_op_params(size_t index) const { return m_decoder->get_input_op_params(m_node_idx, m_input_names[index]); } - int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); } - - ov::element::Type get_output_type() const { - return m_decoder->get_output_type(m_node_idx); + size_t get_view_input_size(size_t index) const { + return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]); } + size_t get_view_input_offset(size_t index, size_t view_index) const { + return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index); + } + + size_t get_view_input_src_offset(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_offset(m_node_idx, m_input_names[index], view_index); + } + + std::vector get_view_input_stride(size_t index, size_t view_index) const { + return m_decoder->get_view_input_stride(m_node_idx, m_input_names[index], view_index); + } + + std::vector get_view_input_src_stride(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_stride(m_node_idx, m_input_names[index], view_index); + } + + ov::Shape get_view_input_ggml_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_ggml_shape(m_node_idx, m_input_names[index], view_index); + } + + ov::Shape get_view_input_src_ggml_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_ggml_shape(m_node_idx, m_input_names[index], view_index); + } + + ov::PartialShape get_view_input_ov_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_ov_shape(m_node_idx, m_input_names[index], view_index); + } + + ov::PartialShape get_view_input_src_ov_shape(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_ov_shape(m_node_idx, m_input_names[index], view_index); + } + + std::string get_view_input_name(size_t index, size_t view_index) const { + return m_decoder->get_view_input_name(m_node_idx, m_input_names[index], view_index); + } + + std::string get_view_input_src_name(size_t index, size_t view_index) const { + return m_decoder->get_view_input_src_name(m_node_idx, m_input_names[index], view_index); + } + + int32_t get_op_dynamic_dim() const { return m_decoder->get_op_dynamic_dim(m_node_idx); } + + int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); } + + size_t get_output_op_offset() const { return m_decoder->get_output_op_offset(m_node_idx); } + + ov::element::Type get_output_type() const { return m_decoder->get_output_type(m_node_idx); } + + std::vector get_output_stride() const { return m_decoder->get_output_stride(m_node_idx); } + Output get_input(int idx) const override { + // Check if this input is a VIEW + size_t view_input_size = m_decoder->get_view_input_size(m_node_idx, m_input_names[idx]); + if (view_input_size > 0) { + // This is a VIEW input, get the base tensor name (last element in the chain) + std::string base_name = + m_decoder->get_view_input_src_name(m_node_idx, m_input_names[idx], view_input_size - 1); + // Check if the VIEW has been resolved (translate_view produced a Slice) + auto view_it = m_tensor_map->find(m_input_names[idx]); + if (!base_name.empty() && view_it != m_tensor_map->end()) { + auto base_it = m_tensor_map->find(base_name); + if (base_it != m_tensor_map->end() && + view_it->second.get_node_shared_ptr() != base_it->second.get_node_shared_ptr()) { + return view_it->second; + } + return base_it->second; + } + if (!base_name.empty()) { + return m_tensor_map->at(base_name); + } + } + // Not a VIEW or failed to get base name, use the original logic return m_tensor_map->at(m_input_names[idx]); } - Output get_input(const std::string& name) const override { + Output get_input(const std::string & name) const override { if (m_tensor_map->find(name) == m_tensor_map->end()) { throw std::runtime_error("'" + name + "' not found in tensor map."); } return m_tensor_map->at(name); } - bool has_input(const std::string& name) const { - return m_tensor_map->find(name) != m_tensor_map->end(); - } + bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); } - const std::string& get_name() const override { - return m_decoder->get_op_name(m_node_idx); - } + const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); } - ov::Any get_attribute_as_any(const std::string& name) const override { - return m_decoder->get_attribute(name); - } + ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); } - int get_op_case() const { - return m_decoder->get_op_case(m_node_idx); - } + int get_op_case() const { return m_decoder->get_op_case(m_node_idx); } bool is_static() const { return m_decoder->is_static(); } @@ -98,14 +155,14 @@ public: private: std::shared_ptr m_decoder; - std::shared_ptr& m_tensor_map; + std::shared_ptr & m_tensor_map; int m_node_idx; - TranslateSession* m_translate_session; + TranslateSession * m_translate_session; std::vector m_input_names; std::vector m_output_names; }; -using CreatorFunction = std::function; +using CreatorFunction = std::function; } // namespace ggml } // namespace frontend diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp new file mode 100644 index 000000000..c8bf08152 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/add_id.cpp @@ -0,0 +1,62 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_add_id(const NodeContext & context) { + num_inputs_check(context, 3, 3); + + auto input = process_view_input_new(context, 0); + auto bias = process_view_input_new(context, 1); + auto ids = process_view_input_new(context, 2); + + // OpenVINO uses reversed GGML dimensions: + // input: [1, n_token, n_used, n_embd] + // bias: [1, 1, n_expert, n_embd] + // ids: [1, 1, n_token, n_used] + auto bias_shape_4d = std::make_shared(bias, ov::element::i64); + auto ids_shape_4d = std::make_shared(ids, ov::element::i64); + + bias = std::make_shared(bias, get_dimensions(bias_shape_4d, {2, 3}), false); + ids = std::make_shared(ids, get_dimensions(ids_shape_4d, {2, 3}), false); + + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); + ov::Output selected_bias = std::make_shared(bias, ids, gather_axis); + selected_bias = std::make_shared( + selected_bias, std::make_shared(input, ov::element::i64), false); + + if (selected_bias.get_element_type() != input.get_element_type()) { + selected_bias = std::make_shared(selected_bias, input.get_element_type()); + } + + ov::Output res = std::make_shared(input, selected_bias); + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/argsort.cpp b/ggml/src/ggml-openvino/openvino/op/argsort.cpp new file mode 100644 index 000000000..bb8344af8 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/argsort.cpp @@ -0,0 +1,47 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_argsort(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + + const int32_t order = context.get_output_op_params()[0]; + + ov::op::v11::TopK::Mode mode; + switch (order) { + case GGML_SORT_ORDER_ASC: + mode = ov::op::v11::TopK::Mode::MIN; + break; + case GGML_SORT_ORDER_DESC: + mode = ov::op::v11::TopK::Mode::MAX; + break; + default: + FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported GGML_OP_ARGSORT order: ", order); + } + + auto k = std::make_shared(get_dimensions(input.get_node_shared_ptr(), {3}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + + auto topk = std::make_shared(input, k, 3, mode, ov::op::v11::TopK::SortType::SORT_VALUES, + context.get_output_type(), false); + + return rename_outputs_with_suffix({topk->output(1)}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/clamp.cpp b/ggml/src/ggml-openvino/openvino/op/clamp.cpp new file mode 100644 index 000000000..070ad33b7 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/clamp.cpp @@ -0,0 +1,33 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_clamp(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CLAMP requires output op params"); + + float min; + float max; + std::memcpy(&min, reinterpret_cast(op_params) + 0, sizeof(float)); + std::memcpy(&max, reinterpret_cast(op_params) + 1, sizeof(float)); + + auto res = std::make_shared(input, min, max); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/concat.cpp b/ggml/src/ggml-openvino/openvino/op/concat.cpp new file mode 100644 index 000000000..4d36a666b --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/concat.cpp @@ -0,0 +1,48 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_concat(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CONCAT requires output op params"); + + const auto output_shape = context.get_output_shape(); + FRONT_END_CHECK_IMPLEMENTED(output_shape.rank().is_static(), "CONCAT requires static output rank"); + + const auto rank = output_shape.rank().get_length(); + const int32_t ggml_dim = op_params[0]; + FRONT_END_CHECK_IMPLEMENTED(ggml_dim >= 0 && ggml_dim < rank, "CONCAT axis is out of range"); + + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + const auto output_type = context.get_output_type(); + + if (input_0.get_element_type() != output_type) { + input_0 = std::make_shared(input_0, output_type); + } + if (input_1.get_element_type() != output_type) { + input_1 = std::make_shared(input_1, output_type); + } + + const auto axis = static_cast(rank - 1 - ggml_dim); + auto res = std::make_shared(OutputVector{input_0, input_1}, axis); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp index 6160dd744..1d6cc6721 100644 --- a/ggml/src/ggml-openvino/openvino/op/cont.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cont.cpp @@ -18,27 +18,19 @@ namespace op { OutputVector translate_cont(const NodeContext & context) { num_inputs_check(context, 1, 1); - int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3, "Unsupported CONT case"); - auto src_shape = context.get_input_shape(0).to_shape(); auto dst_shape = context.get_output_shape().to_shape(); - ov::Output res; - if (op_case == 1) { - // The input comes from a PERMUTE - throw std::runtime_error("Code of this case might be outdated"); - dst_shape[1] = -1; - res = std::make_shared( - context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false); - } else if (op_case == 2) { - // The input comes from a TRANSPOSE - return {context.get_input(0)}; - } else { - // The input comes from a VIEW - res = process_view_input(context, 0); + if (context.get_op_dynamic_dim() != -1) { + dst_shape[3 - context.get_op_dynamic_dim()] = -1; } + auto input = process_view_input_new(context, 0); + + ov::Output res; + res = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false); + return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/cpy.cpp b/ggml/src/ggml-openvino/openvino/op/cpy.cpp index 831117208..3a4355021 100644 --- a/ggml/src/ggml-openvino/openvino/op/cpy.cpp +++ b/ggml/src/ggml-openvino/openvino/op/cpy.cpp @@ -3,7 +3,9 @@ #include "../utils.h" #include +#include #include +#include namespace ov { namespace frontend { @@ -11,7 +13,18 @@ namespace ggml { namespace op { OutputVector translate_cpy(const NodeContext & context) { - auto res = std::make_shared(context.get_input(0), context.get_output_type()); + auto input = process_view_input_new(context, 0); + auto input_shape = context.get_input_shape(0); + auto output_shape = context.get_output_shape(); + + // Non-cast CPY may need a reshape (e.g. [3,192,1,1] -> [576,1,1,1]) + if (input_shape != output_shape) { + auto new_shape = ov::op::v0::Constant::create( + ov::element::i64, {static_cast(output_shape.rank().get_length())}, output_shape.to_shape()); + input = std::make_shared(input, new_shape, false); + } + + auto res = std::make_shared(input, context.get_output_type()); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/div.cpp b/ggml/src/ggml-openvino/openvino/op/div.cpp new file mode 100644 index 000000000..11dd9dece --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/div.cpp @@ -0,0 +1,146 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +namespace { + +bool is_silu_div_pattern(const ov::Output & numerator, + const ov::Output & denominator, + const NodeContext & context) { + if (context.get_input_size() != 2) { + return false; + } + + const auto * unary_op = reinterpret_cast(context.get_input_op_params(0)); + if (unary_op == nullptr || *unary_op != GGML_UNARY_OP_SILU) { + return false; + } + + auto mul = std::dynamic_pointer_cast(numerator.get_node_shared_ptr()); + if (!mul) { + return false; + } + + const auto denom_node = denominator.get_node_shared_ptr(); + const auto mul_input_0 = mul->input_value(0).get_node_shared_ptr(); + const auto mul_input_1 = mul->input_value(1).get_node_shared_ptr(); + + auto sigmoid = std::dynamic_pointer_cast(mul_input_1); + if (mul_input_0 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node) { + return true; + } + + sigmoid = std::dynamic_pointer_cast(mul_input_0); + return mul_input_1 == denom_node && sigmoid && sigmoid->input_value(0).get_node_shared_ptr() == denom_node; +} + +ov::Output repeat_input_to_match(const NodeContext & context, + const ov::Output & input, + const ov::Output & target, + size_t input_index) { + const auto input_shape = context.get_input_shape(input_index); + const auto target_shape = context.get_input_shape(0); + + if (input_shape == target_shape) { + return input; + } + + if (input_shape.rank().is_static() && target_shape.rank().is_static()) { + const auto rank = static_cast(input_shape.rank().get_length()); + std::vector repeats(rank, 1); + bool needs_repeat = false; + + for (size_t axis = 0; axis < rank; ++axis) { + FRONT_END_OP_CONVERSION_CHECK(input_shape[axis].is_static() && target_shape[axis].is_static(), + "DIV repeat requires static dimensions on both inputs"); + + const int64_t input_dim = input_shape[axis].get_length(); + const int64_t target_dim = target_shape[axis].get_length(); + + FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && target_dim > 0 && target_dim % input_dim == 0, + "DIV input shape ", input_shape, " cannot repeat to match ", target_shape); + + repeats[axis] = target_dim / input_dim; + needs_repeat = needs_repeat || repeats[axis] != 1; + } + + if (!needs_repeat) { + return input; + } + + auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats); + return std::make_shared(input, repeats_node); + } + + auto input_shape_node = std::make_shared(input, ov::element::i64); + auto target_shape_node = std::make_shared(target, ov::element::i64); + auto repeats_node = std::make_shared(target_shape_node, input_shape_node); + return std::make_shared(input, repeats_node); +} + +} // namespace + +OutputVector translate_div(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + + if (is_silu_div_pattern(input_0, input_1, context)) { + ov::Output res = std::make_shared(input_1); + if (res.get_element_type() != context.get_output_type()) { + res = std::make_shared(res, context.get_output_type()); + } + return rename_outputs_with_suffix({res}, context.get_name()); + } + + input_1 = repeat_input_to_match(context, input_1, input_0, 1); + + const auto output_type = context.get_output_type(); + const bool use_f32_compute = input_0.get_element_type() != ov::element::f32 || + input_1.get_element_type() != ov::element::f32 || output_type != ov::element::f32; + + if (use_f32_compute) { + input_0 = std::make_shared(input_0, ov::element::f32); + input_1 = std::make_shared(input_1, ov::element::f32); + } + + ov::Output res = std::make_shared(input_0, input_1); + if (use_f32_compute) { + // Keep the reciprocal/divide path in FP32. Without this hint, the GPU + // plugin can still compress the subgraph back to FP16 and overflow on + // small shexp gate values (e.g. silu(x) / x in qwen2moe). + ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(0)); + ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(1)); + } + if (res.get_element_type() != output_type) { + auto output_convert = std::make_shared(res, output_type); + if (use_f32_compute) { + ov::mark_as_precision_sensitive(output_convert->input(0)); + } + res = output_convert; + } + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 42602a730..582df0130 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -1,15 +1,21 @@ #include "../node_context.h" #include "../op_table.h" #include "../utils.h" +#include "ggml-openvino/ggml-openvino-extra.h" #include +#include #include +#include #include #include #include #include +#include +#include #include #include +#include #include #include #include @@ -34,36 +40,115 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { auto q = std::make_shared(q_f32, ov::element::f16); auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale}); - ov::Output mask_sliced, res; + ov::Output res; + + // For stateful std::string mask_name = "KQ_mask_sliced"; if (context.get_input_names()[3].find("swa") != std::string::npos) { mask_name = "KQ_mask_swa_sliced"; } if (context.has_input(mask_name)) { - mask_sliced = context.get_input(mask_name); - } else { - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - auto token_len = get_dimensions(q, {2}); - mask_sliced = std::make_shared(mask, zero, token_len, one, two); + mask = context.get_input(mask_name); } - if (mask_sliced.get_element_type() != ov::element::f16) { - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); + if (mask.get_element_type() != ov::element::f16) { + mask = std::make_shared(mask, ov::element::f16); } - auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) { - int64_t factor = num_heads / num_heads_kv; - if (factor > 1 && num_heads_kv > 1) { + //auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) { + // int64_t factor = num_heads / num_heads_kv; + // if (factor > 1 && num_heads_kv > 1) { + auto q_shape = context.get_input_shape(0).to_shape(); + auto k_shape = context.get_input_shape(1).to_shape(); + const int64_t num_heads = q_shape[1]; + const int64_t num_heads_kv = k_shape[1]; + const int64_t head_size = q_shape[3]; + const int64_t factor = num_heads / num_heads_kv; + + // Manual GQA attention: enabled by default on GPU in stateless mode. + // Set GGML_OPENVINO_MANUAL_GQA_ATTN to a positive value (e.g. 1) to force-enable, + // or to 0 to force-disable. Unset falls back to the device-based default. + static const bool manual_gqa_enabled = []() { + const char * env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); + if (env != nullptr) { + return ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0; + } + const char * dev = ggml_openvino_getenv_str("GGML_OPENVINO_DEVICE"); + return dev != nullptr && std::string(dev) == "GPU"; + }(); + const bool use_manual_gqa_attention = + manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful(); + + if (use_manual_gqa_attention) { + // Q, K, V arrive as [B, n_heads(_kv), S, head_size], where B is the active + // batch (n_seq_active) and may be > 1 (llama-perplexity, llama-server -np > 1) + // or dynamic. Reshape to + // K_r: [B, num_heads_kv, 1, S, head_size] + // Q_r: [B, num_heads_kv, factor, S_q, head_size] + // and let MatMul broadcast across the factor dim without materialising + // an expanded K/V. The leading 0 + special_zero=true copies B at runtime, + // so this is correct for B == 1, B > 1, and dynamic B alike. Only the head + // dims and head_size are baked in as literals; the sequence dim stays -1. + auto k_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + std::vector{0, num_heads_kv, 1, -1, head_size}); + auto v_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + std::vector{0, num_heads_kv, 1, -1, head_size}); + auto q_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + std::vector{0, num_heads_kv, factor, -1, head_size}); + + auto k_r = std::make_shared(k, k_5d_shape, true); + auto v_r = std::make_shared(v, v_5d_shape, true); + auto q_r = std::make_shared(q, q_5d_shape, true); + + // QK^T → [B, num_heads_kv, factor, S_q, S_k] + auto qk = std::make_shared(q_r, k_r, /*tA=*/false, /*tB=*/true); + auto qk_scaled = std::make_shared(qk, scale_node); + + // Mask arrives as [B, 1, S_q, S_k]. Unsqueeze a factor axis at position 2 to + // get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the + // [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and + // 1→factor on the head dims. + auto mask_unsq1 = + std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2})); + // mask_unsq1: [B, 1, 1, S_q, S_k] (rank 5) + ov::Output qk_masked = std::make_shared(qk_scaled, mask_unsq1); + + auto softmax = std::make_shared(qk_masked, /*axis=*/-1); + + // softmax @ V → [B, num_heads_kv, factor, S_q, head_size] + auto attn = std::make_shared(softmax, v_r); + + // Reshape back to [B, num_heads, S_q, head_size] (combine num_heads_kv * factor). + // Leading 0 + special_zero=true copies B at runtime. + auto out_4d_shape = + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, num_heads, -1, head_size}); + auto out_4d = std::make_shared(attn, out_4d_shape, true); + + // The standard SDPA path's downstream is Transpose(0,2,1,3) → Convert(f32). + // Replicate it here so callers see the same output layout/dtype. + res = std::make_shared( + out_4d, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); + res = std::make_shared(res, ov::element::f32); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + // Default path: explicit Broadcast → SDPA. Kept as the fallback because + // (a) it goes through the GPU plugin's micro-SDPA fast path (FlashAttention + // tiles via DPAS), and (b) the manual path above is still being validated. + auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output kv) { + int64_t f = n_heads / n_heads_kv; + if (f > 1 && n_heads_kv > 1) { ov::Output kv_broadcast_shape, kv_unsqueezed, new_kv_shape; auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2}); kv_unsqueezed = std::make_shared(kv, unsqueeze_axes); - kv_broadcast_shape = ov::op::v0::Constant::create( - ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); + kv_broadcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, + {(int64_t) 1, (int64_t) 1, f, (int64_t) 1, (int64_t) 1}); new_kv_shape = - ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size}); + ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, n_heads, (int64_t) -1, hs}); + // ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); + //new_kv_shape = + // ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size}); kv = std::make_shared(kv_unsqueezed, kv_broadcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); @@ -72,12 +157,14 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { return kv; }; - auto q_shape = context.get_input_shape(0).to_shape(); - auto k_shape = context.get_input_shape(1).to_shape(); - k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k); - v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v); + //auto q_shape = context.get_input_shape(0).to_shape(); + //auto k_shape = context.get_input_shape(1).to_shape(); + //k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k); + //v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v); + k = tile_kv(num_heads, num_heads_kv, head_size, k); + v = tile_kv(num_heads, num_heads_kv, head_size, v); - auto sdpa = std::make_shared(q, k, v, mask_sliced, scale_node, false); + auto sdpa = std::make_shared(q, k, v, mask, scale_node, false); res = std::make_shared(sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); res = std::make_shared(res, ov::element::f32); diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp new file mode 100644 index 000000000..26c4bbfa9 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp @@ -0,0 +1,282 @@ +#include "gated_delta_net.hpp" + +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +static OutputVector translate_gated_delta_net_ref(const NodeContext & context); + +OutputVector translate_gated_delta_net(const NodeContext & context) { + // auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v] + // auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k] + + // // Fused GatedDeltaNet op only supports scalar gate (kda=0). + // // Fall back to reference implementation for per-key-dimension gating. + // // if (kda) { + // // return translate_gated_delta_net_ref(context); + // // } + + // auto q = context.get_input(0); + // auto k = context.get_input(1); + // auto v = context.get_input(2); + // auto g = context.get_input(3); + // auto beta = context.get_input(4); + // auto state = context.get_input(5); + + // const int64_t B = v_shape[0]; + // const int64_t T = v_shape[1]; + // const int64_t H_v = v_shape[2]; + // const int64_t S_v = v_shape[3]; + // const int64_t S_k = q_shape[3]; + + // // ggml state layout (OV notation): [B, H_v, value_dim, key_dim] + // // GatedDeltaNet op expects: [B, H_v, key_dim, value_dim] + // auto state_reshape_shape = + // ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, S_v, S_k}); + // state = std::make_shared(state, state_reshape_shape, false); + // auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 1, 3, 2}); + // state = std::make_shared(state, state_perm); + + // g = std::make_shared(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3})); + // beta = std::make_shared(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3})); + + // auto gdn = std::make_shared(q, k, v, state, g, beta); + + // auto attn_4d = gdn->output(0); + // auto state_4d = gdn->output(1); // [B, H_v, key_dim, value_dim] + // // Transpose output state back to ggml layout [B, H_v, value_dim, key_dim] + // auto state_transposed = std::make_shared(state_4d, state_perm); + // auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + // auto attn = std::make_shared(attn_4d, flat_shape_1d, false); + // auto new_state = std::make_shared(state_transposed, flat_shape_1d, false); + // auto packed = std::make_shared(ov::OutputVector{attn, new_state}, 0); + // auto out_shape = + // ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, T * B + S_v * B, S_v * H_v}); + // auto res = std::make_shared(packed, out_shape, false); + + // return rename_outputs_with_suffix({res}, context.get_name()); + + // The OV version in CI does not have the GatedDeltaNet op, so use reference implementation for now. + return translate_gated_delta_net_ref(context); +} + +static OutputVector translate_gated_delta_net_ref(const NodeContext & context) { + num_inputs_check(context, 6, 6); + + // Inputs (OV shapes are reversed from ggml): + // ggml: q[S_k, H_k, T, B], k[S_k, H_k, T, B], v[S_v, H_v, T, B] + // OV: q[B, T, H_k, S_k], k[B, T, H_k, S_k], v[B, T, H_v, S_v] + // ggml: g[1 or S_v, H_v, T, B], beta[1, H_v, T, B] + // OV: g[B, T, H_v, 1 or S_v], beta[B, T, H_v, 1] + // ggml: state[S_v, S_v, H_v, B] + // OV: state[B, H_v, S_v, S_v] + auto q = process_view_input_new(context, 0); + auto k = process_view_input_new(context, 1); + auto v = process_view_input_new(context, 2); + auto g = process_view_input_new(context, 3); + auto beta = process_view_input_new(context, 4); + auto state = process_view_input_new(context, 5); + + auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v] + auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k] + auto g_shape = context.get_input_shape(3).to_shape(); // [B, T, H_v, 1 or S_v] + + const int64_t B = v_shape[0]; + const int64_t T = v_shape[1]; + const int64_t H_v = v_shape[2]; + const int64_t S_v = v_shape[3]; + const int64_t H_k = q_shape[2]; + const bool kda = (g_shape[3] == (size_t) S_v); + + const int64_t rq1 = H_v / H_k; // head repeat factor + const float scale = 1.0f / std::sqrt((float) S_v); + + auto axis_1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis_2 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); + + // Transpose inputs from [B, T, H, S] to [B, H, T, S] for easier per-head processing + auto perm_0213 = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 2, 1, 3}); + auto q_t = std::make_shared(q, perm_0213); // [B, H_k, T, S_k] + auto k_t = std::make_shared(k, perm_0213); // [B, H_k, T, S_k] + auto v_t = std::make_shared(v, perm_0213); // [B, H_v, T, S_v] + auto g_t = std::make_shared(g, perm_0213); // [B, H_v, T, 1 or S_v] + auto beta_t = std::make_shared(beta, perm_0213); // [B, H_v, T, 1] + + // Broadcast Q, K heads to match V heads if GQA is used (H_v > H_k) + ov::Output q_bh = q_t; + ov::Output k_bh = k_t; + if (rq1 > 1) { + auto q_unsq = std::make_shared(q_t, axis_2); // [B, H_k, 1, T, S] + auto k_unsq = std::make_shared(k_t, axis_2); // [B, H_k, 1, T, S] + + auto bcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{1, 1, rq1, 1, 1}); + auto q_bcast = + std::make_shared(q_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); + auto k_bcast = + std::make_shared(k_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); + + // Transpose [B, H_k, rq1, T, S] -> [B, rq1, H_k, T, S] so that reshape merges + // as [rq1, H_k] giving repeat-blocks pattern matching CPU: iq1 = iv1 % H_k + auto perm_5d = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{0, 2, 1, 3, 4}); + auto q_transposed = std::make_shared(q_bcast, perm_5d); + auto k_transposed = std::make_shared(k_bcast, perm_5d); + + auto new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, T, S_v}); + q_bh = std::make_shared(q_transposed, new_shape, false); + k_bh = std::make_shared(k_transposed, new_shape, false); + } + + // Merge batch and head dims: [B*H_v, T, S_v] + auto merge_bh = [&](ov::Output x, int64_t last_dim) { + auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, T, last_dim}); + return std::make_shared(x, shape, false); + }; + + auto q_m = merge_bh(q_bh, S_v); // [B*H_v, T, S_v] + auto k_m = merge_bh(k_bh, S_v); // [B*H_v, T, S_v] + auto v_m = merge_bh(v_t, S_v); // [B*H_v, T, S_v] + auto g_m = merge_bh(g_t, kda ? S_v : 1); // [B*H_v, T, 1 or S_v] + auto beta_m = merge_bh(beta_t, 1); // [B*H_v, T, 1] + + // State: [B, H_v, S_v, S_v] -> [B*H_v, S_v, S_v] + auto state_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, S_v, S_v}); + auto state_m = std::make_shared(state, state_shape, false); + + auto scale_const = ov::op::v0::Constant::create(ov::element::f32, {}, std::vector{scale}); + + // --- Build Loop body --- + // Body parameters (no iteration counter needed, use -1 in special ports) + auto body_state = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_q = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_k = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_v = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_g = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_beta = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); + auto body_iter = std::make_shared(ov::element::i64, ov::Shape{1}); + + // Condition output (always true - we rely on trip_count for termination) + auto body_cond_out = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector{true}); + + // Gather current token from invariant inputs using iteration counter + auto q_t_cur = std::make_shared(body_q, body_iter, axis_1); // [B*H_v, 1, S_v] + auto k_t_cur = std::make_shared(body_k, body_iter, axis_1); // [B*H_v, 1, S_v] + auto v_t_cur = std::make_shared(body_v, body_iter, axis_1); // [B*H_v, 1, S_v] + auto g_t_cur = std::make_shared(body_g, body_iter, axis_1); // [B*H_v, 1, 1 or S_v] + auto b_t_cur = std::make_shared(body_beta, body_iter, axis_1); // [B*H_v, 1, 1] + + // Squeeze token dim + auto q_cur = std::make_shared(q_t_cur, axis_1); // [B*H_v, S_v] + auto k_cur = std::make_shared(k_t_cur, axis_1); // [B*H_v, S_v] + auto v_cur = std::make_shared(v_t_cur, axis_1); // [B*H_v, S_v] + auto g_cur = std::make_shared(g_t_cur, axis_1); // [B*H_v, 1 or S_v] + auto b_cur = std::make_shared(b_t_cur, axis_1); // [B*H_v, 1] + + // Step 1: Apply decay gate to state + auto exp_g = std::make_shared(g_cur); // [B*H_v, 1 or S_v] + auto exp_g_unsq = std::make_shared(exp_g, axis_1); // [B*H_v, 1, 1 or S_v] + auto state_decayed = std::make_shared(body_state, exp_g_unsq); // [B*H_v, S_v, S_v] + + // Step 2: delta = (v - S @ k) * beta + auto k_col = std::make_shared(k_cur, axis_2); // [B*H_v, S_v, 1] + auto sk = std::make_shared(state_decayed, k_col, false, false); // [B*H_v, S_v, 1] + auto sk_sq = std::make_shared(sk, axis_2); // [B*H_v, S_v] + auto v_minus_sk = std::make_shared(v_cur, sk_sq); // [B*H_v, S_v] + auto delta = std::make_shared(v_minus_sk, b_cur); // [B*H_v, S_v] + + // Step 3: state += outer(delta, k) + auto delta_col = std::make_shared(delta, axis_2); // [B*H_v, S_v, 1] + auto k_row = std::make_shared(k_cur, axis_1); // [B*H_v, 1, S_v] + auto outer_prod = std::make_shared(delta_col, k_row, false, false); // [B*H_v, S_v, S_v] + auto state_updated = std::make_shared(state_decayed, outer_prod); // [B*H_v, S_v, S_v] + + // Step 4: attn_out = S @ q * scale + auto q_col = std::make_shared(q_cur, axis_2); // [B*H_v, S_v, 1] + auto sq = std::make_shared(state_updated, q_col, false, false); // [B*H_v, S_v, 1] + auto sq_squeezed = std::make_shared(sq, axis_2); // [B*H_v, S_v] + auto attn_out = std::make_shared(sq_squeezed, scale_const); // [B*H_v, S_v] + + // Unsqueeze attn_out to [B*H_v, 1, S_v] for scan output concatenation + auto attn_out_unsq = std::make_shared(attn_out, axis_1); // [B*H_v, 1, S_v] + + // --- Assemble Loop --- + // Body: results = [condition, state_updated, attn_out_unsq] + auto body = std::make_shared( + ov::OutputVector{body_cond_out, state_updated, attn_out_unsq}, + ov::ParameterVector{body_iter, body_state, body_q, body_k, body_v, body_g, body_beta}); + + auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector{T}); + auto exec_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector{true}); + + auto loop = std::make_shared(trip_count, exec_cond); + loop->set_function(body); + loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0}); + + // Carried state: feeds back from body output 1 to body_state param + loop->set_merged_input(body_state, state_m, state_updated); + // Invariant inputs: passed through unchanged each iteration + loop->set_invariant_input(body_q, q_m); + loop->set_invariant_input(body_k, k_m); + loop->set_invariant_input(body_v, v_m); + loop->set_invariant_input(body_g, g_m); + loop->set_invariant_input(body_beta, beta_m); + + // Loop outputs: + // 1) Final state (last iteration value of state_updated) + auto final_state_out = loop->get_iter_value(state_updated, -1); // [B*H_v, S_v, S_v] + // 2) Concatenated attention outputs across all iterations along axis 1 + auto attn_concat_out = loop->get_concatenated_slices(attn_out_unsq, 0, 1, 1, -1, 1); // [B*H_v, T, S_v] + + // --- Pack outputs to match ggml layout --- + // ggml output ne = {S_v*H, T*B + S_v*B, 1, 1} -> OV [1, 1, T*B+S_v*B, S_v*H_v] + // attn: [B, T, H_v, S_v] row-major, state: [B, H_v, S_v, S_v] row-major + + // attn: [B*H_v, T, S_v] -> [B, H_v, T, S_v] -> transpose to [B, T, H_v, S_v] -> flatten + auto attn_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, T, S_v}); + auto attn_4d = std::make_shared(attn_concat_out, attn_4d_shape, false); + auto attn_perm = std::make_shared(attn_4d, perm_0213); // [B, T, H_v, S_v] + + auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{-1}); + auto attn_1d = std::make_shared(attn_perm, flat_shape_1d, false); + + // state: [B*H_v, S_v, S_v] -> [B, H_v, S_v, S_v] -> flatten + auto state_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, S_v, S_v}); + auto state_4d = std::make_shared(final_state_out, state_4d_shape, false); + auto state_1d = std::make_shared(state_4d, flat_shape_1d, false); + + // Concat [attn | state] and reshape to final output + auto packed = std::make_shared(ov::OutputVector{attn_1d, state_1d}, 0); + auto out_shape = + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, T * B + S_v * B, S_v * H_v}); + auto res = std::make_shared(packed, out_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp new file mode 100644 index 000000000..20a4cfdfe --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include "openvino/op/op.hpp" + +namespace ov::op::internal { +/// \note GatedDeltaNet op class is under development and subject to change +/// +/// \brief Operator performing Gated Delta Net computation +/// \ingroup ov_ops_cpp_api +class OPENVINO_API GatedDeltaNet : public ov::op::Op { +public: + OPENVINO_OP("GatedDeltaNet") + + GatedDeltaNet() = default; + /// \brief Constructs a GatedDeltaNet operation. + /// + /// \param query Query tensor input. + /// \param key Key tensor input. + /// \param value Value tensor input. + /// \param recurrent_state Initial recurrent state tensor. + /// \param gate Gate tensor controlling state decay/update. + /// \param beta Beta tensor scaling the delta update. + /// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op. + /// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled. + /// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled. + GatedDeltaNet(const Output& query, + const Output& key, + const Output& value, + const Output& recurrent_state, + const Output& gate, + const Output& beta, + const bool fuse_qk_l2norm = false, + const float q_l2_norm_eps = 1e-6F, + const float k_l2_norm_eps = 1e-6F); + + /// \brief Constructs a GatedDeltaNet operation from input vector. + /// + /// \param args Input tensor vector in order: query, key, value, recurrent_state, gate, beta. + /// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op. + /// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled. + /// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled. + GatedDeltaNet(const ov::OutputVector& args, + const bool fuse_qk_l2norm = false, + const float q_l2_norm_eps = 1e-6F, + const float k_l2_norm_eps = 1e-6F); + void validate_and_infer_types() override; + bool visit_attributes(AttributeVisitor& visitor) override; + std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; + bool get_fuse_qk_l2norm() const { + return m_fuse_qk_l2norm; + } + float get_q_l2_norm_eps() const { + return m_q_l2_norm_eps; + } + float get_k_l2_norm_eps() const { + return m_k_l2_norm_eps; + } + +private: + bool m_fuse_qk_l2norm = false; + float m_q_l2_norm_eps = 1e-6F; + float m_k_l2_norm_eps = 1e-6F; +}; + +} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp index 49f51b7ca..380e70a72 100644 --- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp @@ -18,16 +18,9 @@ namespace op { OutputVector translate_get_rows(const NodeContext & context) { num_inputs_check(context, 2, 2); - int op_case = context.get_op_case(); - Output res; - auto data = context.get_input(0); - auto indices = context.get_input(1); - - if (op_case == 2) { - // The input comes from a VIEW - indices = process_view_input(context, 1); - } + auto data = process_view_input_new(context, 0); + auto indices = process_view_input_new(context, 1); // data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case // data[x,y] ind[1,1,1,x'] normal case diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp index d9fa4c243..a54870d9d 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -21,23 +22,26 @@ OutputVector translate_glu_geglu(const NodeContext & context) { ov::Output src0; ov::Output src1; if (context.get_input_size() == 2) { - src0 = context.get_input(0); - src1 = context.get_input(1); + // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts): + // resolve them so each half has its real sliced shape, not the base tensor. + src0 = process_view_input_new(context, 0); + src1 = process_view_input_new(context, 1); } else { // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2. // Both halves are nc elements; if the dimension is odd, the last element is dropped. // Use Slice instead of Split to handle odd dimensions correctly. - auto combined = context.get_input(0); + // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first. + auto combined = process_view_input_new(context, 0); auto combined_shape = combined.get_partial_shape(); int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); int64_t nc = last_dim_val / 2; - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); + auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); + auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); src0 = std::make_shared(combined, start0, stop0, step, axis); src1 = std::make_shared(combined, start1, stop1, step, axis); @@ -49,6 +53,16 @@ OutputVector translate_glu_geglu(const NodeContext & context) { std::swap(src0, src1); } + if (context.is_static()) { + // TODO: Temporary solution for NPU accuracy issue due to fp16 overflow + // To be removed once permanent solution is implemented + // Justification: + // For |x| > 5, GELU(x) ≈ max(x, 0) (behaves like ReLU) + // So Clamp(-10, 10) only affects values where GELU would return ≈ x anyway. + // The only loss: values > 10 get mapped to 10 instead of x. + // In practice, FFN intermediates rarely exceed 10 after GEGLU gating. + src0 = std::make_shared(src0, -10.0, 10.0); + } auto gelu = std::make_shared(src0); auto res = std::make_shared(gelu, src1); diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp index 00ed7951a..5c46e0713 100644 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp @@ -21,23 +21,26 @@ OutputVector translate_glu_swiglu(const NodeContext & context) { ov::Output src0; ov::Output src1; if (context.get_input_size() == 2) { - src0 = context.get_input(0); - src1 = context.get_input(1); + // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts): + // resolve them so each half has its real sliced shape, not the base tensor. + src0 = process_view_input_new(context, 0); + src1 = process_view_input_new(context, 1); } else { // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2. // Both halves are nc elements; if the dimension is odd, the last element is dropped. // Use Slice instead of Split to handle odd dimensions correctly. - auto combined = context.get_input(0); + // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first. + auto combined = process_view_input_new(context, 0); auto combined_shape = combined.get_partial_shape(); int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); int64_t nc = last_dim_val / 2; - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); + auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); + auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); src0 = std::make_shared(combined, start0, stop0, step, axis); src1 = std::make_shared(combined, start1, stop1, step, axis); diff --git a/ggml/src/ggml-openvino/openvino/op/im2col.cpp b/ggml/src/ggml-openvino/openvino/op/im2col.cpp new file mode 100644 index 000000000..856e97f79 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/im2col.cpp @@ -0,0 +1,120 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml-impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_im2col(const NodeContext & context) { + num_inputs_check(context, 2, 2); + const int32_t * params = context.get_output_op_params(); + int32_t s0 = params[0]; + int32_t s1 = params[1]; + int32_t p0 = params[2]; + int32_t p1 = params[3]; + int32_t d0 = params[4]; + int32_t d1 = params[5]; + bool is_2D = params[6] == 1; + ov::Output res; + + ov::Output image = context.get_input(1); + const ov::Shape kernel_shape = context.get_input(0).get_shape(); + + const size_t IC = is_2D ? kernel_shape[1] : kernel_shape[2]; + const size_t KH = is_2D ? kernel_shape[2] : 1; + const size_t KW = kernel_shape[3]; + + int32_t stride_w = s0; + int32_t stride_h = is_2D ? s1 : 1; + int32_t pad_w = p0; + int32_t pad_h = is_2D ? p1 : 0; + int32_t dil_w = d0; + int32_t dil_h = is_2D ? d1 : 1; + + if (!is_2D) { + // GGML input shape: [IW, IC, N, 1] + // OpenVINO input shape: [1, N, IC, IW] + // Reshape image to: [N, IC, 1, IW] + const ov::Shape image_shape = image.get_shape(); + const size_t N = image_shape[1]; + const size_t IW = image_shape[3]; + auto image_reshape_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N), static_cast(IC), 1, static_cast(IW)}); + image = std::make_shared(image, image_reshape_shape, false); + } + + const ov::Shape patch_sizes = {KH, KW}; + const ov::Strides strides = {static_cast(stride_h), static_cast(stride_w)}; + const ov::Shape rates = {static_cast(dil_h), static_cast(dil_w)}; + + auto pads_begin = + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 0, pad_h, pad_w}); + auto pads_end = + ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 0, pad_h, pad_w}); + + auto pad = std::make_shared(image, pads_begin, pads_end, ov::op::PadMode::CONSTANT); + auto patches = + std::make_shared(pad, patch_sizes, strides, rates, ov::op::PadType::VALID); + + // [N, KH*KW*IC, OH, OW] → [N, OH, OW, KH*KW*IC] + auto perm1 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 2, 3, 1}); + auto t1 = std::make_shared(patches, perm1); + + // [N, OH, OW, KH*KW*IC] → [N, OH, OW, KH*KW, IC] + const ov::Shape out_shape = t1->get_output_shape(0); + const size_t N = out_shape[0]; + const size_t OH = out_shape[1]; + const size_t OW = out_shape[2]; + auto reshape1_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{5}, + std::vector{static_cast(N), static_cast(OH), static_cast(OW), + static_cast(KH * KW), static_cast(IC)}); + auto r1 = std::make_shared(t1, reshape1_shape, false); + + // [N, OH, OW, KH*KW, IC] → [N, OH, OW, IC, KH*KW] + auto perm2 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, std::vector{0, 1, 2, 4, 3}); + auto t2 = std::make_shared(r1, perm2); + + // flatten back to [N, OH, OW, IC*KH*KW] + auto r2_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{static_cast(N), static_cast(OH), static_cast(OW), + static_cast(IC * KH * KW)}); + res = std::make_shared(t2, r2_shape, false); + + if (!is_2D) { + // [N, 1, OW, IC * KW] -> [1, N, OW, IC * KW] + auto final_reshape_shape = ov::op::v0::Constant::create( + ov::element::i64, ov::Shape{4}, + std::vector{1, static_cast(N), static_cast(OW), static_cast(IC * KW)}); + res = std::make_shared(res, final_reshape_shape, false); + } + + auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp b/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp new file mode 100644 index 000000000..4b8ed3b6c --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp @@ -0,0 +1,44 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_l2_norm(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input_node = process_view_input_new(context, 0); + + auto squared = std::make_shared(input_node, input_node); + + auto sum_squared = std::make_shared( + squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + + auto l2_norm = std::make_shared(sum_squared); + + float eps; + memcpy(&eps, context.get_output_op_params(), sizeof(float)); + + auto eps_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}); + auto clamped_norm = std::make_shared(l2_norm, eps_const); + + auto res = std::make_shared(input_node, clamped_norm); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp new file mode 100644 index 000000000..09e29d4cc --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp @@ -0,0 +1,108 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_mul_mat_id(const NodeContext & context) { + num_inputs_check(context, 3, 3); + + auto expert_weights = process_view_input_new(context, 0); + auto activations = process_view_input_new(context, 1); + auto ids = process_view_input_new(context, 2); + + // OpenVINO sees GGML tensors in reversed dimension order: + // weights: [1, n_expert, m, k] + // activations: [1, n_tokens, n_used_or_1, k] + // ids: [1, 1, n_tokens, n_used] + // Rebuild the logical ranks explicitly from the 4D inputs instead of relying + // on fixed squeeze axes: real graphs can arrive through VIEW/RESHAPE chains + // where singleton axes are still represented differently at this point. + auto expert_weights_shape_4d = std::make_shared(expert_weights, ov::element::i64); + auto activations_shape_4d = std::make_shared(activations, ov::element::i64); + auto ids_shape_4d = std::make_shared(ids, ov::element::i64); + + auto expert_weights_shape_3d = get_dimensions(expert_weights_shape_4d, {1, 2, 3}); + auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); + auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); + + expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); + activations = std::make_shared(activations, activations_shape_3d, false); + ids = std::make_shared(ids, ids_shape_2d, false); + + if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { + ids = std::make_shared(ids, ov::element::i32); + } + + auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); + ov::Output selected_weights = std::make_shared(expert_weights, ids, gather_axis); + + const auto output_type = context.get_output_type(); + if (selected_weights.get_element_type() != ov::element::f32) { + selected_weights = std::make_shared(selected_weights, ov::element::f32); + } + if (activations.get_element_type() != ov::element::f32) { + activations = std::make_shared(activations, ov::element::f32); + } + + auto activations_shape = std::make_shared(activations, ov::element::i64); + auto ids_shape = std::make_shared(ids, ov::element::i64); + ov::Output acts_target_dims = std::make_shared( + ov::OutputVector{ + get_dimensions(activations_shape, {0}), + get_dimensions(ids_shape, {1}), + get_dimensions(activations_shape, {2}), + }, + 0); + ov::Output acts_broadcasted = + std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); + + auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); + auto activations_expanded = std::make_shared(acts_broadcasted, unsqueeze_axes); + + auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto output_shape = context.get_output_shape(); + FRONT_END_OP_CONVERSION_CHECK(output_shape.rank().is_static() && output_shape.rank().get_length() == 4, + "Unexpected MUL_MAT_ID output rank"); + FRONT_END_OP_CONVERSION_CHECK(output_shape[3].is_static(), "Expected static row dimension for MUL_MAT_ID output"); + const auto row_dim_value = output_shape[3].get_length(); + auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {row_dim_value}); + + ov::Output result = + std::make_shared(activations_expanded, selected_weights, false, true); + + auto result_target_dims = std::make_shared( + ov::OutputVector{ + batch_dim, + get_dimensions(ids_shape, {0, 1}), + row_dim, + }, + 0); + result = std::make_shared(result, result_target_dims, false); + + if (result.get_element_type() != output_type) { + result = std::make_shared(result, output_type); + } + + return rename_outputs_with_suffix({result}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp index 38edec85d..41d7c54ae 100644 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp @@ -30,17 +30,16 @@ OutputVector translate_mulmat(const NodeContext & context) { int op_case = context.get_op_case(); ov::Output res; - ov::Output B = context.get_input(0); - ov::Output A = context.get_input(1); - - bool transpose_b = true; - if (op_case == 2) { - B = B.get_node_shared_ptr()->input_value(0); - transpose_b = false; - } else if (op_case == 3) { + ov::Output B; + ov::Output A; + if (op_case == 3) { B = process_view_input(context, 0); A = process_view_input(context, 1); + } else { + B = process_view_input_new(context, 0); + A = process_view_input_new(context, 1); } + if (A.get_element_type() != B.get_element_type()) { B = std::make_shared(context.get_input(0), context.get_input_type(1)); } @@ -55,6 +54,7 @@ OutputVector translate_mulmat(const NodeContext & context) { auto batch_small = A_batch_larger ? B_batch : A_batch; Output Z = A_batch_larger ? B : A; + auto Z_shape = A_batch_larger ? B_shape : A_shape; int64_t factor = batch_large / batch_small; if (factor > 1 && batch_small > 1) { auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_large}); @@ -67,7 +67,11 @@ OutputVector translate_mulmat(const NodeContext & context) { auto broadcast_shape = ov::op::v0::Constant::create( ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) A_shape[3]}); + {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) Z_shape[3]}); + if (op_case == 2) { + new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 0, batch_large, (int64_t) Z_shape[2], (int64_t) -1}); + } auto Z_broadcasted = std::make_shared(Z_unsqueezed, broadcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); @@ -79,8 +83,14 @@ OutputVector translate_mulmat(const NodeContext & context) { A = Z; } + bool transpose_b = true; res = std::make_shared(A, B, false, transpose_b); + const auto output_type = context.get_output_type(); + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/norm.cpp b/ggml/src/ggml-openvino/openvino/op/norm.cpp new file mode 100644 index 000000000..c8bedb6db --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/norm.cpp @@ -0,0 +1,58 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_norm(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input_node = process_view_input_new(context, 0); + + // Step 1: Calculate mean along the last dimension + // mean = reduce_mean(input, axis=-1, keepdims=true) + auto mean = std::make_shared( + input_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + + // Step 2: Calculate (input - mean) + auto centered = std::make_shared(input_node, mean); + + // Step 3: Calculate squared differences (input - mean)^2 + auto squared = std::make_shared( + centered, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f})); + + // Step 4: Calculate variance = mean((input - mean)^2) + auto variance = std::make_shared( + squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + + // Step 5: Get epsilon from op_params + float eps; + memcpy(&eps, context.get_output_op_params(), sizeof(float)); + + // Step 6: Calculate std = sqrt(variance + eps) + auto std_dev = std::make_shared(std::make_shared( + variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}))); + + // Step 7: Normalize: output = (input - mean) / std + auto res = std::make_shared(centered, std_dev); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp new file mode 100644 index 000000000..492033d1b --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/pad.cpp @@ -0,0 +1,95 @@ +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +namespace { + +ov::Output translate_circular_pad(ov::Output input, + const std::array & pads, + const ov::Shape & input_shape) { + ov::Output result = input; + + const std::array pads_begin = {pads[6], pads[4], pads[2], pads[0]}; + const std::array pads_end = {pads[7], pads[5], pads[3], pads[1]}; + + for (size_t axis = 0; axis < input_shape.size(); ++axis) { + const int64_t input_dim = static_cast(input_shape[axis]); + const int64_t pad_begin = pads_begin[axis]; + const int64_t pad_end = pads_end[axis]; + + if (pad_begin == 0 && pad_end == 0) { + continue; + } + + FRONT_END_CHECK_IMPLEMENTED(input_dim > 0, "Circular PAD requires static non-zero input dimensions"); + + std::vector indices(static_cast(input_dim + pad_begin + pad_end)); + for (int64_t index = 0; index < static_cast(indices.size()); ++index) { + int64_t wrapped = (index - pad_begin) % input_dim; + if (wrapped < 0) { + wrapped += input_dim; + } + indices[static_cast(index)] = wrapped; + } + + auto gather_indices = ov::op::v0::Constant::create(ov::element::i64, {indices.size()}, indices); + auto gather_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {axis}); + result = std::make_shared(result, gather_indices, gather_axis); + } + + return result; +} + +} // namespace + +OutputVector translate_pad(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + if (context.get_input_shape(0) == context.get_output_shape()) { + auto input_shape = std::make_shared(input); + auto res = std::make_shared(input, input_shape, false); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + const int32_t * op_params = context.get_output_op_params(); + FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "PAD requires output op params"); + + const std::array pads = {op_params[0], op_params[1], op_params[2], op_params[3], + op_params[4], op_params[5], op_params[6], op_params[7]}; + const bool circular = op_params[8] != 0; + + if (circular) { + auto res = translate_circular_pad(input, pads, context.get_input_shape(0).to_shape()); + return rename_outputs_with_suffix({res}, context.get_name()); + } + + const std::vector pads_begin = {pads[6], pads[4], pads[2], pads[0]}; + const std::vector pads_end = {pads[7], pads[5], pads[3], pads[1]}; + + auto pads_begin_node = ov::op::v0::Constant::create(ov::element::i64, {pads_begin.size()}, pads_begin); + auto pads_end_node = ov::op::v0::Constant::create(ov::element::i64, {pads_end.size()}, pads_end); + auto pad_value = ov::op::v0::Constant::create(context.get_input_type(0), ov::Shape{}, {0}); + auto res = + std::make_shared(input, pads_begin_node, pads_end_node, pad_value, ov::op::PadMode::CONSTANT); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 4c800f9ee..85550bff3 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace ov { namespace frontend { @@ -22,16 +23,33 @@ OutputVector translate_permute(const NodeContext & context) { num_inputs_check(context, 1, 1); int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4, - "Unsupported PERMUTE case"); + FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case"); + // op_case 1 is trivial permute + // op_case 2 is to permute Q. It has a preceding VIEW that reshapes Q to restore the sequqence dimension + // op_case 3 4 it to permute KV cache in the default layout + // op_case 5 6 is to permute V cache when `-fa off`, where v_trans=true ov::Output res; - auto src = context.get_input(0); - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}); + ov::Output src; + if (op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6) { + src = context.get_input(0); + } else { + src = process_view_input_new(context, 0); + } + std::vector perm_values{0, 2, 1, 3}; + const int32_t * op_params = context.get_output_op_params(); + if (op_params != nullptr) { + for (size_t input_axis = 0; input_axis < perm_values.size(); ++input_axis) { + const size_t output_axis = static_cast(op_params[input_axis]); + perm_values[perm_values.size() - 1 - output_axis] = + static_cast(perm_values.size() - 1 - input_axis); + } + } + auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); if (op_case == 1 || context.is_stateful()) { res = std::make_shared(src, perm); - } else if (op_case == 4) { + } else if (op_case == 2) { auto output_shape = context.get_output_shape().to_shape(); auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); @@ -54,13 +72,17 @@ OutputVector translate_permute(const NodeContext & context) { auto output_shape = context.get_output_shape().to_shape(); int64_t head_size = output_shape[3]; int64_t n_heads = output_shape[1]; + if (op_case == 5 || op_case == 6) { + head_size = output_shape[2]; + n_heads = output_shape[1]; + } int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1; int64_t n_seq = cache_shape[1].get_length(); Output attention_size; if (!context.has_input("attention_size")) { attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]}); - } else if (op_case == 2) { + } else if (op_case == 3 || op_case == 5) { attention_size = context.get_input("attention_size"); } else { attention_size = context.get_input("attention_size_swa"); @@ -80,18 +102,41 @@ OutputVector translate_permute(const NodeContext & context) { seq_active_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_end_val}); } - // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] + // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] (for `-fa off` [n_seq, n_heads, head_size, ctx_per_seq]) // 2. slice out the active sequences // 3. slice out the attention part in each sequence - // 4. permute + // 4. permute (skip for `-fa off`) auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto src_reshaped = std::make_shared( - src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), false); - auto slice1 = std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); - auto slice2 = std::make_shared(slice1, zero, attention_size, one, one); - res = std::make_shared(slice2, perm); + if (op_case == 3 || op_case == 4) { + auto src_reshaped = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), + false); + ov::Output after_seq_slice; + if (n_seq == 1) { + after_seq_slice = src_reshaped; + } else { + after_seq_slice = + std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); + } + auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, one); + res = std::make_shared(slice2, perm); + } else { + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto src_reshaped = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}), + false); + ov::Output after_seq_slice; + if (n_seq == 1) { + after_seq_slice = src_reshaped; + } else { + after_seq_slice = + std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); + } + auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, three); + res = slice2; + } } return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/repeat.cpp b/ggml/src/ggml-openvino/openvino/op/repeat.cpp new file mode 100644 index 000000000..4b742134b --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/repeat.cpp @@ -0,0 +1,74 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +// GGML_OP_REPEAT tiles src[0] to fill the destination shape. Every destination +// dimension is an integer multiple of the corresponding source dimension. +OutputVector translate_repeat(const NodeContext & context) { + num_inputs_check(context, 1, 2); + + auto input = process_view_input_new(context, 0); + + const auto input_shape = context.get_input_shape(0); + const auto output_shape = context.get_output_shape(); + + if (input_shape.rank().is_static() && output_shape.rank().is_static() && + input_shape.rank() == output_shape.rank()) { + const auto rank = static_cast(input_shape.rank().get_length()); + std::vector repeats(rank, 1); + bool all_static = true; + + for (size_t axis = 0; axis < rank; ++axis) { + if (!input_shape[axis].is_static() || !output_shape[axis].is_static()) { + all_static = false; + break; + } + + const int64_t input_dim = input_shape[axis].get_length(); + const int64_t output_dim = output_shape[axis].get_length(); + + FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && output_dim > 0 && output_dim % input_dim == 0, + "REPEAT input shape ", input_shape, " cannot tile to match ", output_shape); + + repeats[axis] = output_dim / input_dim; + } + + if (all_static) { + auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats); + ov::Output res = std::make_shared(input, repeats_node); + return rename_outputs_with_suffix({res}, context.get_name()); + } + } + + // Dynamic fallback: tile by the ratio of output to input shape. + auto input_shape_node = std::make_shared(input, ov::element::i64); + std::shared_ptr target_shape_node; + if (output_shape.rank().is_static() && output_shape.is_static()) { + target_shape_node = + ov::op::v0::Constant::create(ov::element::i64, {output_shape.to_shape().size()}, output_shape.to_shape()); + } else { + target_shape_node = std::make_shared(context.get_input(1), ov::element::i64); + } + auto repeats_node = std::make_shared(target_shape_node, input_shape_node); + ov::Output res = std::make_shared(input, repeats_node); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/reshape.cpp b/ggml/src/ggml-openvino/openvino/op/reshape.cpp index efd9a5a86..602d3387c 100644 --- a/ggml/src/ggml-openvino/openvino/op/reshape.cpp +++ b/ggml/src/ggml-openvino/openvino/op/reshape.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include namespace ov { @@ -20,7 +19,8 @@ namespace op { OutputVector translate_reshape(const NodeContext & context) { num_inputs_check(context, 1, 1); - if (context.get_input_shape(0) == context.get_output_shape()) { + if (context.get_input(0).get_partial_shape().is_static() && + context.get_input_shape(0) == context.get_output_shape()) { return {context.get_input(0)}; } @@ -34,12 +34,12 @@ OutputVector translate_reshape(const NodeContext & context) { if (op_case == 1) { if (context.is_stateful()) { new_shape_node = ov::op::v0::Constant::create( - ov::element::i64, {3}, - std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + ov::element::i64, {3}, std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); } else { new_shape_node = ov::op::v0::Constant::create( ov::element::i64, {4}, - std::vector{(int64_t) output_shape[0], -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + std::vector{(int64_t) output_shape[0], -1, (int64_t) output_shape[2], + (int64_t) output_shape[3]}); } } else if (op_case == 2) { new_shape_node = ov::op::v0::Constant::create( @@ -47,7 +47,14 @@ OutputVector translate_reshape(const NodeContext & context) { std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, (int64_t) output_shape[3]}); } else if (op_case == 3) { - throw std::runtime_error("might be outdated RESHAPE case"); + // - 14: [ 1, 1024, 1, 1] RESHAPE Vcur-0 (reshaped) (reshaped) + // [ 512, 2, 1, 1] 0: RESHAPE Vcur-0 (reshaped) + // - 15: [ 1, 524288, 1, 1] RESHAPE cache_v_l0 (reshaped) + // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 + // - 16: [ 1, 524288, 1, 1] SET_ROWS cache_v_l0 (reshaped) (view) + // [ 1, 1024, 1, 1] 0: RESHAPE Vcur-0 (reshaped) (reshaped) + // [ 1024, 1, 1, 1] 1: NONE leaf_11 + // [ 1, 524288, 1, 1] 2: RESHAPE cache_v_l0 (reshaped) new_shape_node = ov::op::v0::Constant::create( ov::element::i64, {4}, std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, 1}); diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp index 72cf92283..e76ec55b8 100644 --- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp @@ -19,7 +19,7 @@ namespace op { OutputVector translate_rms_norm(const NodeContext & context) { num_inputs_check(context, 1, 1); - auto input_node = context.get_input(0); + auto input_node = process_view_input_new(context, 0); auto square = std::make_shared( input_node, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {2.0f})); diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp index a8db9b389..9bb2d75d0 100644 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rope.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -38,8 +39,7 @@ OutputVector translate_rope(const NodeContext & context) { auto data_node = context.get_input(0).get_node_shared_ptr(); auto output_shape = context.get_output_shape().to_shape(); int32_t * op_params = context.get_output_op_params(); - const int mode = (op_case & 0xFFFF0000) >> 16; - op_case = (op_case & 0x0000FFFF); + const int mode = op_case; constexpr int TYPE_NORMAL = 0; constexpr int TYPE_NEOX = 1; @@ -56,55 +56,146 @@ OutputVector translate_rope(const NodeContext & context) { if (context.get_input_size() == 3) { rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); } - auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE); + auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode == TYPE_IMROPE, false); sin_theta_node = sin_cos.first; cos_theta_node = sin_cos.second; } - if (op_case == 2) { - // The input comes from a VIEW - int slice_len = output_shape[2] * output_shape[3]; - data_node = process_view_input(context, 0, slice_len).get_node_shared_ptr(); + if (context.get_view_input_size(0) > 0) { + data_node = process_view_input_new(context, 0).get_node_shared_ptr(); if (context.is_stateful()) { auto data_shape = ov::op::v0::Constant::create( ov::element::i64, {3}, std::vector{-1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); data_node = std::make_shared(data_node, data_shape, false); } else { auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + ov::element::i64, {4}, + std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); data_node = std::make_shared(data_node, data_shape, false); } } + auto output_type = context.get_output_type(); + if (data_node->get_element_type() != ov::element::f32) { + data_node = std::make_shared(data_node, ov::element::f32); + } + + // TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the + // OpenVINO GPU plugin is updated. + // + // For TYPE_NORMAL rope (both stateful and stateless) we emit the Flux-style + // interleaved pattern below so the GPU plugin's RoPEFusionFlux matcher folds it + // into ov::op::internal::RoPE. The matcher requires rank-4 inputs, which is why + // the original even/odd Slice translation (kept in the `else if (mode == + // TYPE_NORMAL)` branch below for reference) does not get fused. + // + // Once the GPU plugin's RoPE fusion is extended to also recognize the original + // even/odd Slice form, this Flux rewrite should be removed and both modes should + // be restored to the captured even/odd translation. Until then, keep both paths: + // the active Flux rewrite here and the previous translation preserved below. if (mode == TYPE_NORMAL) { - auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); - Output even_slice; - Output odd_slice; - int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4; - even_slice = std::make_shared(data_node, zero, end, two, neg_one); - odd_slice = std::make_shared(data_node, one, end, two, neg_one); + // Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's + // RoPEFusionFlux matcher folds this subgraph into ov::op::internal::RoPE: + // x_paired = Reshape(x, [1, S, n_heads, head_size/2, 2]) + // x0, x1 = Split(x_paired, axis=-1, num_splits=2) + // x1_neg = x1 * -1 + // x_rotated = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, head_size]) + // y = x * t_cos + x_rotated * t_sin + // Mathematically equivalent to the even/odd Slice form below. + // + // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin + // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In + // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it + // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced + // rank-4 output, so downstream attention is unaffected. + if (context.is_stateful()) { + auto r4_shape = ov::op::v0::Constant::create( + ov::element::i64, {4}, + std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + data_node = std::make_shared(data_node, r4_shape, false); + } + const int64_t head_size = static_cast(output_shape[3]); + const int64_t n_heads = static_cast(output_shape[2]); + const int64_t half = head_size / 2; - Output first_half = - std::make_shared(std::make_shared(even_slice, cos_theta_node), - std::make_shared(odd_slice, sin_theta_node)); - Output second_half = - std::make_shared(std::make_shared(even_slice, sin_theta_node), - std::make_shared(odd_slice, cos_theta_node)); + auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); - first_half = std::make_shared(first_half, - ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - second_half = std::make_shared(second_half, - ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); - auto stack = std::make_shared(OutputVector{first_half, second_half}, unsqueeze_dim); + auto paired_shape = + ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{1, -1, n_heads, half, 2}); + auto x_paired = std::make_shared(data_node, paired_shape, false); - auto data_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - res = std::make_shared(stack, data_shape, false); - } else if (mode == TYPE_NEOX) { + auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}); + auto data_split = std::make_shared(x_paired, split_axis, 2); + Output x0 = data_split->outputs()[0]; + Output x1 = data_split->outputs()[1]; + + auto x1_neg = std::make_shared(x1, neg_one_f); + auto x_rotated_paired = std::make_shared(ov::OutputVector{x1_neg, x0}, -1); + + auto flat_shape = + ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, -1, n_heads, head_size}); + auto x_rotated = std::make_shared(x_rotated_paired, flat_shape, false); + + // Expand cos/sin from [..., head_size/2] to [..., head_size] by repeating each + // entry twice. Use special_zero on the final Reshape so the seq dim passes + // through dynamically. Final rank is 4 to satisfy the matcher's predicate. + auto expand_cos_sin = [&](Output cs) { + auto cs_unsq = + std::make_shared(cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})); + auto bcast_target = + ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{1, 1, 1, half, 2}); + auto bcast = + std::make_shared(cs_unsq, bcast_target, ov::op::BroadcastType::BIDIRECTIONAL); + auto flat = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 0, 0, head_size}); + return std::make_shared(bcast, flat, true); + }; + Output cos_full = expand_cos_sin(cos_theta_node); + Output sin_full = expand_cos_sin(sin_theta_node); + + auto y1 = std::make_shared(data_node, cos_full); + auto y2 = std::make_shared(x_rotated, sin_full); + res = std::make_shared(y1, y2); + } + // PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once + // the GPU plugin's RoPE fusion is updated to recognize the even/odd Slice form; + // see the TODO(openvino-gpu-rope-fusion) note above. Do not delete. + // + // Original even/odd Slice form. In stateless mode it ran on rank-4 data + // ([1, S, n_heads, head_size]); in stateful mode on rank-3 data + // ([S, n_heads, head_size]). Either way it does not match RoPEFusionFlux + // (which needs rank-4 x in the interleaved layout), so the RoPE stays as + // discrete elementwise ops. + // + // } else if (mode == TYPE_NORMAL) { + // auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + // auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + // auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + // auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); + // auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); + // Output even_slice; + // Output odd_slice; + // // stateful data is rank 3 (unsqueeze at axis 3), stateless is rank 4 (axis 4) + // int32_t unsqueeze_dim = context.is_stateful() ? 3 : 4; + // even_slice = std::make_shared(data_node, zero, end, two, neg_one); + // odd_slice = std::make_shared(data_node, one, end, two, neg_one); + // + // Output first_half = + // std::make_shared(std::make_shared(even_slice, cos_theta_node), + // std::make_shared(odd_slice, sin_theta_node)); + // Output second_half = + // std::make_shared(std::make_shared(even_slice, sin_theta_node), + // std::make_shared(odd_slice, cos_theta_node)); + // + // first_half = std::make_shared(first_half, + // ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); + // second_half = std::make_shared(second_half, + // ov::op::v0::Constant::create(ov::element::i64, {1}, {unsqueeze_dim})); + // auto stack = std::make_shared(OutputVector{first_half, second_half}, unsqueeze_dim); + // + // auto data_shape = ov::op::v0::Constant::create( + // ov::element::i64, {4}, std::vector{1, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); + // res = std::make_shared(stack, data_shape, false); + else if (mode == TYPE_NEOX) { auto data_split = std::make_shared( data_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}), 2); Output slice_data_node_0 = data_split->outputs()[0]; @@ -120,8 +211,9 @@ OutputVector translate_rope(const NodeContext & context) { res = std::make_shared(ov::OutputVector{first_half_node, second_half_node}, -1); } else if (mode == TYPE_IMROPE) { - int64_t n_dims = data_node->get_shape()[3]; - auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, std::vector{1,-1,1,(n_dims >> 1)}); + int64_t n_dims = data_node->get_output_partial_shape(0)[3].get_length(); + auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, + std::vector{1, -1, 1, (n_dims >> 1)}); auto cos_reshaped = std::make_shared(cos_theta_node, cos_sin_shape, true); auto sin_reshaped = std::make_shared(sin_theta_node, cos_sin_shape, true); @@ -140,6 +232,10 @@ OutputVector translate_rope(const NodeContext & context) { res = std::make_shared(ov::OutputVector{sub, add}, 3); } + if (res.get_element_type() != output_type) { + res = std::make_shared(res, output_type); + } + return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 136e4265b..18643371e 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -28,20 +28,20 @@ namespace op { OutputVector translate_set_rows(const NodeContext & context) { num_inputs_check(context, 3, 3); - auto data = context.get_input(0); + auto data = process_view_input_new(context, 0); auto indices = context.get_input(1); auto dst = context.get_input(2); data = std::make_shared(data, context.get_output_type()); - auto dst_shape = context.get_output_shape().to_shape(); + auto row_size = context.get_input_shape(2)[3].get_length(); auto ind_squeezed = std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); auto data_reshaped = std::make_shared( data, ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) dst_shape[3]}), + {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), false); auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index 9f6330862..287faedbb 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -2,18 +2,16 @@ #include "../op_table.h" #include "../utils.h" -#include +#include #include +#include #include -#include -#include +#include #include -#include #include #include -#include #include -#include +#include #include #include @@ -22,63 +20,82 @@ namespace frontend { namespace ggml { namespace op { +// Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend: +// 1) logits = src0 * scale +// 2) logits += mask (if provided) +// 3) softmax over the last dimension OutputVector translate_soft_max(const NodeContext & context) { - // TODO code is outdated num_inputs_check(context, 1, 2); - auto input_node = context.get_input(0).get_node_shared_ptr(); - ov::Output res; - float scale = 1.0f; float max_bias = 0.0f; - auto * op_params = context.get_output_op_params(); - memcpy(&scale, (float *) op_params + 0, sizeof(float)); - memcpy(&max_bias, (float *) op_params + 1, sizeof(float)); - auto src0_shape = context.get_input_shape(0).get_shape(); - const uint32_t h = src0_shape[2]; - const uint32_t n_head = src0_shape[0]; - const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head)); + memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float)); + memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float)); - const float m0 = powf(2.0f, -(max_bias) / n_head_log2); - const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2); - const float slope = - (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f; + ov::Output logits = context.get_input(0); - auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); - auto scaled_input = std::make_shared(input_node, scale_node); - - if (context.get_input_size() < 2) { - res = std::make_shared(scaled_input, 2); - return rename_outputs_with_suffix({res}, context.get_name()); + // Apply scale first: logits = src0 * scale + if (scale != 1.0f) { + auto scale_const = + std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); + logits = std::make_shared(logits, scale_const); } - ov::Output mask_node_sliced; - if (context.has_input("KQ_mask_sliced")) { - mask_node_sliced = context.get_input("KQ_mask_sliced"); - } else { - auto token_len = get_dimensions(input_node, {1}); - auto mask_node = context.get_input(1); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - mask_node_sliced = std::make_shared(mask_node, zero, token_len, one, one); + FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && context.get_input_size() < 2), + "OpenVINO softmax ALiBi path requires mask input"); + + // Optional mask add: logits += mask + // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. + if (context.get_input_size() > 1) { + ov::Output mask = context.get_input(1); + + // For stateful + std::string mask_name = "KQ_mask_sliced"; + if (context.get_input_names()[1].find("swa") != std::string::npos) { + mask_name = "KQ_mask_swa_sliced"; + } + if (context.has_input(mask_name)) { + mask = context.get_input(mask_name); + } + + if (mask.get_element_type() != logits.get_element_type()) { + mask = std::make_shared(mask, logits.get_element_type()); + } + + if (max_bias > 0.0f) { + auto out_shape = context.get_output_shape().to_shape(); + FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, "OpenVINO softmax ALiBi path expects rank-4 tensor"); + + const uint32_t n_head = static_cast(out_shape[1]); + FRONT_END_CHECK_IMPLEMENTED(n_head > 0, "OpenVINO softmax ALiBi path expects n_head > 0"); + + const uint32_t n_head_log2 = 1u << static_cast(std::floor(std::log2(static_cast(n_head)))); + const float m0 = std::pow(2.0f, -(max_bias) / static_cast(n_head_log2)); + const float m1 = std::pow(2.0f, -(max_bias / 2.0f) / static_cast(n_head_log2)); + + std::vector slopes(n_head); + for (uint32_t h = 0; h < n_head; ++h) { + slopes[h] = h < n_head_log2 ? std::pow(m0, static_cast(h + 1)) : + std::pow(m1, static_cast(2 * (h - n_head_log2) + 1)); + } + + ov::Output slope_node = + std::make_shared(ov::element::f32, ov::Shape{n_head}, slopes); + if (slope_node.get_element_type() != mask.get_element_type()) { + slope_node = std::make_shared(slope_node, mask.get_element_type()); + } + + auto slope_shape = std::make_shared( + ov::element::i64, ov::Shape{4}, std::vector{1, static_cast(n_head), 1, 1}); + auto slope_4d = std::make_shared(slope_node, slope_shape, false); + mask = std::make_shared(mask, slope_4d); + } + + logits = std::make_shared(logits, mask); } - if (mask_node_sliced.get_element_type() != context.get_output_type()) { - mask_node_sliced = std::make_shared(mask_node_sliced, context.get_output_type()); - } - - Output slope_mask; - if (slope != 1.0f) { - auto slope_node = - std::make_shared(ov::element::f32, ov::Shape{}, std::vector{slope}); - slope_mask = std::make_shared(mask_node_sliced, slope_node); - throw std::runtime_error("Slope != 1.0f in softmax has not been tested, verify it before use."); - } - slope_mask = mask_node_sliced; - - auto input_slope_mask_node = std::make_shared(scaled_input, slope_mask); - - res = std::make_shared(input_slope_mask_node, 2); + // Softmax along last dimension (equivalent to ggml softmax over ne[0]). + auto res = std::make_shared(logits, -1); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp b/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp new file mode 100644 index 000000000..522308726 --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp @@ -0,0 +1,59 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_ssm_conv(const NodeContext & context) { + num_inputs_check(context, 2, 2); + + auto sx = context.get_input(0); // conv state + input: OV shape [1, n_s, d_inner, ncs] + auto c = context.get_input(1); // conv1d weight: OV shape [1, 1, d_inner, d_conv] + + auto sx_shape = context.get_input_shape(0).to_shape(); // [1, n_s, d_inner, ncs] + auto c_shape = context.get_input_shape(1).to_shape(); // [1, 1, d_inner, d_conv] + + int64_t n_s = sx_shape[1]; + int64_t d_inner = sx_shape[2]; + int64_t ncs = sx_shape[3]; // d_conv - 1 + n_t + int64_t d_conv = c_shape[3]; + int64_t n_t = ncs - d_conv + 1; + + // Reshape sx from [1, n_s, d_inner, ncs] to [n_s, d_inner, ncs] for 1D GroupConvolution + auto sx_new_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{n_s, d_inner, ncs}); + auto sx_reshaped = std::make_shared(sx, sx_new_shape, false); + + // Reshape c from [1, 1, d_inner, d_conv] to [d_inner, 1, 1, d_conv] + // GroupConvolution filter: [groups, out_channels/groups, in_channels/groups, kernel_size] + auto c_new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{d_inner, 1, 1, d_conv}); + auto c_reshaped = std::make_shared(c, c_new_shape, false); + + // Depthwise 1D convolution: groups=d_inner, stride=1, no padding, no dilation + // Input: [n_s, d_inner, ncs], Filter: [d_inner, 1, 1, d_conv] + // Output: [n_s, d_inner, n_t] + auto conv = std::make_shared( + sx_reshaped, c_reshaped, ov::Strides{1}, ov::CoordinateDiff{0}, ov::CoordinateDiff{0}, ov::Strides{1}); + + // Transpose from [n_s, d_inner, n_t] to [n_s, n_t, d_inner] + auto perm = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{0, 2, 1}); + auto transposed = std::make_shared(conv, perm); + + // Reshape to output shape [1, n_s, n_t, d_inner] + auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, n_s, n_t, d_inner}); + auto res = std::make_shared(transposed, out_shape, false); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp b/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp new file mode 100644 index 000000000..d04e6443b --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp @@ -0,0 +1,27 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_sum_rows(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared( + input, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/transpose.cpp b/ggml/src/ggml-openvino/openvino/op/transpose.cpp index 8e62e83c0..8d89ca556 100644 --- a/ggml/src/ggml-openvino/openvino/op/transpose.cpp +++ b/ggml/src/ggml-openvino/openvino/op/transpose.cpp @@ -12,8 +12,39 @@ namespace op { OutputVector translate_transpose(const NodeContext & context) { num_inputs_check(context, 1, 1); + // Compute permute order from input/output shape and stride information + // so it adapts to different input and output layouts. + auto input_shape = context.get_input_shape(0).to_shape(); + auto input_stride = context.get_input_stride(0); + auto output_shape = context.get_output_shape().to_shape(); + auto output_stride = context.get_output_stride(); + + // Compute permute order by matching output and input stride rankings. + // Build pairs. + std::vector> output_stride_dims; + std::vector> input_stride_dims; + + for (int i = 0; i < 4; ++i) { + output_stride_dims.push_back({output_stride[i], i}); + input_stride_dims.push_back({input_stride[i], i}); + } + + // Sort by stride in descending order. + std::sort(output_stride_dims.rbegin(), output_stride_dims.rend()); + std::sort(input_stride_dims.rbegin(), input_stride_dims.rend()); + + // Build permute order. + std::vector permute_order(4); + for (int i = 0; i < 4; ++i) { + int output_dim = output_stride_dims[i].second; + int input_dim = input_stride_dims[i].second; + permute_order[output_dim] = input_dim; + } + + auto input = process_view_input_new(context, 0); + auto res = std::make_shared( - context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 1, 3, 2})); + input, ov::op::v0::Constant::create(ov::element::i64, {4}, permute_order)); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/ggml/src/ggml-openvino/openvino/op/unary_gelu.cpp b/ggml/src/ggml-openvino/openvino/op/unary_gelu.cpp deleted file mode 100644 index d1e9efc33..000000000 --- a/ggml/src/ggml-openvino/openvino/op/unary_gelu.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_unary_gelu(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = context.get_input(0); - auto res = std::make_shared(input); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp index 037e0b94d..48ee0431f 100644 --- a/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp +++ b/ggml/src/ggml-openvino/openvino/op/unary_silu.cpp @@ -14,7 +14,7 @@ namespace op { OutputVector translate_unary_silu(const NodeContext & context) { num_inputs_check(context, 1, 1); - auto input = context.get_input(0); + auto input = process_view_input_new(context, 0); auto sigmoid = std::make_shared(input); auto res = std::make_shared(input, sigmoid); diff --git a/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp b/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp new file mode 100644 index 000000000..756d9c33d --- /dev/null +++ b/ggml/src/ggml-openvino/openvino/op/unary_softplus.cpp @@ -0,0 +1,38 @@ +#include "../node_context.h" +#include "../op_table.h" +#include "../utils.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ov { +namespace frontend { +namespace ggml { +namespace op { + +OutputVector translate_unary_softplus(const NodeContext & context) { + num_inputs_check(context, 1, 1); + + auto input = process_view_input_new(context, 0); + const auto element_type = input.get_element_type(); + auto one = ov::op::v0::Constant::create(element_type, ov::Shape{}, {1.0f}); + + auto positive = std::make_shared(input); + auto abs = std::make_shared(input); + auto neg_abs = std::make_shared(abs); + auto exp_neg_abs = std::make_shared(neg_abs); + auto log_term = std::make_shared(std::make_shared(one, exp_neg_abs)); + auto res = std::make_shared(positive, log_term); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace ggml +} // namespace frontend +} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp index 8528d2523..28004dcd2 100644 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ b/ggml/src/ggml-openvino/openvino/op/view.cpp @@ -1,6 +1,11 @@ #include "../op_table.h" #include "../utils.h" + +#include #include +#include +#include + namespace ov { namespace frontend { namespace ggml { @@ -9,42 +14,102 @@ namespace op { OutputVector translate_view(const NodeContext & context) { num_inputs_check(context, 1, 1); - if (context.get_op_case() == 2) { - auto dst_shape = context.get_output_shape().to_shape(); - return rename_outputs_with_suffix({process_view_input(context, 0, dst_shape[2] * dst_shape[3])}, - context.get_name()); + if (!context.is_static()) { + return {context.get_input(0)}; } - // op_case 3 - if (context.get_op_case() == 3) { - auto input = context.get_input(0); - auto input_ov_shape = input.get_partial_shape(); - auto input_llama_shape = context.get_input_shape(0).to_shape(); + auto input = context.get_input(0); + auto src_shape = context.get_input_shape(0); + auto dst_shape = context.get_output_shape(); - // if the input ov shape size is different from the input llama shape size, it means the input is already reshaped and we need to reshape it back to the original shape before slicing - if (input_ov_shape.size() != input_llama_shape.size()) { - input = std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {input_llama_shape.size()}, input_llama_shape), false); + if (src_shape.rank().is_dynamic() || dst_shape.rank().is_dynamic()) { + return {input}; + } + + int64_t src_elems = 1, dst_elems = 1; + for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) { + if (src_shape[i].is_dynamic()) { + return {input}; } + src_elems *= src_shape[i].get_length(); + } + for (int64_t i = 0; i < dst_shape.rank().get_length(); ++i) { + if (dst_shape[i].is_dynamic()) { + return {input}; + } + dst_elems *= dst_shape[i].get_length(); + } - auto dst_shape = context.get_output_shape().to_shape(); + if (dst_elems >= src_elems) { + return {input}; + } - // find the index of dst_shape that is different from input shape, and use that index to slice the input - int slice_dim = -1; - for (size_t i = 0; i < dst_shape.size(); ++i) { - if (dst_shape[i] != input_llama_shape[i]) { - slice_dim = i; + auto src_stride = context.get_input_stride(0); + auto dst_stride = context.get_output_stride(); + size_t view_offset = context.get_output_op_offset(); + + bool same_stride = (src_stride.size() == dst_stride.size()); + if (same_stride) { + for (size_t i = 0; i < src_stride.size(); ++i) { + if (src_stride[i] != dst_stride[i]) { + same_stride = false; break; } } - - auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {dst_shape[slice_dim]}); - auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim}); - auto sliced = std::make_shared(input, begin, end, stride, axes); - return {sliced}; } - return {context.get_input(0)}; + + if (!same_stride) { + return {input}; + } + + auto src_ov_shape = src_shape.to_shape(); + auto dst_ov_shape = dst_shape.to_shape(); + size_t ndims = src_ov_shape.size(); + if (dst_ov_shape.size() != ndims) { + return {input}; + } + + std::vector diff_dims; + for (size_t i = 0; i < ndims; ++i) { + if (src_ov_shape[i] != dst_ov_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + + if (diff_dims.size() != 1) { + return {input}; + } + + int slice_dim = diff_dims[0]; + int64_t dim_size = static_cast(src_ov_shape[slice_dim]); + + size_t ov_stride_for_dim = 1; + for (size_t i = slice_dim + 1; i < ndims; ++i) { + ov_stride_for_dim *= src_ov_shape[i]; + } + size_t elem_size = src_stride.back(); + if (elem_size == 0) { + elem_size = 1; + } + + int64_t begin_val = 0; + if (ov_stride_for_dim > 0 && elem_size > 0) { + begin_val = static_cast((view_offset / elem_size) / ov_stride_for_dim); + } + int64_t end_val = begin_val + static_cast(dst_ov_shape[slice_dim]); + + if (begin_val < 0 || end_val > dim_size) { + return {input}; + } + + auto sliced = + std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); + + sliced->set_friendly_name(context.get_output_name()); + return {sliced->output(0)}; } } // namespace op diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp index 138553927..f84a1bf93 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ b/ggml/src/ggml-openvino/openvino/op_table.cpp @@ -5,9 +5,11 @@ #include #include #include +#include #include #include #include +#include namespace ov { namespace frontend { @@ -16,29 +18,44 @@ namespace ggml { std::unordered_map get_supported_ops() { using namespace ov::op; return { - {"GGML_OP_ADD", op::translate_1to1_match_2_inputs }, - {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, - {"GGML_OP_CONT", op::translate_cont }, - {"GGML_OP_DIV", op::translate_1to1_match_2_inputs }, - {"GGML_OP_GET_ROWS", op::translate_get_rows }, - {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, - {"GGML_OP_MUL_MAT", op::translate_mulmat }, - {"GGML_OP_PERMUTE", op::translate_permute }, - {"GGML_OP_RESHAPE", op::translate_reshape }, - {"GGML_OP_RMS_NORM", op::translate_rms_norm }, - {"GGML_OP_ROPE", op::translate_rope }, - {"GGML_OP_SCALE", op::translate_scale }, - {"GGML_OP_SOFT_MAX", op::translate_soft_max }, - {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, - {"GGML_OP_TRANSPOSE", op::translate_transpose }, - {"GGML_UNARY_OP_GELU", op::translate_unary_gelu }, - {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, - {"GGML_OP_VIEW", op::translate_view }, - {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, - {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, - {"GGML_OP_SET_ROWS", op::translate_set_rows }, - {"GGML_OP_CPY", op::translate_cpy }, - {"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext }, + {"GGML_OP_ADD", op::translate_1to1_match_2_inputs }, + {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, + {"GGML_OP_ADD_ID", op::translate_add_id }, + {"GGML_OP_CONCAT", op::translate_concat }, + {"GGML_OP_CONT", op::translate_cont }, + {"GGML_OP_DIV", op::translate_div }, + {"GGML_OP_GET_ROWS", op::translate_get_rows }, + {"GGML_OP_IM2COL", op::translate_im2col }, + {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, + {"GGML_OP_MUL_MAT", op::translate_mulmat }, + {"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id }, + {"GGML_OP_PERMUTE", op::translate_permute }, + {"GGML_OP_RESHAPE", op::translate_reshape }, + {"GGML_OP_RMS_NORM", op::translate_rms_norm }, + {"GGML_OP_NORM", op::translate_norm }, + {"GGML_OP_L2_NORM", op::translate_l2_norm }, + {"GGML_OP_SUM_ROWS", op::translate_sum_rows }, + {"GGML_OP_ROPE", op::translate_rope }, + {"GGML_OP_SCALE", op::translate_scale }, + {"GGML_OP_SOFT_MAX", op::translate_soft_max }, + {"GGML_OP_ARGSORT", op::translate_argsort }, + {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, + {"GGML_OP_TRANSPOSE", op::translate_transpose }, + {"GGML_UNARY_OP_GELU", op::translate_1to1_match_1_input }, + {"GGML_UNARY_OP_SILU", op::translate_unary_silu }, + {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, + {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, + {"GGML_OP_VIEW", op::translate_view }, + {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, + {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, + {"GGML_OP_SET_ROWS", op::translate_set_rows }, + {"GGML_OP_CPY", op::translate_cpy }, + {"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext }, + {"GGML_OP_CLAMP", op::translate_clamp }, + {"GGML_OP_PAD", op::translate_pad }, + {"GGML_OP_SSM_CONV", op::translate_ssm_conv }, + {"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net }, + {"GGML_OP_REPEAT", op::translate_repeat }, }; } diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h index f546796d2..c90ff8377 100644 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ b/ggml/src/ggml-openvino/openvino/op_table.h @@ -8,20 +8,26 @@ namespace ggml { namespace op { -#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext& context) +#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext & context) -GGML_OP_CONVERTER(translate_add); GGML_OP_CONVERTER(translate_cont); +GGML_OP_CONVERTER(translate_concat); +GGML_OP_CONVERTER(translate_add_id); +GGML_OP_CONVERTER(translate_div); GGML_OP_CONVERTER(translate_get_rows); -GGML_OP_CONVERTER(translate_mul); +GGML_OP_CONVERTER(translate_im2col); GGML_OP_CONVERTER(translate_mulmat); +GGML_OP_CONVERTER(translate_mul_mat_id); GGML_OP_CONVERTER(translate_permute); GGML_OP_CONVERTER(translate_reshape); GGML_OP_CONVERTER(translate_rms_norm); +GGML_OP_CONVERTER(translate_norm); +GGML_OP_CONVERTER(translate_l2_norm); +GGML_OP_CONVERTER(translate_sum_rows); GGML_OP_CONVERTER(translate_rope); GGML_OP_CONVERTER(translate_scale); GGML_OP_CONVERTER(translate_unary_silu); -GGML_OP_CONVERTER(translate_unary_gelu); +GGML_OP_CONVERTER(translate_unary_softplus); GGML_OP_CONVERTER(translate_soft_max); GGML_OP_CONVERTER(translate_transpose); GGML_OP_CONVERTER(translate_view); @@ -29,9 +35,15 @@ GGML_OP_CONVERTER(translate_glu_swiglu); GGML_OP_CONVERTER(translate_glu_geglu); GGML_OP_CONVERTER(translate_set_rows); GGML_OP_CONVERTER(translate_cpy); +GGML_OP_CONVERTER(translate_argsort); GGML_OP_CONVERTER(translate_flash_attn_ext); +GGML_OP_CONVERTER(translate_clamp); +GGML_OP_CONVERTER(translate_pad); +GGML_OP_CONVERTER(translate_ssm_conv); +GGML_OP_CONVERTER(translate_gated_delta_net); +GGML_OP_CONVERTER(translate_repeat); -} // namespace op +} // namespace op std::unordered_map get_supported_ops(); diff --git a/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h b/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h index b95385611..c229e25fb 100644 --- a/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +++ b/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h @@ -1,8 +1,8 @@ #pragma once #include "mark_decompression_convert_constant_folding.h" -#include "openvino/pass/matcher_pass.hpp" #include "openvino/core/visibility.hpp" +#include "openvino/pass/matcher_pass.hpp" #ifdef OPENVINO_STATIC_LIBRARY # define TRANSFORMATIONS_API diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index 0f68a1f50..d00c438e2 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -77,49 +78,48 @@ ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs( return pairs; } -void add_sliced_mask(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { - - auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name, bool is_static) { +void add_sliced_mask_stateful(TensorMap & tensor_map) { + auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name) { if ((tensor_map.find(mask_name) != tensor_map.end()) && (tensor_map.find("token_len_per_seq") != tensor_map.end())) { auto token_len_per_seq = tensor_map.at("token_len_per_seq").get_node_shared_ptr(); auto mask = tensor_map.at(mask_name).get_node_shared_ptr(); - std::shared_ptr mask_sliced; - if (is_static) { - mask_sliced = mask; - } else if (ggml_model_decoder.is_stateful()) { - auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0}); - auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1}); - auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto three_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto neg_one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {-2,-1}); - auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); - auto gather_inp_pos = std::make_shared(inp_pos, neg_one_1d, three_1d); - auto reshaped_inp_pos = std::make_shared(gather_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); - auto inp_pos_incremented = std::make_shared(reshaped_inp_pos, ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {1})); - auto stop = std::make_shared(ov::OutputVector{token_len_per_seq, std::make_shared(inp_pos_incremented, token_len_per_seq)}, 0); - mask_sliced = - std::make_shared(mask, zero_2d, stop, one_2d, axes); - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); - mask_sliced->set_friendly_name(sliced_name); - } else { - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - mask_sliced = std::make_shared(mask, zero, token_len_per_seq, one, two); - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); - mask_sliced->set_friendly_name(sliced_name); - } + std::shared_ptr mask_sliced = mask; + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); + auto last_inp_pos = std::make_shared(inp_pos, neg_one, three); + auto last_inp_pos_1d = std::make_shared( + last_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); + auto last_inp_pos_cvt = std::make_shared(last_inp_pos_1d, ov::element::i64); + auto last_inp_pos_inc = std::make_shared(last_inp_pos_cvt, one); + + mask_sliced = std::make_shared(mask, zero, last_inp_pos_inc, step, axes); + mask_sliced = std::make_shared(mask_sliced, ov::element::f16); + mask_sliced->set_friendly_name(sliced_name); + tensor_map.insert({sliced_name, mask_sliced->output(0)}); } }; - create_sliced_mask("self_kq_mask", "KQ_mask_sliced", ggml_model_decoder.is_static()); - create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced", ggml_model_decoder.is_static()); + create_sliced_mask("self_kq_mask", "KQ_mask_sliced"); + create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced"); } void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { + // When ROPE ops in the graph have divergent op_params (e.g. gemma4's mixed + // SWA/non-SWA layers with different n_dims or freq_base), a shared sin/cos + // precompute cannot broadcast across every ROPE use. Skip it here and let + // translate_rope() build sin/cos per-op from its own op_params. + if (ggml_model_decoder.has_mixed_rope_params()) { + return; + } int32_t * rope_params = ggml_model_decoder.get_rope_params(); if (tensor_map.find("inp_pos") == tensor_map.end() || rope_params == nullptr) { return; @@ -142,8 +142,11 @@ void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) // Create common patterns void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { - add_sliced_mask(tensor_map, ggml_model_decoder); - add_rope_sin_cos(tensor_map, ggml_model_decoder); + if (ggml_model_decoder.is_stateful()) { + add_sliced_mask_stateful(tensor_map); + } + // This optimization is error-prone + // add_rope_sin_cos(tensor_map, ggml_model_decoder); } } // namespace @@ -288,19 +291,19 @@ std::shared_ptr TranslateSession::apply_transformations(std::shared_ptris_stateful()) { auto output_names = ggml_model_decoder->get_model_output_names(); std::map model_output_indexes; - for (size_t i=0; iget_output_size(); i++) { + for (size_t i = 0; i < model->get_output_size(); i++) { auto output_friendly_name = model->output(i).get_node_shared_ptr()->get_friendly_name(); auto output_id = model_output_indexes[output_friendly_name]; auto model_output_shape = model->output(i).get_partial_shape(); auto decoder_output_shape = ggml_model_decoder->get_output_shape(output_id); - if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static() - && model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length() - && decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) { - ppp.output(i).postprocess().custom([](const ov::Output& node) { + if (model_output_shape.rank().is_static() && decoder_output_shape.rank().is_static() && + model_output_shape.rank().get_length() + 1 == decoder_output_shape.rank().get_length() && + decoder_output_shape[0].is_static() && decoder_output_shape[0].get_length() == 1) { + ppp.output(i).postprocess().custom([](const ov::Output & node) { auto axes = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {0}); return std::make_shared(node, axes); }); diff --git a/ggml/src/ggml-openvino/openvino/translate_session.h b/ggml/src/ggml-openvino/openvino/translate_session.h index 56a14ae7c..675e63223 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.h +++ b/ggml/src/ggml-openvino/openvino/translate_session.h @@ -9,16 +9,17 @@ namespace ggml { class TranslateSession { public: - TranslateSession(const frontend::InputModel::Ptr& input_model, - const std::unordered_map& translator_map, bool naive = false); + TranslateSession(const frontend::InputModel::Ptr & input_model, + const std::unordered_map & translator_map, + bool naive = false); std::shared_ptr get_converted_model(); - std::shared_ptr translate_graph(const frontend::InputModel::Ptr& input_model); + std::shared_ptr translate_graph(const frontend::InputModel::Ptr & input_model); private: std::shared_ptr apply_transformations(std::shared_ptr model); const frontend::InputModel::Ptr m_input_model; - const std::unordered_map& m_translator_map; + const std::unordered_map & m_translator_map; std::shared_ptr m_ov_model; bool m_naive; }; diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp index 0baaf88e1..4e4f5dd04 100644 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ b/ggml/src/ggml-openvino/openvino/utils.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -123,7 +124,8 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params bool imrope, bool stateful) { if (stateful) { - inp_pos = std::make_shared(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); + inp_pos = + std::make_shared(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); inp_pos = std::make_shared(inp_pos, ov::element::f32); auto pos_perm = std::make_shared(ov::element::i64, ov::Shape{3}, std::vector{2, 1, 0}); @@ -212,8 +214,9 @@ std::pair, ov::Output> make_sin_cos(int32_t * rope_params } auto one_minus_ramp = std::make_shared(one, ramp_mix); - theta = std::make_shared(std::make_shared(theta_interp, one_minus_ramp), - std::make_shared(theta_extrap, ramp_mix)); + theta = + std::make_shared(std::make_shared(theta_interp, one_minus_ramp), + std::make_shared(theta_extrap, ramp_mix)); mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale)); } } @@ -252,6 +255,548 @@ ov::Output process_view_input(const NodeContext & context, int input_i return sliced; } +ov::Output process_view_input_new(const NodeContext & context, int input_index) { + auto input = context.get_input(input_index); + + // Check if this input has view inputs + size_t view_input_size = context.get_view_input_size(input_index); + if (view_input_size == 0) { + // No view inputs, return the input as is + return input; + } + + // If translate_view already resolved this VIEW (produced a Slice), the input + // will already have the expected shape — skip re-slicing. + auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0); + auto actual_shape = input.get_partial_shape(); + if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() && + expected_ov_shape.rank() == actual_shape.rank()) { + bool shapes_match = true; + for (int64_t i = 0; i < expected_ov_shape.rank().get_length(); ++i) { + if (!expected_ov_shape[i].is_static() || !actual_shape[i].is_static()) { + shapes_match = false; + break; + } + if (expected_ov_shape[i] != actual_shape[i]) { + shapes_match = false; + break; + } + } + if (shapes_match) { + return input; + } + } + + // In static mode, use Split instead of Slice for single-dimension reductions. + // This ensures NPUW's FOLD doesn't parametrize per-layer slice indices (which + // would introduce dynamic shapes). A shared Split node sits outside the repeated + // subgraph boundary; each layer receives one of its output ports. + if (context.is_static() && view_input_size == 1) { + auto view_stride_v = context.get_view_input_stride(input_index, 0); + auto view_src_stride_v = context.get_view_input_src_stride(input_index, 0); + auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, 0); + auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, 0); + auto view_offset = context.get_view_input_offset(input_index, 0); + auto view_src_offset = context.get_view_input_src_offset(input_index, 0); + + size_t ndims = view_ggml_shape.size(); + std::vector diff_dims; + if (view_src_ggml_shape.size() == ndims) { + for (size_t i = 0; i < ndims; ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + } + + if (diff_dims.size() == 1) { + int split_dim = diff_dims[0]; + int64_t num_splits = static_cast(view_src_ggml_shape[split_dim]); + int64_t chunk_size = static_cast(view_ggml_shape[split_dim]); + + // Only apply when slicing exactly 1 element from a multi-element dimension + if (chunk_size == 1 && num_splits > 1) { + // Check suffix strides match (dimensions after split_dim) + bool suffix_ok = view_stride_v.size() == view_src_stride_v.size(); + if (suffix_ok) { + for (size_t i = static_cast(split_dim) + 1; i < ndims; ++i) { + if (view_stride_v[i] != view_src_stride_v[i]) { + suffix_ok = false; + break; + } + } + } + + if (suffix_ok && view_src_stride_v[split_dim] > 0) { + size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; + int64_t split_index = static_cast(relative_offset / view_src_stride_v[split_dim]); + + if (split_index >= 0 && split_index < num_splits) { + auto src_node = input.get_node_shared_ptr(); + std::string rt_key = "split_dim_" + std::to_string(split_dim); + auto & rt_info = src_node->get_rt_info(); + + if (rt_info.find(rt_key) == rt_info.end()) { + auto axis_const = + ov::op::v0::Constant::create(ov::element::i64, {}, {static_cast(split_dim)}); + auto split_node = + std::make_shared(input, axis_const, static_cast(num_splits)); + split_node->set_friendly_name(src_node->get_friendly_name() + "_split"); + rt_info[rt_key] = split_node; + } + + auto split_node = rt_info[rt_key].as>(); + return split_node->output(static_cast(split_index)); + } + } + } + } + } + + // Lambda function to process a single view operation + auto process_single_view = + [](ov::Output current, size_t view_offset, const std::vector & view_stride, + const ov::Shape & view_ggml_shape, const ov::PartialShape & view_ov_shape, const std::string & view_name, + size_t view_src_offset, const std::vector & view_src_stride, const ov::Shape & view_src_ggml_shape, + const ov::PartialShape & view_src_ov_shape, const std::string & view_src_name) -> ov::Output { + auto build_reshape_pattern = [](const ov::PartialShape & target_ov_shape, + const ov::Shape & target_ggml_shape) -> std::vector { + const size_t ndims = target_ggml_shape.size(); + std::vector reshape_pattern(ndims); + size_t dynamic_dims = 0; + + if (target_ov_shape.rank().is_static() && + target_ov_shape.rank().get_length() == static_cast(ndims)) { + for (size_t i = 0; i < ndims; ++i) { + if (target_ov_shape[i].is_static()) { + reshape_pattern[i] = target_ov_shape[i].get_length(); + } else { + reshape_pattern[i] = -1; + ++dynamic_dims; + } + } + } else { + dynamic_dims = 2; + } + + if (dynamic_dims > 1) { + for (size_t i = 0; i < ndims; ++i) { + reshape_pattern[i] = static_cast(target_ggml_shape[i]); + } + } + + return reshape_pattern; + }; + + auto build_prefix_tail_reshape_pattern = [](const ov::PartialShape & target_ov_shape, + const ov::Shape & target_ggml_shape, size_t prefix_dims, + int64_t tail_dim) -> std::vector { + std::vector reshape_pattern(prefix_dims + 1); + size_t dynamic_dims = 0; + + if (target_ov_shape.rank().is_static() && + target_ov_shape.rank().get_length() == static_cast(target_ggml_shape.size())) { + for (size_t i = 0; i < prefix_dims; ++i) { + if (target_ov_shape[i].is_static()) { + reshape_pattern[i] = target_ov_shape[i].get_length(); + } else { + reshape_pattern[i] = -1; + ++dynamic_dims; + } + } + } else { + dynamic_dims = 2; + } + + if (dynamic_dims > 1) { + for (size_t i = 0; i < prefix_dims; ++i) { + reshape_pattern[i] = static_cast(target_ggml_shape[i]); + } + } + + reshape_pattern[prefix_dims] = tail_dim; + return reshape_pattern; + }; + + bool same_stride = view_stride.size() == view_src_stride.size(); + if (same_stride) { + for (size_t i = 0; i < view_stride.size(); ++i) { + if (view_stride[i] != view_src_stride[i]) { + same_stride = false; + break; + } + } + } + + bool same_ggml_shape = view_ggml_shape.size() == view_src_ggml_shape.size(); + if (same_ggml_shape) { + for (size_t i = 0; i < view_ggml_shape.size(); ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + same_ggml_shape = false; + break; + } + } + } + + if (same_stride && same_ggml_shape) { + return current; + } + + if (same_stride) { + const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; + const size_t ndims = view_stride.size(); + + std::vector diff_dims; + if (view_ggml_shape.size() == ndims && view_src_ggml_shape.size() == ndims) { + for (size_t i = 0; i < ndims; ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + } + + if (diff_dims.size() == 1) { + const int slice_dim = diff_dims[0]; + const int64_t dim_size = static_cast(view_src_ggml_shape[slice_dim]); + + if (view_stride[slice_dim] > 0 && relative_offset % view_stride[slice_dim] == 0) { + const int64_t begin_val = static_cast((relative_offset / view_stride[slice_dim]) % + static_cast(dim_size)); + const int64_t end_val = begin_val + static_cast(view_ggml_shape[slice_dim]); + + if (begin_val >= 0 && end_val <= dim_size) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); + + if (view_ov_shape.is_static()) { + auto reshaped = std::make_shared( + sliced, + ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + + sliced->set_friendly_name(view_name); + return sliced; + } + } + + int64_t tail_src_elems = 1; + int64_t tail_dst_elems = 1; + for (size_t i = slice_dim; i < ndims; ++i) { + tail_src_elems *= static_cast(view_src_ggml_shape[i]); + tail_dst_elems *= static_cast(view_ggml_shape[i]); + } + + const size_t elem_stride = view_stride[ndims - 1]; + int64_t tail_begin = 0; + if (elem_stride > 0) { + tail_begin = + static_cast((relative_offset / elem_stride) % static_cast(tail_src_elems)); + } + const int64_t tail_end = tail_begin + tail_dst_elems; + + if (tail_begin >= 0 && tail_end <= tail_src_elems) { + std::vector flat_shape; + for (int i = 0; i < slice_dim; ++i) { + flat_shape.push_back(static_cast(view_src_ggml_shape[i])); + } + flat_shape.push_back(tail_src_elems); + const size_t flat_ndims = flat_shape.size(); + + auto flat = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {flat_ndims}, flat_shape), false); + + auto sliced = std::make_shared( + flat, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); + + if (view_ov_shape.is_static()) { + auto reshaped = std::make_shared( + sliced, ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + + sliced->set_friendly_name(view_name); + return sliced; + } + } + + std::vector begin(ndims, 0); + std::vector end(ndims, 0); + std::vector step(ndims, 1); + std::vector axes(ndims, 0); + + size_t remaining_offset = relative_offset; + for (size_t i = 0; i < ndims; ++i) { + axes[i] = static_cast(i); + if (view_stride[i] > 0) { + begin[i] = static_cast(remaining_offset / view_stride[i]); + remaining_offset %= view_stride[i]; + } + end[i] = begin[i] + static_cast(view_ggml_shape[i]); + } + + bool in_bounds = view_src_ggml_shape.size() == ndims && view_ggml_shape.size() == ndims; + if (in_bounds) { + for (size_t i = 0; i < ndims; ++i) { + if (end[i] > static_cast(view_src_ggml_shape[i])) { + in_bounds = false; + break; + } + } + } + + if (in_bounds && remaining_offset == 0) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, end), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, step), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes)); + + sliced->set_friendly_name(view_name); + return sliced; + } + } else { + bool same_rank = view_stride.size() == view_src_stride.size() && + view_ggml_shape.size() == view_src_ggml_shape.size() && + view_stride.size() == view_ggml_shape.size(); + const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; + + if (same_rank) { + const size_t ndims = view_ggml_shape.size(); + std::vector diff_dims; + for (size_t i = 0; i < ndims; ++i) { + if (view_ggml_shape[i] != view_src_ggml_shape[i]) { + diff_dims.push_back(static_cast(i)); + } + } + + if (diff_dims.size() == 1) { + const size_t slice_dim = static_cast(diff_dims[0]); + bool suffix_stride_match = true; + for (size_t i = slice_dim + 1; i < ndims; ++i) { + if (view_stride[i] != view_src_stride[i]) { + suffix_stride_match = false; + break; + } + } + + if (suffix_stride_match && view_src_stride[slice_dim] > 0 && + relative_offset % view_src_stride[slice_dim] == 0) { + const int64_t begin_val = static_cast(relative_offset / view_src_stride[slice_dim]); + const int64_t end_val = begin_val + static_cast(view_ggml_shape[slice_dim]); + const int64_t dim_size = static_cast(view_src_ggml_shape[slice_dim]); + + if (begin_val >= 0 && end_val <= dim_size) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {static_cast(slice_dim)})); + sliced->set_friendly_name(view_name); + return sliced; + } + } + } + } + + size_t view_elems = 1; + size_t src_elems = 1; + if (same_rank) { + for (size_t i = 0; i < view_ggml_shape.size(); ++i) { + view_elems *= view_ggml_shape[i]; + src_elems *= view_src_ggml_shape[i]; + } + } + + bool same_num_elements = same_rank && view_elems == src_elems; + + if (same_rank && relative_offset == 0 && same_num_elements) { + auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape); + + auto reshaped = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, reshape_pattern), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + + if (same_rank) { + const size_t ndims = view_ggml_shape.size(); + + // Match views that can be expressed as a regular strided slice over the + // already reconstructed source tensor, e.g. offset on one axis plus step > 1 + // on another axis. + bool is_regular_slice = view_src_ggml_shape.size() == ndims; + std::vector begin(ndims, 0); + std::vector end(ndims, 0); + std::vector step(ndims, 1); + std::vector axes(ndims, 0); + size_t remaining_offset = relative_offset; + + if (is_regular_slice) { + for (size_t i = 0; i < ndims; ++i) { + axes[i] = static_cast(i); + + if (view_src_stride[i] == 0 || view_stride[i] == 0 || + view_stride[i] % view_src_stride[i] != 0) { + is_regular_slice = false; + break; + } + + step[i] = static_cast(view_stride[i] / view_src_stride[i]); + if (step[i] <= 0) { + is_regular_slice = false; + break; + } + + begin[i] = static_cast(remaining_offset / view_src_stride[i]); + remaining_offset %= view_src_stride[i]; + + if (view_ggml_shape[i] == 0) { + end[i] = begin[i]; + continue; + } + + end[i] = begin[i] + step[i] * static_cast(view_ggml_shape[i] - 1) + 1; + + if (begin[i] < 0 || end[i] > static_cast(view_src_ggml_shape[i])) { + is_regular_slice = false; + break; + } + } + } + + if (is_regular_slice && remaining_offset == 0) { + auto sliced = std::make_shared( + current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, end), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, step), + ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes)); + + sliced->set_friendly_name(view_name); + return sliced; + } + + const size_t elem_stride = view_src_stride.back(); + const bool aligned_offset = elem_stride > 0 && relative_offset % elem_stride == 0; + + if (aligned_offset) { + size_t suffix_start = 0; + size_t expected_stride = elem_stride; + for (int i = static_cast(ndims) - 1; i >= 0; --i) { + if (view_stride[i] != expected_stride) { + suffix_start = static_cast(i + 1); + break; + } + expected_stride *= view_ggml_shape[i]; + } + + size_t prefix_elems = 1; + size_t suffix_elems = 1; + for (size_t i = 0; i < suffix_start; ++i) { + prefix_elems *= view_ggml_shape[i]; + } + for (size_t i = suffix_start; i < ndims; ++i) { + suffix_elems *= view_ggml_shape[i]; + } + + if (prefix_elems > 0 && src_elems % prefix_elems == 0) { + const size_t src_tail_elems = src_elems / prefix_elems; + const int64_t tail_begin = static_cast(relative_offset / elem_stride); + const int64_t tail_end = tail_begin + static_cast(suffix_elems); + + if (tail_begin >= 0 && tail_end <= static_cast(src_tail_elems)) { + auto prefix_tail_pattern = build_prefix_tail_reshape_pattern( + view_ov_shape, view_ggml_shape, suffix_start, static_cast(src_tail_elems)); + + auto prefix_tail = std::make_shared( + current, + ov::op::v0::Constant::create(ov::element::i64, {prefix_tail_pattern.size()}, + prefix_tail_pattern), + false); + + ov::Output selected = prefix_tail; + if (tail_begin != 0 || tail_end != static_cast(src_tail_elems)) { + selected = std::make_shared( + prefix_tail, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}), + ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), + ov::op::v0::Constant::create(ov::element::i64, {1}, + {static_cast(suffix_start)})); + } + + auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape); + auto reshaped = std::make_shared( + selected, + ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, + reshape_pattern), + false); + reshaped->set_friendly_name(view_name); + return reshaped; + } + } + } + } + + return current; + } + + (void) view_name; + (void) view_src_ov_shape; + (void) view_src_name; + + return current; + }; + + // Process views from the base tensor (last) to the current view (first) + // Start with the base tensor + ov::Output current = input; + + // Process each view in reverse order (from base to current) + for (int view_idx = view_input_size - 1; view_idx >= 0; view_idx--) { + auto view_offset = context.get_view_input_offset(input_index, view_idx); + auto view_stride = context.get_view_input_stride(input_index, view_idx); + auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, view_idx); + auto view_ov_shape = context.get_view_input_ov_shape(input_index, view_idx); + auto view_name = context.get_view_input_name(input_index, view_idx); + + // print view info + // std::cout << "View " << view_idx << ": name = " << view_name << ", offset = " << view_offset << ", stride = [" + // << view_stride[0] << "," << view_stride[1] << "," << view_stride[2] << "," << view_stride[3] + // << "], ggml shape = [" << view_ggml_shape[0] << "," << view_ggml_shape[1] << "," + // << view_ggml_shape[2] << "," << view_ggml_shape[3] << "], ov shape = " << view_ov_shape << std::endl; + + auto view_src_offset = context.get_view_input_src_offset(input_index, view_idx); + auto view_src_stride = context.get_view_input_src_stride(input_index, view_idx); + auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, view_idx); + auto view_src_ov_shape = context.get_view_input_src_ov_shape(input_index, view_idx); + auto view_src_name = context.get_view_input_src_name(input_index, view_idx); + // print source view info + // std::cout << "View " << view_idx << ": source name = " << view_src_name + // << ", source offset = " << view_src_offset << ", source stride = [" << view_src_stride[0] << "," + // << view_src_stride[1] << "," << view_src_stride[2] << "," << view_src_stride[3] + // << "], source ggml shape = [" << view_src_ggml_shape[0] << "," << view_src_ggml_shape[1] << "," + // << view_src_ggml_shape[2] << "," << view_src_ggml_shape[3] + // << "], source ov shape = " << view_src_ov_shape << std::endl; + + current = process_single_view(current, view_offset, view_stride, view_ggml_shape, view_ov_shape, view_name, + view_src_offset, view_src_stride, view_src_ggml_shape, view_src_ov_shape, + view_src_name); + } + + return current; +} + } // namespace ggml } // namespace frontend } // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h index 767dd4c53..8dc3e8765 100644 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ b/ggml/src/ggml-openvino/openvino/utils.h @@ -1,13 +1,13 @@ #pragma once +#include "node_context.h" + #include #include #include #include #include -#include "node_context.h" - namespace ov { namespace frontend { namespace ggml { @@ -16,30 +16,23 @@ std::string getCurrentTime(); void dump_ov_model(std::shared_ptr model); -void num_inputs_check(const NodeContext& context, size_t min_inputs, size_t max_inputs); +void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs); int non_cont_dim(std::vector ne, std::vector nb); -template -std::vector argsort_descend(const std::vector& v) { +template std::vector argsort_descend(const std::vector & v) { std::vector idx(v.size()); std::iota(idx.begin(), idx.end(), 0); - std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) { - return v[i1] > v[i2]; - }); + std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) { return v[i1] > v[i2]; }); return idx; } -template -std::vector sorted_descend(std::vector v) { - std::sort(v.begin(), v.end(), [](T a, T b) { - return a > b; - }); +template std::vector sorted_descend(std::vector v) { + std::sort(v.begin(), v.end(), [](T a, T b) { return a > b; }); return v; } -template -bool is_permuted(const std::vector& strides) { +template bool is_permuted(const std::vector & strides) { for (size_t i = 0; i < strides.size() - 1; ++i) { if (strides[i] < strides[i + 1]) { return true; @@ -48,8 +41,7 @@ bool is_permuted(const std::vector& strides) { return false; } -template -std::vector permute(const std::vector& x, const std::vector& perm) { +template std::vector permute(const std::vector & x, const std::vector & perm) { std::vector result; result.reserve(perm.size()); for (int i : perm) { @@ -58,25 +50,35 @@ std::vector permute(const std::vector& x, const std::vector& perm) { return result; } -std::shared_ptr get_dimensions(const std::shared_ptr& shape, - const std::vector& dims); -std::shared_ptr get_dimensions(const std::shared_ptr& node, const std::vector& dims); +std::shared_ptr get_dimensions(const std::shared_ptr & shape, + const std::vector & dims); +std::shared_ptr get_dimensions(const std::shared_ptr & node, const std::vector & dims); -OutputVector rename_outputs_with_suffix(const OutputVector& outputs, const std::string& suffix); +OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std::string & suffix); -std::pair, ov::Output> make_sin_cos(int32_t* rope_params, +std::pair, ov::Output> make_sin_cos(int32_t * rope_params, std::shared_ptr inp_pos, std::shared_ptr rope_freqs_weight = nullptr, bool imrope = false, bool stateful = false); -ov::Output process_view_input(const NodeContext& context, int input_index, int slice_len = 0); +ov::Output process_view_input(const NodeContext & context, int input_index, int slice_len = 0); + +ov::Output process_view_input_new(const NodeContext & context, int input_index); namespace op { -template -OutputVector translate_1to1_match_2_inputs(const NodeContext& context) { +template OutputVector translate_1to1_match_2_inputs(const NodeContext & context) { num_inputs_check(context, 2, 2); - auto res = std::make_shared(context.get_input(0), context.get_input(1)); + auto input_0 = process_view_input_new(context, 0); + auto input_1 = process_view_input_new(context, 1); + auto res = std::make_shared(input_0, input_1); + return rename_outputs_with_suffix({res}, context.get_name()); +} + +template OutputVector translate_1to1_match_1_input(const NodeContext & context) { + num_inputs_check(context, 1, 1); + auto input = process_view_input_new(context, 0); + auto res = std::make_shared(input); return rename_outputs_with_suffix({res}, context.get_name()); } } // namespace op diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 998ef7c9e..70af08bdf 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -25,9 +26,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -39,7 +42,7 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) { ggml_backend_openvino_context * ctx = (ggml_backend_openvino_context *) backend->context; try { - if (getenv("GGML_OPENVINO_DUMP_CGRAPH")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_CGRAPH")) { std::string filename = "cgraph_ov.txt"; GgmlOvDecoder::dump_cgraph(cgraph, filename); } @@ -62,10 +65,92 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) } } +// For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size +// for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions: +// * non-static (CPU/GPU) backend, single sequence, seq_active_start == 0 +// * ggml KV layout is a contiguous [1, 1, ctx_per_seq, n_heads_kv*head_size] +// so the first n_kv rows are the live prefix and shrinking the ctx axis +// gives a valid tensor over the same host storage +// * not an SWA layer (ring cache): once the window has wrapped the first +// n_kv rows no longer contain the live prefix +// On any unmet pre-condition returns std::nullopt; the caller falls back to +// the full-size tensor. +static std::optional try_make_kv_sliced_tensor(std::shared_ptr ggml_decoder, + const std::string & name, + const ggml_tensor * ggml_tensor) { + static const bool kv_slice_disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE"); + if (kv_slice_disabled) { + return std::nullopt; + } + if (ggml_decoder->is_static() || ggml_decoder->is_stateful()) { + return std::nullopt; + } + if (ggml_tensor->op != GGML_OP_NONE || ggml_tensor->view_src != nullptr) { + return std::nullopt; + } + const auto * op = ggml_decoder->get_tensor_used_op(ggml_tensor); + if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op)) { + return std::nullopt; + } + + const auto & compute_params = ggml_decoder->get_compute_params(); + if (compute_params.n_seq_active != 1 || compute_params.seq_active_start != 0) { + return std::nullopt; + } + + int layer; + if (auto layer_opt = extract_layer_from_name(name); layer_opt.has_value()) { + layer = layer_opt.value(); + } else { + return std::nullopt; + } + + const bool is_swa = ggml_decoder->is_swa_layer(layer); + if (is_swa) { + return std::nullopt; + } + const int ctx_per_seq = ggml_decoder->get_ctx_per_seq(); + const int n_kv = compute_params.attention_size; + if (ctx_per_seq <= 0 || n_kv <= 0 || n_kv >= ctx_per_seq) { + return std::nullopt; + } + + ov::Shape full_shape = ggml_decoder->get_shape(ggml_tensor); + if (full_shape.size() != 4 || full_shape[0] != 1 || full_shape[1] != 1 || + static_cast(full_shape[2]) != ctx_per_seq) { + return std::nullopt; + } + + ov::Shape sliced_shape = full_shape; + sliced_shape[2] = static_cast(n_kv); + + // Disabling for now as gpu has bug with in-place ScatterUpdate with remote tensors, can re-enable once CVS-186519 is fixed + // if (ggml_openvino_buffer_is_remote(ggml_tensor)) { + // auto remote_context = ggml_openvino_get_remote_context(); + // auto gpu_context = remote_context->as(); + // return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); + // } + + return ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), sliced_shape, ggml_tensor->data); +} + ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, std::shared_ptr infer_request, int output_index, const ggml_tensor * ggml_tensor) { + if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) { + return *sliced; + } + + // Disabling for now as gpu has bug with in-place ScatterUpdate with remote tensors, can re-enable once CVS-186519 is fixed + // if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { + // auto * extra_base = static_cast(ggml_tensor->extra); + // if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { + // auto * tensor_extra = static_cast(extra_base); + // return *tensor_extra->tensor; + // } + // } + auto output_type = ggml_decoder->get_ov_type(ggml_tensor); ov::Shape output_shape; if (ggml_decoder->is_static()) { @@ -86,7 +171,9 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< static auto is_static = false; if (is_naive(cgraph)) { - return naive_compute(cgraph, core, device, config); + if (!is_model_splitted(cgraph)) { + return naive_compute(cgraph, core, device, config); + } } auto start_time = ggml_time_us(); @@ -98,18 +185,20 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); graph_key key(cgraph); - bool cache_hit; + static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); + bool cache_hit = false; int64_t decoder_end_time; int64_t conversion_end_time; int64_t compile_end_time; int64_t infer_end_time; + int64_t ov_raw_infer_start; { std::shared_ptr entry; ModelParams old_m_params; - { + if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); auto it = r_ctx->decoder_cache.find(key); cache_hit = it != r_ctx->decoder_cache.end(); @@ -120,6 +209,10 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< entry = std::make_shared(mutex); r_ctx->decoder_cache[key] = entry; } + } else { + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + cache_hit = false; } std::lock_guard lock(*(entry->mutex)); @@ -127,9 +220,14 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< if (cache_hit) { ggml_decoder = entry->ptr; old_m_params = ggml_decoder->get_model_params(); - cache_hit = old_m_params.can_reuse_dynamically(m_params); + if (!ggml_decoder->is_splited_model()) { + cache_hit = old_m_params.can_reuse_dynamically(m_params); + } } + std::vector ov_input_names; + std::vector ov_output_names; + if (cache_hit) { std::map> model_weights; ggml_decoder->set_compute_params(c_params); @@ -141,6 +239,8 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< { std::lock_guard map_lock(r_ctx->ctx_mutex); infer_request = r_ctx->infer_request_cache.at(key); + ov_input_names = r_ctx->ov_input_names_cache.at(key); + ov_output_names = r_ctx->ov_output_names_cache.at(key); } if (stateful) { @@ -162,14 +262,15 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< try { state_name = r_ctx->kv_state_input_name_map.at(state.get_name()); } catch (...) { - GGML_LOG_ERROR("GGML OpenVINO backend stateful inference failed: no input found for the state\n"); + GGML_LOG_ERROR( + "GGML OpenVINO backend stateful inference failed: no input found for the state\n"); return GGML_STATUS_FAILED; } auto kv_tensor = get_ov_input_tensor(ggml_decoder, state_name); - kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2], - state_tensor_shape[2], state_tensor_shape[3]}); - state_tensor = kv_tensor; - state_tensor_shape = state_tensor.get_shape(); + kv_tensor.set_shape({state_tensor_shape[0], kv_tensor.get_shape()[2], state_tensor_shape[2], + state_tensor_shape[3]}); + state_tensor = kv_tensor; + state_tensor_shape = state_tensor.get_shape(); } ov::Coordinate begin = {0, 0, 0, 0}; ov::Coordinate end = {state_tensor_shape[0], static_cast(pos_data[0]), @@ -177,7 +278,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ov::Tensor new_state_tensor(state_tensor, begin, end); state.set_state(new_state_tensor); } - r_ctx->stateful_kv_size = pos_data[0] + 1; + r_ctx->stateful_kv_size = pos_data[0] + pos_shape[3]; } } @@ -185,15 +286,17 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { - { + if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache.erase(key); } + bool model_is_splitted = is_model_splitted(cgraph); std::shared_ptr model; auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, stateful); + ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + stateful, model_is_splitted); decoder_end_time = ggml_time_us(); auto input_model = std::make_shared(ggml_decoder); @@ -201,7 +304,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ggml_decoder->clear_model_weights(); conversion_end_time = ggml_time_us(); - if (getenv("GGML_OPENVINO_DUMP_IR")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { char timestamped_filename[64]; auto timestamp = (long long) ggml_time_us(); snprintf(timestamped_filename, sizeof(timestamped_filename), "model_%lld.xml", timestamp); @@ -219,8 +322,6 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< infer_request = std::make_shared(compiled_model.create_infer_request()); entry->ptr = ggml_decoder; - std::vector ov_input_names; - std::vector ov_output_names; for (const auto & ov_param : model->get_parameters()) { ov_input_names.push_back(ov_param->get_friendly_name()); } @@ -228,66 +329,64 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< ov_output_names.push_back(ov_output->get_friendly_name()); } - { + if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache[key] = infer_request; - r_ctx->ov_input_names_cache[key] = std::move(ov_input_names); - r_ctx->ov_output_names_cache[key] = std::move(ov_output_names); + r_ctx->ov_input_names_cache[key] = ov_input_names; + r_ctx->ov_output_names_cache[key] = ov_output_names; } - if (stateful) { + if (stateful && cache_enabled) { const auto * inp_pos = get_inp_pos_tensor(cgraph); auto pos_shape = ggml_decoder->get_shape(inp_pos); r_ctx->stateful_kv_size = pos_shape[3]; const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); - for (const auto& pair : kv_param_res_names) { - r_ctx->kv_state_input_name_map[pair.first+pair.second] = pair.first; + for (const auto & pair : kv_param_res_names) { + r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; } } } - std::vector ov_input_names; - std::vector ov_output_names; - { - std::lock_guard map_lock(r_ctx->ctx_mutex); - ov_input_names = r_ctx->ov_input_names_cache[key]; - ov_output_names = r_ctx->ov_output_names_cache[key]; - } - for (size_t i = 0; i < ov_input_names.size(); i++) { auto param_name = ov_input_names[i]; auto input_tensor = get_ov_input_tensor(ggml_decoder, param_name); infer_request->set_input_tensor(i, input_tensor); - if (getenv("GGML_OPENVINO_DEBUG_INPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { print_input_tensor_info(param_name, input_tensor); } } for (size_t i = 0; i < ov_output_names.size(); i++) { auto * ggml_tensor = ggml_decoder->get_model_outputs().at(ov_output_names[i]); + if (ggml_nbytes(ggml_tensor) == 0) { + continue; + } auto output_tensor = create_ov_output_tensor(ggml_decoder, infer_request, i, ggml_tensor); infer_request->set_output_tensor(i, output_tensor); } + ov_raw_infer_start = ggml_time_us(); infer_request->infer(); infer_end_time = ggml_time_us(); - if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) { for (size_t i = 0; i < ov_output_names.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); print_output_tensor_info(ov_output_names[i], output_tensor, output_tensor.data()); } } - if (getenv("GGML_OPENVINO_PROFILING")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING")) { GGML_LOG_INFO("\nGGML OpenVINO Backend: \n"); - GGML_LOG_INFO(" - Graph decoder time: %ld ms \n", (decoder_end_time - start_time) / 1000); + GGML_LOG_INFO(" - Graph decoder time: %.3f ms \n", (decoder_end_time - start_time) / 1000.0); if (!cache_hit) { - GGML_LOG_INFO(" - Graph conversion time: %ld ms \n", (conversion_end_time - decoder_end_time) / 1000); - GGML_LOG_INFO(" - Graph compile time: %ld ms \n", (compile_end_time - conversion_end_time) / 1000); + GGML_LOG_INFO(" - Graph conversion time: %.3f ms \n", + (conversion_end_time - decoder_end_time) / 1000.0); + GGML_LOG_INFO(" - Graph compile time: %.3f ms \n", (compile_end_time - conversion_end_time) / 1000.0); } - GGML_LOG_INFO(" - Graph inference time: %ld ms \n", (infer_end_time - compile_end_time) / 1000); + GGML_LOG_INFO(" - Graph inference time: %.3f ms \n", (infer_end_time - compile_end_time) / 1000.0); + GGML_LOG_INFO(" - OV raw infer time: %.3f ms \n", (infer_end_time - ov_raw_infer_start) / 1000.0); } } @@ -298,17 +397,18 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr 0) { - return atoi(chunk_size_str); - } - return 256; + static const int chunk_size = []() { + int env_prefill_chunk_size = ggml_openvino_getenv_int("GGML_OPENVINO_PREFILL_CHUNK_SIZE"); + return env_prefill_chunk_size > 0 ? env_prefill_chunk_size : 256; + }(); + return chunk_size; }; static std::string device = "NPU"; static auto is_static = true; static auto stateful = false; - static auto prefill_chunk_size = get_prefill_chunk_size(); + + auto prefill_chunk_size = get_prefill_chunk_size(); const auto & config = ggml_openvino_get_compile_config(); if (is_naive(cgraph)) { @@ -326,17 +426,20 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr entry; ModelParams old_m_params; - { + if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); auto it = r_ctx->decoder_cache.find(key); cache_hit = it != r_ctx->decoder_cache.end(); @@ -347,6 +450,10 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr(mutex); r_ctx->decoder_cache[key] = entry; } + } else { + auto mutex = std::make_shared(); + entry = std::make_shared(mutex); + cache_hit = false; } std::lock_guard lock(*(entry->mutex)); @@ -357,6 +464,9 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr ov_input_names_local; + std::vector ov_output_names_local; + if (cache_hit) { std::map> model_weights; ggml_decoder->m_is_prefill = is_prefill; @@ -370,13 +480,15 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr map_lock(r_ctx->ctx_mutex); infer_request = is_prefill ? r_ctx->infer_request_cache_prefill.at(key) : r_ctx->infer_request_cache.at(key); + ov_input_names_local = r_ctx->ov_input_names_cache.at(key); + ov_output_names_local = r_ctx->ov_output_names_cache.at(key); } decoder_end_time = ggml_time_us(); conversion_end_time = decoder_end_time; compile_end_time = decoder_end_time; } else { - { + if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache.erase(key); r_ctx->infer_request_cache_prefill.erase(key); @@ -385,10 +497,14 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptr model; auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - auto ggml_decoder_prefill = std::make_shared(cgraph, m_params, c_params, model_weights, - is_static, stateful, true, prefill_chunk_size); + if (m_params.n_heads_kv == -1) { + // graph is not a LLM, e.g. context-shift graph + prefill_chunk_size = inp_pos->ne[0]; + } + auto ggml_decoder_prefill = std::make_shared( + cgraph, m_params, c_params, model_weights, is_static, stateful, false, true, prefill_chunk_size); auto ggml_decoder_decode = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, - stateful, false, prefill_chunk_size); + stateful, false, false, prefill_chunk_size); decoder_end_time = ggml_time_us(); auto input_model_prefill = std::make_shared(ggml_decoder_prefill); @@ -400,7 +516,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrclear_model_weights(); conversion_end_time = ggml_time_us(); - if (getenv("GGML_OPENVINO_DUMP_IR")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { char timestamped_filename[64]; auto timestamp = (long long) ggml_time_us(); snprintf(timestamped_filename, sizeof(timestamped_filename), "model_prefill_%lld.xml", timestamp); @@ -429,32 +545,22 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrptr = ggml_decoder; - std::vector ov_input_names; - std::vector ov_output_names; for (const auto & ov_param : model->get_parameters()) { - ov_input_names.push_back(ov_param->get_friendly_name()); + ov_input_names_local.push_back(ov_param->get_friendly_name()); } for (const auto & ov_output : model->get_results()) { - ov_output_names.push_back(ov_output->get_friendly_name()); + ov_output_names_local.push_back(ov_output->get_friendly_name()); } - { + if (cache_enabled) { std::lock_guard map_lock(r_ctx->ctx_mutex); r_ctx->infer_request_cache_prefill[key] = infer_request_prefill; r_ctx->infer_request_cache[key] = infer_request_decode; - r_ctx->ov_input_names_cache[key] = std::move(ov_input_names); - r_ctx->ov_output_names_cache[key] = std::move(ov_output_names); + r_ctx->ov_input_names_cache[key] = ov_input_names_local; + r_ctx->ov_output_names_cache[key] = ov_output_names_local; } } - std::vector ov_input_names_local; - std::vector ov_output_names_local; - { - std::lock_guard map_lock(r_ctx->ctx_mutex); - ov_input_names_local = r_ctx->ov_input_names_cache[key]; - ov_output_names_local = r_ctx->ov_output_names_cache[key]; - } - if (is_prefill) { auto inp_len = inp_pos->ne[0]; for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { @@ -463,7 +569,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrset_input_tensor(i, input_tensor); - if (getenv("GGML_OPENVINO_DEBUG_INPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { const auto input_tensor = infer_request->get_input_tensor(i); print_input_tensor_info(param_name, input_tensor); } @@ -475,9 +581,11 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrset_output_tensor(i, output_tensor); } + ov_raw_infer_start = ggml_time_us(); infer_request->infer(); + ov_raw_infer_total += ggml_time_us() - ov_raw_infer_start; - if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) { for (size_t i = 0; i < ov_output_names_local.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); @@ -491,7 +599,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrset_input_tensor(i, input_tensor); - if (getenv("GGML_OPENVINO_DEBUG_INPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_INPUT")) { const auto input_tensor = infer_request->get_input_tensor(i); print_input_tensor_info(param_name, input_tensor); } @@ -503,10 +611,12 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrset_output_tensor(i, output_tensor); } + ov_raw_infer_start = ggml_time_us(); infer_request->infer(); infer_end_time = ggml_time_us(); + ov_raw_infer_total = infer_end_time - ov_raw_infer_start; - if (getenv("GGML_OPENVINO_DEBUG_OUTPUT")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DEBUG_OUTPUT")) { for (size_t i = 0; i < ov_output_names_local.size(); i++) { const auto output_tensor = infer_request->get_output_tensor(i); print_output_tensor_info(ov_output_names_local[i], output_tensor, output_tensor.data()); @@ -514,19 +624,75 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, std::shared_ptrsrc. +// Step 2 verifies that node inputs come from model nodes/weights/leafs; external sources imply split. +bool is_model_splitted(ggml_cgraph * cgraph) { + // check the nodes of the model are used by the following nodes, through compare the node's use count and the count of nodes that use it as input. If does not match, return true, else return false. + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + int use_count = cgraph->use_counts[ggml_hash_find(&cgraph->visited_hash_set, node)]; + // TODO: this is a workround for the tests case from llama.cpp, fix should from the root cause in the future. + if ((cgraph->n_nodes <= 1 && use_count == 0) || + (cgraph->n_nodes <= 1 && node->op == GGML_OP_VIEW && use_count == 1 && node->src[0] != nullptr && + node->src[0]->op == GGML_OP_NONE)) { + return false; + } + if (cgraph->n_nodes == 1 && + (cgraph->nodes[0]->op == GGML_OP_TRANSPOSE || cgraph->nodes[0]->op == GGML_OP_PERMUTE)) { + return false; + } + int input_use_count = 0; + for (int j = 0; j < cgraph->n_nodes; j++) { + ggml_tensor * other_node = cgraph->nodes[j]; + for (int k = 0; k < GGML_MAX_SRC; k++) { + if (other_node->src[k] == node) { + input_use_count++; + } + } + } + if (use_count != input_use_count && node->op != GGML_OP_NONE) { + return true; + } + } + // if all nodes's src node's src is not come from the nodes in the model, we think the model is splitted. This is a complementary check for the above check, because for some special case like the output node is not used by any node, the use count and input use count are both 0, we can not determine whether the model is splitted or not just based on the first check. + auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, true); + std::set model_nodes(cgraph->nodes, cgraph->nodes + cgraph->n_nodes); + // leaf nodes + std::set model_leafs(cgraph->leafs, cgraph->leafs + cgraph->n_leafs); + for (int i = 0; i < cgraph->n_nodes; i++) { + ggml_tensor * node = cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + ggml_tensor * src = node->src[j]; + // the src is also not the model weights, we think the model is splitted. + // the src is also not in model leafs, we think the model is splitted. + if (src != nullptr && model_nodes.find(src) == model_nodes.end() && + model_weights.find(std::string(src->name)) == model_weights.end() && !model_leafs.empty() == false && + model_leafs.find(src) == model_leafs.end()) { + if (GgmlOvDecoder::is_inp_tok(src, node)) { + return false; + } + return true; + } + } + } + return false; +} + bool is_naive(ggml_cgraph * cgraph) { constexpr int naive_graph_size_threshold = 20; int count = 0; @@ -551,7 +717,7 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph, auto decoder = std::make_shared(cgraph, model_weights); auto input_model = std::make_shared(decoder); auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); - if (getenv("GGML_OPENVINO_DUMP_IR")) { + if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { ov::serialize(model, "IR_naive.xml"); } @@ -578,40 +744,92 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph, infer_request->set_input_tensor(i, input_tensor); } - auto ov_results = model->get_results(); - for (size_t i = 0; i < ov_results.size(); i++) { - auto * ggml_tensor = decoder->get_model_outputs().at(ov_results[i]->get_friendly_name()); - auto output_tensor = create_ov_output_tensor(decoder, infer_request, i, ggml_tensor); - infer_request->set_output_tensor(i, output_tensor); - } + // Use get_output_tensor + memcpy instead of set_output_tensor to avoid memory overwritten + // when i/o buffer overlaps, e.g. the cgraph is a single PERMUTE infer_request->infer(); + + auto ov_results = model->get_results(); + for (size_t i = 0; i < ov_results.size(); i++) { + auto output_tensor = infer_request->get_output_tensor(i); + auto * ggml_tensor = decoder->get_model_outputs().at(ov_results[i]->get_friendly_name()); + std::memcpy(ggml_tensor->data, output_tensor.data(), output_tensor.get_byte_size()); + } return GGML_STATUS_SUCCESS; } namespace { +template void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols, T zero_value = T{}) { + for (size_t i = 0; i < rows; ++i) { + size_t diag_col = std::min(i, cols - 1); + matrix[i * cols + diag_col] = zero_value; + } +} + +ov::Tensor make_contiguous_split_input_tensor(std::shared_ptr ggml_decoder, + const struct ggml_tensor * ggml_tensor, + const ov::Shape & input_shape) { + const size_t element_size = ggml_type_size(ggml_tensor->type); + const size_t block_size = ggml_blck_size(ggml_tensor->type); + + GGML_ASSERT(block_size == 1 && "non-contiguous split inputs must be plain element types"); + + const struct ggml_tensor * source_tensor = ggml_tensor->view_src != nullptr ? ggml_tensor->view_src : ggml_tensor; + const size_t source_offset = ggml_tensor->view_src != nullptr ? ggml_tensor->view_offs : 0; + + std::vector source_data(ggml_nbytes(source_tensor)); + ggml_backend_tensor_get(source_tensor, source_data.data(), 0, source_data.size()); + + ov::Tensor input_tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape); + auto * dst = static_cast(input_tensor.data()); + size_t dst_offset = 0; + + for (size_t i3 = 0; i3 < static_cast(ggml_tensor->ne[3]); ++i3) { + for (size_t i2 = 0; i2 < static_cast(ggml_tensor->ne[2]); ++i2) { + for (size_t i1 = 0; i1 < static_cast(ggml_tensor->ne[1]); ++i1) { + for (size_t i0 = 0; i0 < static_cast(ggml_tensor->ne[0]); ++i0) { + const size_t src_offset = source_offset + i3 * ggml_tensor->nb[3] + i2 * ggml_tensor->nb[2] + + i1 * ggml_tensor->nb[1] + i0 * ggml_tensor->nb[0]; + std::memcpy(dst + dst_offset, source_data.data() + src_offset, element_size); + dst_offset += element_size; + } + } + } + } + + return input_tensor; +} + ov::Tensor convert_ggml_input_to_ov(std::shared_ptr ggml_decoder, const std::string & name) { const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); - if (ggml_tensor->extra != nullptr) { - // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); + if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { + return *sliced; + } + + if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { auto * extra_base = static_cast(ggml_tensor->extra); - if (extra_base->type != ggml_openvino_extra_base::Type::TENSOR) { - throw std::runtime_error("ggml tensor extra is not of type TENSOR for input: " + name); + if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { + // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); + auto * tensor_extra = static_cast(extra_base); + return *tensor_extra->tensor; } - auto * tensor_extra = static_cast(extra_base); - return *tensor_extra->tensor; } // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); auto * input_data = ggml_tensor->data; ov::Shape input_shape; - if (ggml_tensor->op == GGML_OP_VIEW) { + if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { // This case is added to make test-backend-ops work input_shape = ggml_decoder->get_shape(ggml_tensor->view_src); } else { input_shape = ggml_decoder->get_shape(ggml_tensor); } + + if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { + return make_contiguous_split_input_tensor(ggml_decoder, ggml_tensor, input_shape); + } + auto input_tensor = ov::Tensor(ggml_decoder->get_ov_type(ggml_tensor), input_shape, input_data); return input_tensor; } @@ -660,6 +878,14 @@ ov::Tensor get_ov_input_tensor_static_decode(std::shared_ptr ggml if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + std::vector padded_data = + pad_input(ggml_tensor, 1, context_size, GGML_FP32_TO_FP16(-INFINITY)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, 1, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } + std::vector padded_data = pad_input(ggml_tensor, 1, context_size, -INFINITY); ov::Tensor input_tensor(ov::element::f32, ov::Shape{1, 1, 1, context_size}); auto * data_ptr = input_tensor.data(); @@ -728,9 +954,20 @@ ov::Tensor get_ov_input_tensor_static_prefill(std::shared_ptr ggm if (GgmlOvDecoder::is_inp_mask(ggml_tensor, op)) { size_t cols = ggml_tensor->ne[0]; size_t rows = ggml_tensor->ne[1]; - float * ggml_data = (float *) ggml_tensor->data + chunk_index * chunk_size * cols; size_t chunk_valid_rows = std::min(chunk_size, rows - chunk_index * chunk_size); size_t context_size = ggml_decoder->get_ctx_size(); + if (ggml_tensor->type == GGML_TYPE_F16) { + const auto * ggml_data = + static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; + std::vector padded_data = pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, + context_size, GGML_FP32_TO_FP16(-INFINITY)); + set_zero_diagonal(padded_data, chunk_size, context_size, GGML_FP32_TO_FP16(0.0f)); + ov::Tensor input_tensor(ov::element::f16, ov::Shape{1, 1, chunk_size, context_size}); + std::memcpy(input_tensor.data(), padded_data.data(), padded_data.size() * sizeof(ggml_fp16_t)); + return input_tensor; + } + + const auto * ggml_data = static_cast(ggml_tensor->data) + chunk_index * chunk_size * cols; std::vector padded_data = pad_input(ggml_data, chunk_valid_rows, cols, chunk_size, context_size, -INFINITY); set_zero_diagonal(padded_data, chunk_size, context_size); @@ -753,6 +990,65 @@ size_t checksum(const void * data, size_t size) { return sum; } +bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path) { + if (tensor == nullptr || tensor->data == nullptr) { + return false; + } + + std::ofstream out(file_path); + if (!out.is_open()) { + return false; + } + + const size_t n = ggml_nelements(tensor); + out << "name: " << tensor->name << ", type: " << ggml_type_name(tensor->type) << ", shape: [" << tensor->ne[0] + << ", " << tensor->ne[1] << ", " << tensor->ne[2] << ", " << tensor->ne[3] << "]" << ", elements: " << n + << ", data:" << '\n'; + + switch (tensor->type) { + case GGML_TYPE_F32: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << data[i] << '\n'; + } + break; + } + case GGML_TYPE_F16: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << ggml_fp16_to_fp32(data[i]) << '\n'; + } + break; + } + case GGML_TYPE_BF16: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << ggml_bf16_to_fp32(data[i]) << '\n'; + } + break; + } + case GGML_TYPE_I32: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << data[i] << '\n'; + } + break; + } + case GGML_TYPE_I64: { + const auto * data = static_cast(tensor->data); + for (size_t i = 0; i < n; ++i) { + out << data[i] << '\n'; + } + break; + } + default: + out << "unsupported tensor type for text dump" << '\n'; + return false; + } + + return true; +} + void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor) { std::cout << "Input name: " << name << ", Input shape: " << tensor.get_shape() << ", Address: " << tensor.data() << std::endl; @@ -849,13 +1145,6 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso } } -void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols) { - for (size_t i = 0; i < rows; ++i) { - size_t diag_col = std::min(i, cols - 1); - matrix[i * cols + diag_col] = 0.0f; - } -} - const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; ++i) { auto * op = cgraph->nodes[i]; diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 2c72e33c3..c2c7b7cda 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -1,4 +1,3 @@ -#include "ggml-backend-impl.h" #include "ggml-decoder.h" #include "ggml-impl.h" @@ -45,6 +44,7 @@ struct graph_key_hash { struct decoder_runtime_ctx { decoder_runtime_ctx(std::shared_ptr mutex) : mutex(std::move(mutex)) {} + std::shared_ptr mutex; std::shared_ptr ptr; }; @@ -64,11 +64,7 @@ struct ov_runtime_context { std::map kv_state_input_name_map; std::atomic backend_count; - ov_runtime_context() : - device("CPU"), - stateful(false), - stateful_kv_size(0), - backend_count(0) {} + ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0), backend_count(0) {} void clear_caches() { std::lock_guard lock(ctx_mutex); @@ -87,6 +83,8 @@ enum ggml_status ov_graph_compute_static(struct ggml_cgraph * cgraph, std::share size_t checksum(const void * data, size_t size); +bool save_ggml_tensor_data_to_txt(const ggml_tensor * tensor, const std::string & file_path); + void print_input_tensor_info(const std::string & name, const ov::Tensor & tensor); void print_output_tensor_info(const std::string & name, const ov::Tensor & tensor, const void * output_dst); @@ -117,8 +115,6 @@ std::vector pad_input(const ggml_tensor * tensor, size_t padded_rows, size_t padded_rows, padded_cols, pad_value); } -void set_zero_diagonal(std::vector & matrix, size_t rows, size_t cols); - const ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph); bool get_is_prefill(const ggml_tensor * inp_pos); @@ -137,6 +133,13 @@ ov::Tensor create_ov_output_tensor(std::shared_ptr ggml_decoder, bool is_naive(struct ggml_cgraph * cgraph); +/** + * @brief Heuristically checks whether the given computation graph is a split-model fragment. + * @param cgraph Pointer to the GGML computation graph to analyze. + * @return true if the graph is identified as split; otherwise false. + */ +bool is_model_splitted(struct ggml_cgraph * cgraph); + enum ggml_status naive_compute(struct ggml_cgraph * cgraph, ov::Core & core, const std::string & device, From cda63856b804af3aae3e35c1da3eacaeb4bfeb0a Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Tue, 16 Jun 2026 23:19:11 -0700 Subject: [PATCH 009/292] common: update logging to enforce max_capacity and optimize queue resizing (#24490) * common: update logging to enforce max_capacity and optimize queue resizing logic * common/log: remove queue expansion logic --- common/log.cpp | 158 +++++++++++++++++++++++++------------------------ 1 file changed, 82 insertions(+), 76 deletions(-) diff --git a/common/log.cpp b/common/log.cpp index bd62616d8..2d1e74ad1 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -11,8 +11,13 @@ #include #include #include +#include #if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif # include # include # define isatty _isatty @@ -62,16 +67,15 @@ static const char* g_col[] = { }; struct common_log_entry { - enum ggml_log_level level; - - bool prefix; - - int64_t timestamp; + enum ggml_log_level level {GGML_LOG_LEVEL_INFO}; std::vector msg; - // signals the worker thread to stop - bool is_end; + int64_t timestamp { 0 }; + bool is_end { false }; // signals the worker thread to stop + bool prefix { false }; + + common_log_entry(size_t size = 256) : msg(size) { } void print(FILE * file = nullptr) const { FILE * fcur = file; @@ -122,22 +126,15 @@ struct common_log_entry { }; struct common_log { - // default capacity - will be expanded if needed - common_log() : common_log(256) {} - - common_log(size_t capacity) { - file = nullptr; - prefix = false; + // default capacity + common_log(size_t capacity = 512) { + file = nullptr; + prefix = false; timestamps = false; - running = false; - t_start = t_us(); - - // initial message size - will be expanded if longer messages arrive - entries.resize(capacity); - for (auto & entry : entries) { - entry.msg.resize(256); - } + running = false; + t_start = t_us(); + queue.resize(capacity, common_log_entry(256)); head = 0; tail = 0; @@ -152,9 +149,10 @@ struct common_log { } private: - std::mutex mtx; - std::thread thrd; - std::condition_variable cv; + std::mutex mtx; + std::thread thrd; + std::condition_variable cv_new; // new entry + std::condition_variable cv_full; // wait on full FILE * file; @@ -164,24 +162,53 @@ private: int64_t t_start; - // ring buffer of entries - std::vector entries; + // queue of entries + std::vector queue; size_t head; size_t tail; - // worker thread copies into this - common_log_entry cur; + bool print_entry(const common_log_entry & e) const { + if (e.is_end) return true; + + e.print(); + if (file) { + e.print(file); + } + return false; + } + + bool flush_queue(size_t start_head, size_t end_tail, size_t & out_head) const { + bool stop = false; + size_t h = start_head; + while (h != end_tail && !stop) { + stop = print_entry(queue[h]); + h = (h + 1) % queue.size(); + } + out_head = h; + return stop; + } public: + bool is_full() const { + return ((tail + 1) % queue.size()) == head; + } + + bool is_empty() const { + return head == tail; + } + void add(enum ggml_log_level level, const char * fmt, va_list args) { - std::lock_guard lock(mtx); + std::unique_lock lock(mtx); + + // block if the queue is full + cv_full.wait(lock, [this]() { return !running || !is_full(); }); if (!running) { // discard messages while the worker thread is paused return; } - auto & entry = entries[tail]; + auto & entry = queue[tail]; { // cannot use args twice, so make a copy in case we need to expand the buffer @@ -216,38 +243,16 @@ public: va_end(args_copy); } - entry.level = level; - entry.prefix = prefix; + entry.is_end = false; + entry.level = level; + entry.prefix = prefix; entry.timestamp = 0; if (timestamps) { entry.timestamp = t_us() - t_start; } - entry.is_end = false; - tail = (tail + 1) % entries.size(); - if (tail == head) { - // expand the buffer - std::vector new_entries(2*entries.size()); - - size_t new_tail = 0; - - do { - new_entries[new_tail] = std::move(entries[head]); - - head = (head + 1) % entries.size(); - new_tail = (new_tail + 1); - } while (head != tail); - - head = 0; - tail = new_tail; - - for (size_t i = tail; i < new_entries.size(); i++) { - new_entries[i].msg.resize(256); - } - - entries = std::move(new_entries); - } - cv.notify_one(); + tail = (tail + 1) % queue.size(); + cv_new.notify_one(); } void resume() { @@ -261,23 +266,24 @@ public: thrd = std::thread([this]() { while (true) { - { - std::unique_lock lock(mtx); - cv.wait(lock, [this]() { return head != tail; }); - cur = entries[head]; + std::unique_lock lock(mtx); + cv_new.wait(lock, [this]() { return !is_empty(); }); - head = (head + 1) % entries.size(); - } + size_t cached_head = head; + size_t cached_tail = tail; - if (cur.is_end) { + lock.unlock(); // drop the lock during flush + + size_t next_head; + bool stop = flush_queue(cached_head, cached_tail, next_head); + + lock.lock(); + head = next_head; + cv_full.notify_all(); + + if (stop) { break; } - - cur.print(); // stdout and stderr - - if (file) { - cur.print(file); - } } }); } @@ -293,13 +299,13 @@ public: running = false; // push an entry to signal the worker thread to stop - { - auto & entry = entries[tail]; - entry.is_end = true; + auto & entry = queue[tail]; + entry.is_end = true; + tail = (tail + 1) % queue.size(); - tail = (tail + 1) % entries.size(); - } - cv.notify_one(); + // wakeup everyone + cv_new.notify_one(); + cv_full.notify_all(); } thrd.join(); From 51571722aa88a8b24d8a274c5fa603b6c3ee9c36 Mon Sep 17 00:00:00 2001 From: lhez Date: Tue, 16 Jun 2026 23:21:26 -0700 Subject: [PATCH 010/292] opencl: optimize mul_mat_f16_f32_l4 for decode (#24504) --- ggml/src/ggml-opencl/ggml-opencl.cpp | 58 +++- .../ggml-opencl/kernels/mul_mv_f16_f32_l4.cl | 296 ++++++++++++++++++ 2 files changed, 348 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index ca2002424..5ad8d76fa 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -564,6 +564,9 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mat_f16_f32_1row; cl_kernel kernel_mul_mat_f16_f32; cl_kernel kernel_mul_mat_f16_f32_l4; + cl_kernel kernel_mul_mat_f16_f32_l4_dr; + cl_kernel kernel_mul_mat_f16_f32_l4_dr_ls; + cl_kernel kernel_mul_mat_f16_f32_l4_dr_lq; cl_kernel kernel_mul_mat_f16_f32_tiled; cl_kernel kernel_adreno_xmem_pack_src_f32; cl_kernel kernel_adreno_xmem_prepack_weight_f16; @@ -1787,6 +1790,11 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx->context, backend_ctx->device, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4 = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr", &err), err)); + if (backend_ctx->gpu_family == ADRENO) { + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_ls", &err), err)); + CL_CHECK((backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq = clCreateKernel(backend_ctx->program_mul_mv_f16_f32_l4, "kernel_mul_mat_f16_f32_l4_dr_lq", &err), err)); + } GGML_LOG_CONT("."); } @@ -14570,11 +14578,31 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co } if (src1t == GGML_TYPE_F32) { + // heuristic for packing more work for Adreno + const bool adreno_use_lane_split = + backend_ctx->gpu_family == ADRENO && + ne11 == 1 && + ne01 >= 8 && + ne00 % 4 == 0 && + r3 == 1 && r2 >= 1 && r2 <= 8 && + (ne12 % r2) == 0; + if (ne11 * ne12 < 4) { kernel = backend_ctx->kernel_mul_mat_f16_f32_1row; + } else if (adreno_use_lane_split && ne00 >= 64 && ne00 <= 128) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq; + nrows = 1; + } else if (adreno_use_lane_split && r2 >= 2 && ne00 > 128 && ne00 <= 256) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls; + nrows = 1; } else if (ne00 >= 128 && ne01 >= 8 && ne00%4 == 0) { - kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; - nrows = ne11; + if (ne11 == 1) { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4_dr; + nrows = 1; // not used by this kernel + } else { + kernel = backend_ctx->kernel_mul_mat_f16_f32_l4; + nrows = ne11; + } } else { kernel = backend_ctx->kernel_mul_mat_f16_f32; nrows = 4; @@ -15353,12 +15381,30 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { - int64_t ny = (ne11 + nrows - 1)/nrows; + if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr) { + const int NDST_DR = 4; + size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, NDST_DR)*nth0, (size_t)nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; - size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr_ls) { + size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, 2)*nth0, (size_t)nth1, (size_t)ne02*ne03}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; - backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else if (kernel == backend_ctx->kernel_mul_mat_f16_f32_l4_dr_lq) { + size_t global_work_size[] = {(size_t)CEIL_DIV(ne01, 4)*nth0, (size_t)nth1, (size_t)ne02*ne03}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } else { + int64_t ny = (ne11 + nrows - 1)/nrows; + + size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + } } } diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl index cdf8197c4..a639ec664 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f16_f32_l4.cl @@ -82,3 +82,299 @@ kernel void kernel_mul_mat_f16_f32_l4( } } } + +// Each subgroup produces DR_NDST outputs, assumes ne11 == 1 +#define MUL_MAT_F16_F32_L4_DR_NDST 4 + +#ifdef ADRENO_GPU +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mat_f16_f32_l4_dr( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int r0_base = get_group_id(0) * MUL_MAT_F16_F32_L4_DR_NDST; + const int im = get_group_id(2); + + const int i12 = im % ne12; + const int i13 = im / ne12; + + // assume ne11 == 1 + const ulong offset_src1 = i12*nb12 + i13*nb13; + global float4 * y4 = (global float4 *)(src1 + offset_src1); + + global half4 * x4[MUL_MAT_F16_F32_L4_DR_NDST]; + float sumf[MUL_MAT_F16_F32_L4_DR_NDST]; + + const ulong k_head_off = (i12/r2)*nb02 + (i13/r3)*nb03; + + #pragma unroll + for (int n = 0; n < MUL_MAT_F16_F32_L4_DR_NDST; ++n) { + int r0 = r0_base + n; + int r0c = r0 < ne01 ? r0 : 0; + ulong off = (ulong)r0c*nb01 + k_head_off; + x4[n] = (global half4 *)(src0 + off); + sumf[n] = 0.0f; + } + + const int n_chunks = ne00 / 4; + const int sg_size = get_max_sub_group_size(); + const int lid = get_sub_group_local_id(); + + for (int i = lid; i < n_chunks; i += sg_size) { + float4 q = y4[i]; + #pragma unroll + for (int n = 0; n < MUL_MAT_F16_F32_L4_DR_NDST; ++n) { + float4 k = convert_float4(x4[n][i]); + sumf[n] = mad(k.s0, q.s0, sumf[n]); + sumf[n] = mad(k.s1, q.s1, sumf[n]); + sumf[n] = mad(k.s2, q.s2, sumf[n]); + sumf[n] = mad(k.s3, q.s3, sumf[n]); + } + } + + #pragma unroll + for (int n = 0; n < MUL_MAT_F16_F32_L4_DR_NDST; ++n) { + float reduced = sub_group_reduce_add(sumf[n]); + int r0 = r0_base + n; + if (lid == 0 && r0 < ne01) { + dst[im*ne1*ne0 + r0] = reduced; + } + } +} + +// Kernels for decoding, Adreno only for now +#define MUL_MAT_F16_F32_L4_DR_LS_R2_MAX 8 + +#ifdef ADRENO_GPU +#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable +#define sub_group_shuffle_xor(val, mask) qcom_sub_group_shuffle_xor((val), (mask), CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.0f) + +REQD_SUBGROUP_SIZE_64 +kernel void kernel_mul_mat_f16_f32_l4_dr_ls( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int r0_base = get_group_id(0) * 2; + const int kv_grp = get_group_id(2); // KV head group; im = kv_grp*r2 + q + + const int i12_kv = kv_grp % ne02; + const int i13_kv = kv_grp / ne02; + + const int lid = get_sub_group_local_id(); + const int subhalf = lid >> 5; // 0 or 1 (which K row in the WG) + const int intra = lid & 31; // 0..31 (lane within the half) + + const int r0 = r0_base + subhalf; + const int r0c = r0 < ne01 ? r0 : 0; // clamp OOB to row 0; skip write below + + // K row pointer for this lane (one K row per half-wave). + const ulong k_off = (ulong)r0c*nb01 + (ulong)i12_kv*nb02 + (ulong)i13_kv*nb03; + global half4 * x4 = (global half4 *)(src0 + k_off); + + global float4 * y4[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + const int i12_q = i12_kv*r2 + q; + const ulong q_off = (ulong)i12_q*nb12 + (ulong)i13_kv*nb13; + y4[q] = (global float4 *)(src1 + q_off); + } + + float partial[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + partial[q] = 0.0f; + } + + const int n_chunks = ne00 / 4; + + for (int i = intra; i < n_chunks; i += 32) { + float4 k = convert_float4(x4[i]); + + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + float4 v = y4[q][i]; + partial[q] = mad(k.s0, v.s0, partial[q]); + partial[q] = mad(k.s1, v.s1, partial[q]); + partial[q] = mad(k.s2, v.s2, partial[q]); + partial[q] = mad(k.s3, v.s3, partial[q]); + } + } + } + + // half-wave reduction + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + partial[q] += sub_group_shuffle_xor(partial[q], 1u); + partial[q] += sub_group_shuffle_xor(partial[q], 2u); + partial[q] += sub_group_shuffle_xor(partial[q], 4u); + partial[q] += sub_group_shuffle_xor(partial[q], 8u); + partial[q] += sub_group_shuffle_xor(partial[q], 16u); + } + } + + if (intra == 0 && r0 < ne01) { + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + const int im = i12_kv*r2 + q + i13_kv*ne12; + dst[im*ne1*ne0 + r0] = partial[q]; + } + } + } +} + +REQD_SUBGROUP_SIZE_64 +kernel void kernel_mul_mat_f16_f32_l4_dr_lq( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = (global char*)((global char*)src0 + offset0); + src1 = (global char*)((global char*)src1 + offset1); + dst = (global float*)((global char*)dst + offsetd); + + const int r0_base = get_group_id(0) * 4; + const int kv_grp = get_group_id(2); + + const int i12_kv = kv_grp % ne02; + const int i13_kv = kv_grp / ne02; + + const int lid = get_sub_group_local_id(); + const int subq = lid >> 4; // 0..3 (which K row) + const int intra = lid & 15; // 0..15 (lane within quarter) + + const int r0 = r0_base + subq; + const int r0c = r0 < ne01 ? r0 : 0; + + const ulong k_off = (ulong)r0c*nb01 + (ulong)i12_kv*nb02 + (ulong)i13_kv*nb03; + global half4 * x4 = (global half4 *)(src0 + k_off); + + global float4 * y4[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + const int i12_q = i12_kv*r2 + q; + const ulong q_off = (ulong)i12_q*nb12 + (ulong)i13_kv*nb13; + y4[q] = (global float4 *)(src1 + q_off); + } + + float partial[MUL_MAT_F16_F32_L4_DR_LS_R2_MAX]; + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + partial[q] = 0.0f; + } + + const int n_chunks = ne00 / 4; + + for (int i = intra; i < n_chunks; i += 16) { + float4 k = convert_float4(x4[i]); + + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + float4 v = y4[q][i]; + partial[q] = mad(k.s0, v.s0, partial[q]); + partial[q] = mad(k.s1, v.s1, partial[q]); + partial[q] = mad(k.s2, v.s2, partial[q]); + partial[q] = mad(k.s3, v.s3, partial[q]); + } + } + } + + // quarter-wave reduction + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + partial[q] += sub_group_shuffle_xor(partial[q], 1u); + partial[q] += sub_group_shuffle_xor(partial[q], 2u); + partial[q] += sub_group_shuffle_xor(partial[q], 4u); + partial[q] += sub_group_shuffle_xor(partial[q], 8u); + } + } + + if (intra == 0 && r0 < ne01) { + #pragma unroll + for (int q = 0; q < MUL_MAT_F16_F32_L4_DR_LS_R2_MAX; ++q) { + if (q < r2) { + const int im = i12_kv*r2 + q + i13_kv*ne12; + dst[im*ne1*ne0 + r0] = partial[q]; + } + } + } +} +#endif // ADRENO_GPU From bae36efa30012d5dbaf842f893c5d034c0ba0157 Mon Sep 17 00:00:00 2001 From: Harapan Rachman Date: Wed, 17 Jun 2026 13:26:30 +0700 Subject: [PATCH 011/292] =?UTF-8?q?UI=20:=20fix=20SSE=20transport=20detect?= =?UTF-8?q?ion=20and=20routing=20through=20CORS=20proxy.=20Assi=E2=80=A6?= =?UTF-8?q?=20(#24500)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * UI : fix SSE transport detection and routing through CORS proxy. Assisted-by: Antigravity * ui : replace magic strings with constants in MCP transport handling --- tools/ui/src/lib/constants/mcp.ts | 5 ++ tools/ui/src/lib/services/mcp.service.ts | 59 +++++++++++++++++++++++- tools/ui/src/lib/utils/mcp.ts | 23 +++++++-- 3 files changed, 82 insertions(+), 5 deletions(-) diff --git a/tools/ui/src/lib/constants/mcp.ts b/tools/ui/src/lib/constants/mcp.ts index 918eb9f94..5b11f989e 100644 --- a/tools/ui/src/lib/constants/mcp.ts +++ b/tools/ui/src/lib/constants/mcp.ts @@ -84,3 +84,8 @@ export const MCP_TRANSPORT_ICONS: Record = { [MCPTransportType.STREAMABLE_HTTP]: Globe, [MCPTransportType.SSE]: Radio }; + +/** Standard SSE endpoint path indicators */ +export const MCP_SSE_ENDPOINT = '/sse'; +export const MCP_SSE_ENDPOINT_SLASH = '/sse/'; +export const MCP_SSE_ENDPOINT_QUERY = '/sse?'; diff --git a/tools/ui/src/lib/services/mcp.service.ts b/tools/ui/src/lib/services/mcp.service.ts index d596381aa..0aa58dc5d 100644 --- a/tools/ui/src/lib/services/mcp.service.ts +++ b/tools/ui/src/lib/services/mcp.service.ts @@ -16,7 +16,8 @@ import { DEFAULT_MCP_CONFIG, DEFAULT_CLIENT_VERSION, DEFAULT_IMAGE_MIME_TYPE, - MCP_PARTIAL_REDACT_HEADERS + MCP_PARTIAL_REDACT_HEADERS, + CORS_PROXY_ENDPOINT } from '$lib/constants'; import { MCPConnectionPhase, @@ -236,6 +237,36 @@ export class MCPService { return { fetch: async (input, init) => { + if (useProxy && typeof window !== 'undefined') { + let requestUrlStr = ''; + if (typeof input === 'string') { + requestUrlStr = input; + } else if (input instanceof URL) { + requestUrlStr = input.href; + } + + if (requestUrlStr) { + const parsedRequestUrl = new URL(requestUrlStr, window.location.origin); + if ( + parsedRequestUrl.origin === window.location.origin && + !parsedRequestUrl.pathname.includes(CORS_PROXY_ENDPOINT) + ) { + const originalConfigUrl = new URL(config.url); + const realTargetUrl = new URL( + parsedRequestUrl.pathname + parsedRequestUrl.search, + originalConfigUrl.origin + ); + const proxiedUrl = buildProxiedUrl(realTargetUrl.href); + + if (typeof input === 'string') { + input = proxiedUrl.href; + } else if (input instanceof URL) { + input = proxiedUrl; + } + } + } + } + const startedAt = performance.now(); const requestHeaders = new Headers(baseInit.headers); @@ -403,6 +434,32 @@ export class MCPService { }; } + if (config.transport === MCPTransportType.SSE) { + const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); + const { fetch: diagnosticFetch, disable: stopPhaseLogging } = this.createDiagnosticFetch( + serverName, + config, + requestInit, + url, + useProxy, + onLog + ); + + if (import.meta.env.DEV && import.meta.env.VITE_DEBUG) { + console.log(`[MCPService] Creating SSE transport for ${url.href}`); + } + + return { + transport: new SSEClientTransport(url, { + requestInit, + fetch: diagnosticFetch, + eventSourceInit: { fetch: diagnosticFetch } + }), + type: MCPTransportType.SSE, + stopPhaseLogging + }; + } + const url = useProxy ? buildProxiedUrl(config.url) : new URL(config.url); const { fetch: diagnosticFetch, disable: stopPhaseLogging } = this.createDiagnosticFetch( serverName, diff --git a/tools/ui/src/lib/utils/mcp.ts b/tools/ui/src/lib/utils/mcp.ts index 05fe90048..7ce7f19fe 100644 --- a/tools/ui/src/lib/utils/mcp.ts +++ b/tools/ui/src/lib/utils/mcp.ts @@ -19,7 +19,10 @@ import { DISPLAY_NAME_SEPARATOR_REGEX, PATH_SEPARATOR, RESOURCE_TEXT_CONTENT_SEPARATOR, - DEFAULT_RESOURCE_FILENAME + DEFAULT_RESOURCE_FILENAME, + MCP_SSE_ENDPOINT, + MCP_SSE_ENDPOINT_SLASH, + MCP_SSE_ENDPOINT_QUERY } from '$lib/constants'; import { Database, @@ -41,10 +44,22 @@ import type { MimeTypeUnion } from '$lib/types/common'; export function detectMcpTransportFromUrl(url: string): MCPTransportType { const normalized = url.trim().toLowerCase(); - return normalized.startsWith(UrlProtocol.WEBSOCKET) || + if ( + normalized.startsWith(UrlProtocol.WEBSOCKET) || normalized.startsWith(UrlProtocol.WEBSOCKET_SECURE) - ? MCPTransportType.WEBSOCKET - : MCPTransportType.STREAMABLE_HTTP; + ) { + return MCPTransportType.WEBSOCKET; + } + + if ( + normalized.endsWith(MCP_SSE_ENDPOINT) || + normalized.endsWith(MCP_SSE_ENDPOINT_SLASH) || + normalized.includes(MCP_SSE_ENDPOINT_QUERY) + ) { + return MCPTransportType.SSE; + } + + return MCPTransportType.STREAMABLE_HTTP; } /** From d5376cf5d711b9be87232b25fea82fe62d607400 Mon Sep 17 00:00:00 2001 From: kononnable Date: Wed, 17 Jun 2026 09:43:45 +0200 Subject: [PATCH 012/292] ci: fix vulkan docker images (#24595) * Update vulkan-shaders-gen.cpp * Update vulkan-shaders-gen.cpp add comment describing code change intention * Update vulkan-shaders-gen.cpp fix potential UB --- ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 27d088e0e..ca6b44431 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -407,7 +407,7 @@ std::map merge_maps(const std::map> compiles; +static std::deque> compiles; void string_to_spv(std::string name, const std::string& source, const std::map& defines, bool fp16 = true, bool coopmat = false, bool coopmat2 = false, bool f16acc = false, const std::string& suffix = "") { name = name + (f16acc ? "_f16acc" : "") + (coopmat ? "_cm1" : "") + (coopmat2 ? "_cm2" : (fp16 ? "" : "_fp32")) + suffix; std::string out_path = join_paths(output_dir, name + ".spv"); @@ -426,6 +426,11 @@ void string_to_spv(std::string name, const std::string& source, const std::map Date: Wed, 17 Jun 2026 05:31:16 -0300 Subject: [PATCH 013/292] sd: sync with master-707-5a34bc7 (#2274) * sd: sync with master-692-9b0fceb * sd: sync with master-694-276025e * sd: sync with master-697-5db680c * sd: sync to master-700-c2df4e1 * sd: sync with master-704-6e66a1a * sd: sync with master-707-5a34bc7 --- Makefile | 2 +- otherarch/sdcpp/examples/cli/main.cpp | 23 +- otherarch/sdcpp/examples/common/common.cpp | 167 +- otherarch/sdcpp/examples/common/common.h | 12 +- otherarch/sdcpp/include/stable-diffusion.h | 28 +- otherarch/sdcpp/sdtype_adapter.cpp | 7 +- .../sdcpp/src/conditioning/conditioner.hpp | 430 +++--- otherarch/sdcpp/src/core/ggml_extend.hpp | 1009 ++++-------- .../sdcpp/src/core/ggml_extend_backend.cpp | 93 +- .../sdcpp/src/core/ggml_extend_backend.h | 7 +- otherarch/sdcpp/src/core/ggml_graph_cut.cpp | 247 ++- otherarch/sdcpp/src/core/ggml_graph_cut.h | 13 +- otherarch/sdcpp/src/core/layer_registry.cpp | 132 -- otherarch/sdcpp/src/core/layer_registry.h | 50 - otherarch/sdcpp/src/core/util.cpp | 14 +- .../src/extensions/generation_extension.h | 36 +- .../src/extensions/photomaker_extension.cpp | 89 +- .../sdcpp/src/extensions/pulid_extension.cpp | 123 ++ otherarch/sdcpp/src/model.h | 1 + otherarch/sdcpp/src/model/adapter/lora.hpp | 106 +- otherarch/sdcpp/src/model/adapter/pmid.hpp | 56 +- otherarch/sdcpp/src/model/adapter/pulid.hpp | 76 + otherarch/sdcpp/src/model/common/block.hpp | 17 +- otherarch/sdcpp/src/model/diffusion/anima.hpp | 10 +- .../sdcpp/src/model/diffusion/control.hpp | 130 +- .../sdcpp/src/model/diffusion/ernie_image.hpp | 10 +- otherarch/sdcpp/src/model/diffusion/flux.hpp | 158 +- .../sdcpp/src/model/diffusion/hidream_o1.hpp | 55 +- .../sdcpp/src/model/diffusion/ideogram4.hpp | 40 +- otherarch/sdcpp/src/model/diffusion/lens.hpp | 10 +- otherarch/sdcpp/src/model/diffusion/ltxv.hpp | 34 +- otherarch/sdcpp/src/model/diffusion/mmdit.hpp | 35 +- otherarch/sdcpp/src/model/diffusion/model.hpp | 11 +- otherarch/sdcpp/src/model/diffusion/pid.hpp | 8 +- .../sdcpp/src/model/diffusion/qwen_image.hpp | 42 +- otherarch/sdcpp/src/model/diffusion/unet.hpp | 8 +- otherarch/sdcpp/src/model/diffusion/wan.hpp | 39 +- .../sdcpp/src/model/diffusion/z_image.hpp | 38 +- otherarch/sdcpp/src/model/te/clip.hpp | 19 +- otherarch/sdcpp/src/model/te/llm.hpp | 67 +- otherarch/sdcpp/src/model/te/t5.hpp | 57 +- otherarch/sdcpp/src/model/upscaler/esrgan.hpp | 314 ++-- .../model/upscaler/ltx_latent_upscaler.hpp | 279 ++-- .../sdcpp/src/model/vae/auto_encoder_kl.hpp | 32 +- .../sdcpp/src/model/vae/ltx_audio_vae.hpp | 43 +- otherarch/sdcpp/src/model/vae/ltx_vae.hpp | 52 +- otherarch/sdcpp/src/model/vae/tae.hpp | 34 +- otherarch/sdcpp/src/model/vae/vae.hpp | 21 +- otherarch/sdcpp/src/model/vae/wan_vae.hpp | 45 +- otherarch/sdcpp/src/model_loader.cpp | 209 ++- otherarch/sdcpp/src/model_loader.h | 15 +- otherarch/sdcpp/src/model_manager.cpp | 939 ++++++++++++ otherarch/sdcpp/src/model_manager.h | 167 ++ otherarch/sdcpp/src/name_conversion.cpp | 39 + otherarch/sdcpp/src/runtime/guidance.cpp | 4 +- otherarch/sdcpp/src/stable-diffusion.cpp | 1356 +++++++++-------- otherarch/sdcpp/src/upscaler.cpp | 57 +- otherarch/sdcpp/src/upscaler.h | 4 +- otherarch/sdcpp/src/weight_manager.h | 15 + 59 files changed, 4125 insertions(+), 3009 deletions(-) delete mode 100644 otherarch/sdcpp/src/core/layer_registry.cpp delete mode 100644 otherarch/sdcpp/src/core/layer_registry.h create mode 100644 otherarch/sdcpp/src/extensions/pulid_extension.cpp create mode 100644 otherarch/sdcpp/src/model/adapter/pulid.hpp create mode 100644 otherarch/sdcpp/src/model_manager.cpp create mode 100644 otherarch/sdcpp/src/model_manager.h create mode 100644 otherarch/sdcpp/src/weight_manager.h diff --git a/Makefile b/Makefile index 2a80ef65f..4f2420a35 100644 --- a/Makefile +++ b/Makefile @@ -699,7 +699,7 @@ llama-impl.o: src/llama-impl.cpp src/llama-impl.h budget.o: common/reasoning-budget.cpp common/reasoning-budget.h $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_registry.cpp src/core/layer_registry.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp diff --git a/otherarch/sdcpp/examples/cli/main.cpp b/otherarch/sdcpp/examples/cli/main.cpp index decee0a94..bb5d6862c 100644 --- a/otherarch/sdcpp/examples/cli/main.cpp +++ b/otherarch/sdcpp/examples/cli/main.cpp @@ -623,8 +623,6 @@ int main(int argc, const char* argv[]) { } } - bool vae_decode_only = true; - auto load_image_and_update_size = [&](const std::string& path, SDImageOwner& image, bool resize_image = true, @@ -646,21 +644,18 @@ int main(int argc, const char* argv[]) { }; if (gen_params.init_image_path.size() > 0) { - vae_decode_only = false; if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) { return 1; } } if (gen_params.end_image_path.size() > 0) { - vae_decode_only = false; if (!load_image_and_update_size(gen_params.end_image_path, gen_params.end_image)) { return 1; } } if (gen_params.ref_image_paths.size() > 0) { - vae_decode_only = false; gen_params.ref_images.clear(); for (auto& path : gen_params.ref_image_paths) { SDImageOwner ref_image({0, 0, 3, nullptr}); @@ -735,18 +730,7 @@ int main(int argc, const char* argv[]) { } } - if (cli_params.mode == VID_GEN) { - vae_decode_only = false; - } - - if (gen_params.hires_enabled && - (gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_MODEL || - gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_LANCZOS || - gen_params.resolved_hires_upscaler == SD_HIRES_UPSCALER_NEAREST)) { - vae_decode_only = false; - } - - sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(vae_decode_only, true, cli_params.taesd_preview); + sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t(cli_params.taesd_preview); SDImageVec results; int num_results = 0; @@ -798,12 +782,11 @@ int main(int argc, const char* argv[]) { int upscale_factor = 4; // unused for RealESRGAN_x4plus_anime_6B.pth if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) { UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(), - ctx_params.offload_params_to_cpu, ctx_params.diffusion_conv_direct, ctx_params.n_threads, gen_params.upscale_tile_size, - ctx_params.backend.c_str(), - ctx_params.params_backend.c_str())); + sd_ctx_params.backend, + sd_ctx_params.params_backend)); if (upscaler_ctx == nullptr) { LOG_ERROR("new_upscaler_ctx failed"); diff --git a/otherarch/sdcpp/examples/common/common.cpp b/otherarch/sdcpp/examples/common/common.cpp index 3ae5faba7..dd5d35055 100644 --- a/otherarch/sdcpp/examples/common/common.cpp +++ b/otherarch/sdcpp/examples/common/common.cpp @@ -51,6 +51,10 @@ static sd_vae_format_t str_to_vae_format(const std::string& value) { return SD_VAE_FORMAT_COUNT; } +static void prepend_backend_assignment(std::string& spec, const char* assignment) { + spec = spec.empty() ? assignment : std::string(assignment) + "," + spec; +} + #if defined(_WIN32) static std::string utf16_to_utf8(const std::wstring& wstr) { if (wstr.empty()) @@ -411,6 +415,10 @@ ArgOptions SDContextParams::get_options() { "--photo-maker", "path to PHOTOMAKER model", &photo_maker_path}, + {"", + "--pulid-weights", + "path to PuLID Flux weights", + &pulid_weights_path}, {"", "--upscale-model", "path to esrgan model.", @@ -421,8 +429,16 @@ ArgOptions SDContextParams::get_options() { &backend}, {"", "--params-backend", - "parameter backend assignment, e.g. cpu or diffusion=cpu,clip=cpu", + "parameter backend assignment, e.g. disk, cpu, or diffusion=disk,clip=cpu", ¶ms_backend}, + {"", + "--rpc-servers", + "comma-separated list of RPC servers to connect to for offloading, in the format host:port, e.g. localhost:50052,192.168.1.3:50052", + &rpc_servers}, + {"", + "--max-vram", + "maximum VRAM budget in GiB for graph-cut segmented execution. Accepts a single value or assignments by backend/device, e.g. 6 or cuda0=6,vulkan0=4. 0 disables graph splitting; a negative value auto-detects free VRAM, sparing the specified value", + &max_vram}, }; options.int_options = { @@ -437,13 +453,6 @@ ArgOptions SDContextParams::get_options() { &chroma_t5_mask_pad}, }; - options.float_options = { - {"", - "--max-vram", - "maximum VRAM budget in GiB for graph-cut segmented execution. 0 disables graph splitting; a negative value auto-detects free VRAM, sparing the specified value (e.g. -0.5 will keep at least 0.5 GiB free)", - &max_vram}, - }; - options.bool_options = { {"", "--stream-layers", @@ -463,15 +472,15 @@ ArgOptions SDContextParams::get_options() { true, &enable_mmap}, {"", "--control-net-cpu", - "keep controlnet in cpu (for low vram)", + "deprecated; use --backend controlnet=cpu", true, &control_net_cpu}, {"", "--clip-on-cpu", - "keep clip in cpu (for low vram)", + "deprecated; use --backend te=cpu", true, &clip_on_cpu}, {"", "--vae-on-cpu", - "keep vae in cpu (for low vram)", + "deprecated; use --backend vae=cpu", true, &vae_on_cpu}, {"", "--fa", @@ -688,6 +697,25 @@ bool SDContextParams::resolve_and_validate(SDMode mode) { return true; } +void SDContextParams::prepare_backend_assignments() { + effective_backend = backend; + effective_params_backend = params_backend; + + if (offload_params_to_cpu) { + prepend_backend_assignment(effective_params_backend, "*=cpu"); + } + + if (clip_on_cpu) { + prepend_backend_assignment(effective_backend, "te=cpu"); + } + if (vae_on_cpu) { + prepend_backend_assignment(effective_backend, "vae=cpu"); + } + if (control_net_cpu) { + prepend_backend_assignment(effective_backend, "controlnet=cpu"); + } +} + std::string SDContextParams::to_string() const { std::ostringstream emb_ss; emb_ss << "{\n"; @@ -731,7 +759,7 @@ std::string SDContextParams::to_string() const { << " rng_type: " << sd_rng_type_name(rng_type) << ",\n" << " sampler_rng_type: " << sd_rng_type_name(sampler_rng_type) << ",\n" << " offload_params_to_cpu: " << (offload_params_to_cpu ? "true" : "false") << ",\n" - << " max_vram: " << max_vram << ",\n" + << " max_vram: \"" << max_vram << "\",\n" << " stream_layers: " << (stream_layers ? "true" : "false") << ",\n" << " backend: \"" << backend << "\",\n" << " params_backend: \"" << params_backend << "\",\n" @@ -757,7 +785,8 @@ std::string SDContextParams::to_string() const { return oss.str(); } -sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool vae_decode_only, bool free_params_immediately, bool taesd_preview) { +sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool taesd_preview) { + prepare_backend_assignments(); embedding_vec.clear(); embedding_vec.reserve(embedding_map.size()); for (const auto& kv : embedding_map) { @@ -767,57 +796,53 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t(bool vae_decode_only, bool f embedding_vec.emplace_back(item); } - sd_ctx_params_t sd_ctx_params = { - model_path.c_str(), - clip_l_path.c_str(), - clip_g_path.c_str(), - clip_vision_path.c_str(), - t5xxl_path.c_str(), - llm_path.c_str(), - llm_vision_path.c_str(), - diffusion_model_path.c_str(), - high_noise_diffusion_model_path.c_str(), - uncond_diffusion_model_path.c_str(), - embeddings_connectors_path.c_str(), - vae_path.c_str(), - audio_vae_path.c_str(), - taesd_path.c_str(), - control_net_path.c_str(), - embedding_vec.data(), - static_cast(embedding_vec.size()), - photo_maker_path.c_str(), - tensor_type_rules.c_str(), - vae_decode_only, - free_params_immediately, - n_threads, - wtype, - rng_type, - sampler_rng_type, - prediction, - lora_apply_mode, - offload_params_to_cpu, - enable_mmap, - clip_on_cpu, - control_net_cpu, - vae_on_cpu, - flash_attn, - diffusion_flash_attn, - taesd_preview, - diffusion_conv_direct, - vae_conv_direct, - circular || circular_x, - circular || circular_y, - force_sdxl_vae_conv_scale, - chroma_use_dit_mask, - chroma_use_t5_mask, - chroma_t5_mask_pad, - qwen_image_zero_cond_t, - str_to_vae_format(vae_format), - max_vram, - stream_layers, - backend.c_str(), - params_backend.c_str(), - }; + sd_ctx_params_t sd_ctx_params; + sd_ctx_params_init(&sd_ctx_params); + sd_ctx_params.model_path = model_path.c_str(); + sd_ctx_params.clip_l_path = clip_l_path.c_str(); + sd_ctx_params.clip_g_path = clip_g_path.c_str(); + sd_ctx_params.clip_vision_path = clip_vision_path.c_str(); + sd_ctx_params.t5xxl_path = t5xxl_path.c_str(); + sd_ctx_params.llm_path = llm_path.c_str(); + sd_ctx_params.llm_vision_path = llm_vision_path.c_str(); + sd_ctx_params.diffusion_model_path = diffusion_model_path.c_str(); + sd_ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path.c_str(); + sd_ctx_params.uncond_diffusion_model_path = uncond_diffusion_model_path.c_str(); + sd_ctx_params.embeddings_connectors_path = embeddings_connectors_path.c_str(); + sd_ctx_params.vae_path = vae_path.c_str(); + sd_ctx_params.audio_vae_path = audio_vae_path.c_str(); + sd_ctx_params.taesd_path = taesd_path.c_str(); + sd_ctx_params.control_net_path = control_net_path.c_str(); + sd_ctx_params.embeddings = embedding_vec.data(); + sd_ctx_params.embedding_count = static_cast(embedding_vec.size()); + sd_ctx_params.photo_maker_path = photo_maker_path.c_str(); + sd_ctx_params.pulid_weights_path = pulid_weights_path.c_str(); + sd_ctx_params.tensor_type_rules = tensor_type_rules.c_str(); + sd_ctx_params.n_threads = n_threads; + sd_ctx_params.wtype = wtype; + sd_ctx_params.rng_type = rng_type; + sd_ctx_params.sampler_rng_type = sampler_rng_type; + sd_ctx_params.prediction = prediction; + sd_ctx_params.lora_apply_mode = lora_apply_mode; + sd_ctx_params.enable_mmap = enable_mmap; + sd_ctx_params.flash_attn = flash_attn; + sd_ctx_params.diffusion_flash_attn = diffusion_flash_attn; + sd_ctx_params.tae_preview_only = taesd_preview; + sd_ctx_params.diffusion_conv_direct = diffusion_conv_direct; + sd_ctx_params.vae_conv_direct = vae_conv_direct; + sd_ctx_params.circular_x = circular || circular_x; + sd_ctx_params.circular_y = circular || circular_y; + sd_ctx_params.force_sdxl_vae_conv_scale = force_sdxl_vae_conv_scale; + sd_ctx_params.chroma_use_dit_mask = chroma_use_dit_mask; + sd_ctx_params.chroma_use_t5_mask = chroma_use_t5_mask; + sd_ctx_params.chroma_t5_mask_pad = chroma_t5_mask_pad; + sd_ctx_params.qwen_image_zero_cond_t = qwen_image_zero_cond_t; + sd_ctx_params.vae_format = str_to_vae_format(vae_format); + sd_ctx_params.max_vram = max_vram.c_str(); + sd_ctx_params.stream_layers = stream_layers; + sd_ctx_params.backend = effective_backend.c_str(); + sd_ctx_params.params_backend = effective_params_backend.c_str(); + sd_ctx_params.rpc_servers = rpc_servers.c_str(); return sd_ctx_params; } @@ -867,6 +892,10 @@ ArgOptions SDGenerationParams::get_options() { "--pm-id-embed-path", "path to PHOTOMAKER v2 id embed", &pm_id_embed_path}, + {"", + "--pulid-id-embedding", + "path to PuLID id embedding", + &pulid_id_embedding_path}, {"", "--hires-upscaler", "highres fix upscaler, Lanczos, Nearest, Latent, Latent (nearest), Latent (nearest-exact), " @@ -1017,6 +1046,10 @@ ArgOptions SDGenerationParams::get_options() { "--pm-style-strength", "", &pm_style_strength}, + {"", + "--pulid-id-weight", + "strength of PuLID identity injection", + &pulid_id_weight}, {"", "--control-strength", "strength to apply Control Net (default: 0.9). 1.0 corresponds to full destruction of information in init image", @@ -2249,6 +2282,11 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { pm_style_strength, }; + sd_pulid_params_t pulid_params = { + pulid_id_embedding_path.empty() ? nullptr : pulid_id_embedding_path.c_str(), + pulid_id_weight, + }; + params.loras = lora_vec.empty() ? nullptr : lora_vec.data(); params.lora_count = static_cast(lora_vec.size()); params.prompt = prompt.c_str(); @@ -2269,6 +2307,7 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() { params.control_image = control_image.get(); params.control_strength = control_strength; params.pm_params = pm_params; + params.pulid_params = pulid_params; params.vae_tiling_params = vae_tiling_params; params.cache = cache_params; diff --git a/otherarch/sdcpp/examples/common/common.h b/otherarch/sdcpp/examples/common/common.h index a90a33132..fcf9840db 100644 --- a/otherarch/sdcpp/examples/common/common.h +++ b/otherarch/sdcpp/examples/common/common.h @@ -133,6 +133,7 @@ struct SDContextParams { std::string control_net_path; std::string embedding_dir; std::string photo_maker_path; + std::string pulid_weights_path; sd_type_t wtype = SD_TYPE_COUNT; std::string tensor_type_rules; std::string lora_model_dir = "."; @@ -144,10 +145,13 @@ struct SDContextParams { rng_type_t rng_type = CUDA_RNG; rng_type_t sampler_rng_type = RNG_TYPE_COUNT; bool offload_params_to_cpu = false; - float max_vram = 0.f; + std::string max_vram = "0"; bool stream_layers = false; std::string backend; std::string params_backend; + std::string rpc_servers; + std::string effective_backend; + std::string effective_params_backend; bool enable_mmap = false; bool control_net_cpu = false; bool clip_on_cpu = false; @@ -175,11 +179,12 @@ struct SDContextParams { float flow_shift = INFINITY; ArgOptions get_options(); void build_embedding_map(); + void prepare_backend_assignments(); bool resolve(SDMode mode); bool validate(SDMode mode); bool resolve_and_validate(SDMode mode); std::string to_string() const; - sd_ctx_params_t to_sd_ctx_params_t(bool vae_decode_only, bool free_params_immediately, bool taesd_preview); + sd_ctx_params_t to_sd_ctx_params_t(bool taesd_preview); }; struct SDGenerationParams { @@ -230,6 +235,9 @@ struct SDGenerationParams { std::string pm_id_embed_path; float pm_style_strength = 20.f; + std::string pulid_id_embedding_path; + float pulid_id_weight = 1.0f; + int upscale_repeats = 1; int upscale_tile_size = 128; diff --git a/otherarch/sdcpp/include/stable-diffusion.h b/otherarch/sdcpp/include/stable-diffusion.h index 2175f895a..1c04367b1 100644 --- a/otherarch/sdcpp/include/stable-diffusion.h +++ b/otherarch/sdcpp/include/stable-diffusion.h @@ -195,20 +195,15 @@ typedef struct { const sd_embedding_t* embeddings; uint32_t embedding_count; const char* photo_maker_path; + const char* pulid_weights_path; const char* tensor_type_rules; - bool vae_decode_only; - bool free_params_immediately; int n_threads; enum sd_type_t wtype; enum rng_type_t rng_type; enum rng_type_t sampler_rng_type; enum prediction_t prediction; enum lora_apply_mode_t lora_apply_mode; - bool offload_params_to_cpu; bool enable_mmap; - bool keep_clip_on_cpu; - bool keep_control_net_on_cpu; - bool keep_vae_on_cpu; bool flash_attn; bool diffusion_flash_attn; bool tae_preview_only; @@ -222,10 +217,11 @@ typedef struct { int chroma_t5_mask_pad; bool qwen_image_zero_cond_t; enum sd_vae_format_t vae_format; - float max_vram; // GiB budget for graph-cut segmented param offload (0 = disabled, -1 = auto free VRAM minus 1 GiB) + const char* max_vram; // GiB budget or backend assignment spec for graph-cut segmented param offload (0 = disabled, -1 = auto) bool stream_layers; // Enable residency+prefetch streaming on top of --max-vram (no effect without --max-vram) const char* backend; const char* params_backend; + const char* rpc_servers; } sd_ctx_params_t; typedef struct { @@ -277,6 +273,11 @@ typedef struct { float style_strength; } sd_pm_params_t; // photo maker +typedef struct { + const char* id_embedding_path; + float id_weight; +} sd_pulid_params_t; + enum sd_cache_mode_t { SD_CACHE_DISABLED = 0, SD_CACHE_EASYCACHE, @@ -369,6 +370,7 @@ typedef struct { sd_image_t control_image; float control_strength; sd_pm_params_t pm_params; + sd_pulid_params_t pulid_params; sd_tiling_params_t vae_tiling_params; sd_cache_params_t cache; sd_hires_params_t hires; @@ -450,6 +452,17 @@ SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params); SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params); SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* sd_img_gen_params); +enum sd_cancel_mode_t { + // Stop the current generation as soon as possible. + SD_CANCEL_ALL, + // Finish the current image sample, then skip additional batch latents and return completed images. + SD_CANCEL_NEW_LATENTS, + // Clear a pending cancellation request. + SD_CANCEL_RESET +}; + +SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode); + SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params); SD_API bool generate_video(sd_ctx_t* sd_ctx, const sd_vid_gen_params_t* sd_vid_gen_params, @@ -460,7 +473,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, typedef struct upscaler_ctx_t upscaler_ctx_t; SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path, - bool offload_params_to_cpu, bool direct, int n_threads, int tile_size, diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index c37c9fcb8..839f95141 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -412,8 +412,10 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { } else if (inputs.use_mmap) { printf("Using mmap for I/O\n"); } + std::string max_vram; if(inputs.max_vram != 0.f) { printf("Using max VRAM = %0.2f GB\n", inputs.max_vram); + max_vram = std::to_string(inputs.max_vram); } if(inputs.quant > 0) { @@ -470,8 +472,6 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { params.taesd_path = sd_params->taesd_path.c_str(); params.photo_maker_path = sd_params->stacked_id_embeddings_path.c_str(); - params.vae_decode_only = false; - params.free_params_immediately = false; params.rng_type = CUDA_RNG; params.n_threads = sd_params->n_threads; @@ -480,7 +480,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { params.diffusion_conv_direct = sd_params->diffusion_conv_direct; params.vae_conv_direct = sd_params->vae_conv_direct; params.chroma_use_dit_mask = true; - params.max_vram = inputs.max_vram; + params.max_vram = max_vram.c_str(); params.stream_layers = inputs.stream_layers; params.enable_mmap = inputs.use_mmap; params.params_backend = inputs.offload_cpu ? "CPU" : ""; @@ -539,7 +539,6 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { if (upscaler_filename!="") { const int upscale_tile_size = 128; upscaler_ctx = new_upscaler_ctx(upscaler_filename.c_str(), - inputs.offload_cpu, params.diffusion_conv_direct, params.n_threads, upscale_tile_size, diff --git a/otherarch/sdcpp/src/conditioning/conditioner.hpp b/otherarch/sdcpp/src/conditioning/conditioner.hpp index 5e74af073..b5dda4c0e 100644 --- a/otherarch/sdcpp/src/conditioning/conditioner.hpp +++ b/otherarch/sdcpp/src/conditioning/conditioner.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_CONDITIONING_CONDITIONER_HPP__ +#ifndef __SD_CONDITIONING_CONDITIONER_HPP__ #define __SD_CONDITIONING_CONDITIONER_HPP__ #include @@ -113,14 +113,12 @@ struct Conditioner { public: virtual SDCondition get_learned_condition(int n_threads, const ConditionerParams& conditioner_params) = 0; - virtual bool alloc_params_buffer() = 0; - virtual void free_params_buffer() = 0; virtual void get_param_tensors(std::map& tensors) = 0; - virtual size_t get_params_buffer_size() = 0; virtual void set_max_graph_vram_bytes(size_t max_vram_bytes) {} virtual void set_stream_layers_enabled(bool enabled) {} virtual void set_flash_attention_enabled(bool enabled) = 0; virtual void set_weight_adapter(const std::shared_ptr& adapter) {} + virtual void runner_done() {} }; // ldm.modules.encoders.modules.FrozenCLIPEmbedder @@ -138,10 +136,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { std::map> embedding_pos_map; FrozenCLIPEmbedderWithCustomWords(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::map& orig_embedding_map, - SDVersion version = VERSION_SD1) + SDVersion version = VERSION_SD1, + std::shared_ptr weight_manager = nullptr) : version(version), tokenizer(sd_version_is_sd2(version) ? 0 : 49407) { for (const auto& kv : orig_embedding_map) { std::string name = kv.first; @@ -151,12 +149,12 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { } bool force_clip_f32 = !embedding_map.empty(); if (sd_version_is_sd1(version)) { - text_model = std::make_shared(backend, params_backend, tensor_storage_map, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, force_clip_f32); + text_model = std::make_shared(backend, tensor_storage_map, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, force_clip_f32, weight_manager); } else if (sd_version_is_sd2(version)) { - text_model = std::make_shared(backend, params_backend, tensor_storage_map, "cond_stage_model.transformer.text_model", OPEN_CLIP_VIT_H_14, true, force_clip_f32); + text_model = std::make_shared(backend, tensor_storage_map, "cond_stage_model.transformer.text_model", OPEN_CLIP_VIT_H_14, true, force_clip_f32, weight_manager); } else if (sd_version_is_sdxl(version)) { - text_model = std::make_shared(backend, params_backend, tensor_storage_map, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, force_clip_f32); - text_model2 = std::make_shared(backend, params_backend, tensor_storage_map, "cond_stage_model.1.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, force_clip_f32); + text_model = std::make_shared(backend, tensor_storage_map, "cond_stage_model.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, force_clip_f32, weight_manager); + text_model2 = std::make_shared(backend, tensor_storage_map, "cond_stage_model.1.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, force_clip_f32, weight_manager); } } @@ -167,33 +165,6 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { } } - bool alloc_params_buffer() override { - if (!text_model->alloc_params_buffer()) { - return false; - } - if (sd_version_is_sdxl(version)) { - if (!text_model2->alloc_params_buffer()) { - return false; - } - } - return true; - } - - void free_params_buffer() override { - text_model->free_params_buffer(); - if (sd_version_is_sdxl(version)) { - text_model2->free_params_buffer(); - } - } - - size_t get_params_buffer_size() override { - size_t buffer_size = text_model->get_params_buffer_size(); - if (sd_version_is_sdxl(version)) { - buffer_size += text_model2->get_params_buffer_size(); - } - return buffer_size; - } - void set_max_graph_vram_bytes(size_t max_vram_bytes) override { text_model->set_max_graph_vram_bytes(max_vram_bytes); if (sd_version_is_sdxl(version)) { @@ -222,6 +193,13 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { } } + void runner_done() override { + text_model->runner_done(); + if (sd_version_is_sdxl(version)) { + text_model2->runner_done(); + } + } + bool load_embedding(std::string embd_name, std::string embd_path, std::vector& bpe_tokens) { ModelLoader model_loader; if (!model_loader.init_from_file_and_convert_name(embd_path)) { @@ -263,7 +241,8 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { } return true; }; - model_loader.load_tensors(on_load, 1); + model_loader.set_n_threads(1); + model_loader.load_tensors(on_load); int pos_start = num_custom_embeddings; if (embd) { int64_t hidden_size = text_model->model.hidden_size; @@ -432,7 +411,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { token_embed_custom.data(), max_token_idx, false, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!chunk_hidden_states.empty()); if (sd_version_is_sdxl(version)) { auto chunk_hidden_states2 = text_model2->compute(n_threads, @@ -441,7 +423,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { token_embed_custom.data(), max_token_idx, false, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!chunk_hidden_states2.empty()); chunk_hidden_states = sd::ops::concat(chunk_hidden_states, chunk_hidden_states2, 0); @@ -452,7 +437,10 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { token_embed_custom.data(), max_token_idx, true, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!pooled.empty()); } } @@ -523,15 +511,15 @@ struct FrozenCLIPEmbedderWithCustomWords : public Conditioner { struct FrozenCLIPVisionEmbedder : public GGMLRunner { CLIPVisionModelProjection vision_model; + std::string weight_prefix = "cond_stage_model.transformer"; FrozenCLIPVisionEmbedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}) - : GGMLRunner(backend, params_backend) { - std::string prefix = "cond_stage_model.transformer"; - bool proj_in = false; + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager) { + bool proj_in = false; for (const auto& [name, tensor_storage] : tensor_storage_map) { - if (!starts_with(name, prefix)) { + if (!starts_with(name, weight_prefix)) { continue; } if (contains(name, "self_attn.in_proj")) { @@ -540,7 +528,7 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner { } } vision_model = CLIPVisionModelProjection(OPEN_CLIP_VIT_H_14, false, proj_in); - vision_model.init(params_ctx, tensor_storage_map, prefix); + vision_model.init(params_ctx, tensor_storage_map, weight_prefix); } std::string get_desc() override { @@ -548,7 +536,7 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner { } void get_param_tensors(std::map& tensors) { - vision_model.get_param_tensors(tensors, "cond_stage_model.transformer"); + vision_model.get_param_tensors(tensors, weight_prefix); } ggml_cgraph* build_graph(const sd::Tensor& pixel_values_tensor, bool return_pooled, int clip_skip) { @@ -571,7 +559,7 @@ struct FrozenCLIPVisionEmbedder : public GGMLRunner { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(pixel_values, return_pooled, clip_skip); }; - return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true)); + return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true, true, true)); } }; @@ -584,8 +572,8 @@ struct SD3CLIPEmbedder : public Conditioner { std::shared_ptr t5; SD3CLIPEmbedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}) + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) : clip_g_tokenizer(0) { bool use_clip_l = false; bool use_clip_g = false; @@ -604,13 +592,13 @@ struct SD3CLIPEmbedder : public Conditioner { return; } if (use_clip_l) { - clip_l = std::make_shared(backend, params_backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, false); + clip_l = std::make_shared(backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, false, false, weight_manager); } if (use_clip_g) { - clip_g = std::make_shared(backend, params_backend, tensor_storage_map, "text_encoders.clip_g.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false); + clip_g = std::make_shared(backend, tensor_storage_map, "text_encoders.clip_g.transformer.text_model", OPEN_CLIP_VIT_BIGG_14, false, false, weight_manager); } if (use_t5) { - t5 = std::make_shared(backend, params_backend, tensor_storage_map, "text_encoders.t5xxl.transformer"); + t5 = std::make_shared(backend, tensor_storage_map, "text_encoders.t5xxl.transformer", false, weight_manager); } } @@ -626,51 +614,6 @@ struct SD3CLIPEmbedder : public Conditioner { } } - bool alloc_params_buffer() override { - if (clip_l) { - if (!clip_l->alloc_params_buffer()) { - return false; - } - } - if (clip_g) { - if (!clip_g->alloc_params_buffer()) { - return false; - } - } - if (t5) { - if (!t5->alloc_params_buffer()) { - return false; - } - } - return true; - } - - void free_params_buffer() override { - if (clip_l) { - clip_l->free_params_buffer(); - } - if (clip_g) { - clip_g->free_params_buffer(); - } - if (t5) { - t5->free_params_buffer(); - } - } - - size_t get_params_buffer_size() override { - size_t buffer_size = 0; - if (clip_l) { - buffer_size += clip_l->get_params_buffer_size(); - } - if (clip_g) { - buffer_size += clip_g->get_params_buffer_size(); - } - if (t5) { - buffer_size += t5->get_params_buffer_size(); - } - return buffer_size; - } - void set_max_graph_vram_bytes(size_t max_vram_bytes) override { if (clip_l) { clip_l->set_max_graph_vram_bytes(max_vram_bytes); @@ -719,6 +662,18 @@ struct SD3CLIPEmbedder : public Conditioner { } } + void runner_done() override { + if (clip_l) { + clip_l->runner_done(); + } + if (clip_g) { + clip_g->runner_done(); + } + if (t5) { + t5->runner_done(); + } + } + std::vector, std::vector>> tokenize(std::string text, size_t min_length = 0, size_t max_length = 0, @@ -834,7 +789,10 @@ struct SD3CLIPEmbedder : public Conditioner { nullptr, max_token_idx, false, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!chunk_hidden_states_l.empty()); chunk_hidden_states_l = ::apply_token_weights(std::move(chunk_hidden_states_l), chunk_weights); @@ -847,7 +805,10 @@ struct SD3CLIPEmbedder : public Conditioner { nullptr, max_token_idx, true, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!pooled_l.empty()); } } else { @@ -875,7 +836,10 @@ struct SD3CLIPEmbedder : public Conditioner { nullptr, max_token_idx, false, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!chunk_hidden_states_g.empty()); chunk_hidden_states_g = ::apply_token_weights(std::move(chunk_hidden_states_g), chunk_weights); @@ -888,7 +852,10 @@ struct SD3CLIPEmbedder : public Conditioner { nullptr, max_token_idx, true, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!pooled_g.empty()); } } else { @@ -910,7 +877,10 @@ struct SD3CLIPEmbedder : public Conditioner { chunk_hidden_states_t5 = t5->compute(n_threads, input_ids, - sd::Tensor()); + sd::Tensor(), + false, + true, + true); GGML_ASSERT(!chunk_hidden_states_t5.empty()); chunk_hidden_states_t5 = ::apply_token_weights(std::move(chunk_hidden_states_t5), chunk_weights); } else { @@ -971,8 +941,8 @@ struct FluxCLIPEmbedder : public Conditioner { size_t chunk_len = 256; FluxCLIPEmbedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}) { + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) { bool use_clip_l = false; bool use_t5 = false; for (auto pair : tensor_storage_map) { @@ -989,12 +959,12 @@ struct FluxCLIPEmbedder : public Conditioner { } if (use_clip_l) { - clip_l = std::make_shared(backend, params_backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, true); + clip_l = std::make_shared(backend, tensor_storage_map, "text_encoders.clip_l.transformer.text_model", OPENAI_CLIP_VIT_L_14, true, false, weight_manager); } else { LOG_WARN("clip_l text encoder not found! Prompt adherence might be degraded."); } if (use_t5) { - t5 = std::make_shared(backend, params_backend, tensor_storage_map, "text_encoders.t5xxl.transformer"); + t5 = std::make_shared(backend, tensor_storage_map, "text_encoders.t5xxl.transformer", false, weight_manager); } else { LOG_WARN("t5xxl text encoder not found! Prompt adherence might be degraded."); } @@ -1009,40 +979,6 @@ struct FluxCLIPEmbedder : public Conditioner { } } - bool alloc_params_buffer() override { - if (clip_l) { - if (!clip_l->alloc_params_buffer()) { - return false; - } - } - if (t5) { - if (!t5->alloc_params_buffer()) { - return false; - } - } - return true; - } - - void free_params_buffer() override { - if (clip_l) { - clip_l->free_params_buffer(); - } - if (t5) { - t5->free_params_buffer(); - } - } - - size_t get_params_buffer_size() override { - size_t buffer_size = 0; - if (clip_l) { - buffer_size += clip_l->get_params_buffer_size(); - } - if (t5) { - buffer_size += t5->get_params_buffer_size(); - } - return buffer_size; - } - void set_max_graph_vram_bytes(size_t max_vram_bytes) override { if (clip_l) { clip_l->set_max_graph_vram_bytes(max_vram_bytes); @@ -1070,7 +1006,7 @@ struct FluxCLIPEmbedder : public Conditioner { } } - void set_weight_adapter(const std::shared_ptr& adapter) { + void set_weight_adapter(const std::shared_ptr& adapter) override { if (clip_l) { clip_l->set_weight_adapter(adapter); } @@ -1079,6 +1015,15 @@ struct FluxCLIPEmbedder : public Conditioner { } } + void runner_done() override { + if (clip_l) { + clip_l->runner_done(); + } + if (t5) { + t5->runner_done(); + } + } + std::vector, std::vector>> tokenize(std::string text, size_t min_length = 0, size_t max_length = 0) { @@ -1177,7 +1122,10 @@ struct FluxCLIPEmbedder : public Conditioner { nullptr, max_token_idx, true, - clip_skip); + clip_skip, + false, + true, + true); GGML_ASSERT(!pooled.empty()); } else { pooled = sd::Tensor::zeros({768}); @@ -1195,7 +1143,10 @@ struct FluxCLIPEmbedder : public Conditioner { sd::Tensor input_ids({static_cast(chunk_tokens.size())}, chunk_tokens); chunk_hidden_states = t5->compute(n_threads, input_ids, - sd::Tensor()); + sd::Tensor(), + false, + true, + true); GGML_ASSERT(!chunk_hidden_states.empty()); chunk_hidden_states = ::apply_token_weights(std::move(chunk_hidden_states), chunk_weights); if (zero_out_masked) { @@ -1239,11 +1190,11 @@ struct T5CLIPEmbedder : public Conditioner { bool is_umt5 = false; T5CLIPEmbedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - bool use_mask = false, - int mask_pad = 0, - bool is_umt5 = false) + const String2TensorStorage& tensor_storage_map = {}, + bool use_mask = false, + int mask_pad = 0, + bool is_umt5 = false, + std::shared_ptr weight_manager = nullptr) : use_mask(use_mask), mask_pad(mask_pad), t5_tokenizer(is_umt5) { bool use_t5 = false; for (auto pair : tensor_storage_map) { @@ -1256,7 +1207,7 @@ struct T5CLIPEmbedder : public Conditioner { LOG_WARN("IMPORTANT NOTICE: No text encoders provided, cannot process prompts!"); return; } else { - t5 = std::make_shared(backend, params_backend, tensor_storage_map, "text_encoders.t5xxl.transformer", is_umt5); + t5 = std::make_shared(backend, tensor_storage_map, "text_encoders.t5xxl.transformer", is_umt5, weight_manager); } } @@ -1266,29 +1217,6 @@ struct T5CLIPEmbedder : public Conditioner { } } - bool alloc_params_buffer() override { - if (t5) { - if (!t5->alloc_params_buffer()) { - return false; - } - } - return true; - } - - void free_params_buffer() override { - if (t5) { - t5->free_params_buffer(); - } - } - - size_t get_params_buffer_size() override { - size_t buffer_size = 0; - if (t5) { - buffer_size += t5->get_params_buffer_size(); - } - return buffer_size; - } - void set_max_graph_vram_bytes(size_t max_vram_bytes) override { if (t5) { t5->set_max_graph_vram_bytes(max_vram_bytes); @@ -1313,6 +1241,12 @@ struct T5CLIPEmbedder : public Conditioner { } } + void runner_done() override { + if (t5) { + t5->runner_done(); + } + } + std::tuple, std::vector, std::vector> tokenize(std::string text, size_t min_length = 0, size_t max_length = 0) { @@ -1406,7 +1340,10 @@ struct T5CLIPEmbedder : public Conditioner { auto chunk_hidden_states = t5->compute(n_threads, input_ids, - t5_attn_mask_chunk); + t5_attn_mask_chunk, + false, + true, + true); GGML_ASSERT(!chunk_hidden_states.empty()); chunk_hidden_states = apply_token_weights(std::move(chunk_hidden_states), chunk_weights); @@ -1450,36 +1387,21 @@ struct AnimaConditioner : public Conditioner { std::shared_ptr llm; AnimaConditioner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}) { + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) { qwen_tokenizer = std::make_shared(); llm = std::make_shared(LLM::LLMArch::QWEN3, backend, - params_backend, tensor_storage_map, "text_encoders.llm", - false); + false, + weight_manager); } void get_param_tensors(std::map& tensors) override { llm->get_param_tensors(tensors, "text_encoders.llm"); } - bool alloc_params_buffer() override { - if (!llm->alloc_params_buffer()) { - return false; - } - return true; - } - - void free_params_buffer() override { - llm->free_params_buffer(); - } - - size_t get_params_buffer_size() override { - return llm->get_params_buffer_size(); - } - void set_max_graph_vram_bytes(size_t max_vram_bytes) override { llm->set_max_graph_vram_bytes(max_vram_bytes); } @@ -1496,6 +1418,10 @@ struct AnimaConditioner : public Conditioner { llm->set_weight_adapter(adapter); } + void runner_done() override { + llm->runner_done(); + } + std::tuple, std::vector, std::vector, std::vector> tokenize(std::string text) { auto parsed_attention = parse_prompt_attention(text); @@ -1553,7 +1479,11 @@ struct AnimaConditioner : public Conditioner { input_ids, sd::Tensor(), {}, - {}); + {}, + false, + false, + true, + true); GGML_ASSERT(!hidden_states.empty()); hidden_states = apply_token_weights(std::move(hidden_states), qwen_weights); auto t5_ids_tensor = sd::Tensor::from_vector(t5_tokens); @@ -1576,11 +1506,11 @@ struct LLMEmbedder : public Conditioner { std::shared_ptr llm; LLMEmbedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - SDVersion version = VERSION_QWEN_IMAGE, - const std::string prefix = "", - bool enable_vision = false) + const String2TensorStorage& tensor_storage_map = {}, + SDVersion version = VERSION_QWEN_IMAGE, + const std::string prefix = "", + bool enable_vision = false, + std::shared_ptr weight_manager = nullptr) : version(version) { LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL; if (version == VERSION_FLUX2) { @@ -1607,33 +1537,16 @@ struct LLMEmbedder : public Conditioner { } llm = std::make_shared(arch, backend, - params_backend, tensor_storage_map, "text_encoders.llm", - enable_vision); + enable_vision, + weight_manager); } void get_param_tensors(std::map& tensors) override { llm->get_param_tensors(tensors, "text_encoders.llm"); } - bool alloc_params_buffer() override { - if (!llm->alloc_params_buffer()) { - return false; - } - return true; - } - - void free_params_buffer() override { - llm->free_params_buffer(); - } - - size_t get_params_buffer_size() override { - size_t buffer_size = 0; - buffer_size += llm->get_params_buffer_size(); - return buffer_size; - } - void set_max_graph_vram_bytes(size_t max_vram_bytes) override { llm->set_max_graph_vram_bytes(max_vram_bytes); } @@ -1652,6 +1565,12 @@ struct LLMEmbedder : public Conditioner { } } + void runner_done() override { + if (llm) { + llm->runner_done(); + } + } + std::tuple, std::vector, std::vector> tokenize(std::string text, const std::pair& attn_range, size_t min_length = 0, @@ -1747,7 +1666,11 @@ struct LLMEmbedder : public Conditioner { input_ids, attention_mask, image_embeds, - out_layers); + out_layers, + false, + false, + true, + true); GGML_ASSERT(!hidden_states.empty()); hidden_states = apply_token_weights(std::move(hidden_states), weights); GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx); @@ -1825,7 +1748,7 @@ struct LLMEmbedder : public Conditioner { auto resized_image = clip_preprocess(image, w_bar, h_bar); - auto image_embed = llm->encode_image(n_threads, resized_image); + auto image_embed = llm->encode_image(n_threads, resized_image, false, true, true); GGML_ASSERT(!image_embed.empty()); image_embeds.emplace_back(image_embed_idx, image_embed); image_embed_idx += 1 + static_cast(image_embed.shape()[1]) + 6; @@ -1895,7 +1818,7 @@ struct LLMEmbedder : public Conditioner { LOG_DEBUG("resize conditioner ref image %d from %dx%d to %dx%d", i, height, width, h_bar, w_bar); auto resized_image = clip_preprocess(image, w_bar, h_bar); - auto image_embed = llm->encode_image(n_threads, resized_image); + auto image_embed = llm->encode_image(n_threads, resized_image, false, true, true); GGML_ASSERT(!image_embed.empty()); image_embeds.emplace_back(image_embed_idx, image_embed); image_embed_idx += 1 + static_cast(image_embed.shape()[1]) + 6; @@ -2138,10 +2061,10 @@ struct LTXAVTextProjectionRunner : public GGMLRunner { LTXAVTextProjection model; LTXAVTextProjectionRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string& prefix = "") - : GGMLRunner(backend, params_backend), + const String2TensorStorage& tensor_storage_map = {}, + const std::string& prefix = "", + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), model(tensor_storage_map.find(prefix + ".video_aggregate_embed.weight") != tensor_storage_map.end()) { model.init(params_ctx, tensor_storage_map, prefix); } @@ -2163,11 +2086,15 @@ struct LTXAVTextProjectionRunner : public GGMLRunner { return gf; } - sd::Tensor compute(int n_threads, const sd::Tensor& x) { + sd::Tensor compute(int n_threads, + const sd::Tensor& x, + bool auto_free = true, + bool free_compute_buffer = true, + bool free_compute_params = true) { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); }; - return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true)); + return take_or_empty(GGMLRunner::compute(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params)); } }; @@ -2182,22 +2109,22 @@ struct LTXAVEmbedder : public Conditioner { bool dual_projection = false; LTXAVEmbedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string& llm_prefix = "text_encoders.llm", - const std::string& projector_prefix = "text_embedding_projection") { + const String2TensorStorage& tensor_storage_map = {}, + const std::string& llm_prefix = "text_encoders.llm", + const std::string& projector_prefix = "text_embedding_projection", + std::shared_ptr weight_manager = nullptr) { tokenizer = std::make_shared(); llm = std::make_shared(LLM::LLMArch::GEMMA3_12B, backend, - params_backend, tensor_storage_map, llm_prefix, - false); + false, + weight_manager); dual_projection = tensor_storage_map.find(projector_prefix + ".video_aggregate_embed.weight") != tensor_storage_map.end(); projector = std::make_shared(backend, - params_backend, tensor_storage_map, - projector_prefix); + projector_prefix, + weight_manager); } void get_param_tensors(std::map& tensors) override { @@ -2205,25 +2132,6 @@ struct LTXAVEmbedder : public Conditioner { projector->get_param_tensors(tensors, "text_embedding_projection"); } - bool alloc_params_buffer() override { - if (!llm->alloc_params_buffer()) { - return false; - } - if (!projector->alloc_params_buffer()) { - return false; - } - return true; - } - - void free_params_buffer() override { - llm->free_params_buffer(); - projector->free_params_buffer(); - } - - size_t get_params_buffer_size() override { - return llm->get_params_buffer_size() + projector->get_params_buffer_size(); - } - void set_flash_attention_enabled(bool enabled) override { llm->set_flash_attention_enabled(enabled); projector->set_flash_attention_enabled(enabled); @@ -2239,6 +2147,11 @@ struct LTXAVEmbedder : public Conditioner { projector->set_weight_adapter(adapter); } + void runner_done() override { + llm->runner_done(); + projector->runner_done(); + } + std::tuple, std::vector, std::vector> tokenize(std::string text, const std::pair& attn_range) { std::vector> parsed_attention; @@ -2302,6 +2215,9 @@ struct LTXAVEmbedder : public Conditioner { attention_mask, {}, {}, + true, + false, + true, true); GGML_ASSERT(!hidden_states.empty()); hidden_states = apply_token_weights(std::move(hidden_states), weights); @@ -2361,7 +2277,7 @@ struct LTXAVEmbedder : public Conditioner { } hidden_states.reshape_({kNumStates * kHiddenSize, valid_tokens}); - return projector->compute(n_threads, hidden_states); + return projector->compute(n_threads, hidden_states, false, true, true); } SDCondition get_learned_condition(int n_threads, diff --git a/otherarch/sdcpp/src/core/ggml_extend.hpp b/otherarch/sdcpp/src/core/ggml_extend.hpp index c85728529..f10a84ffd 100644 --- a/otherarch/sdcpp/src/core/ggml_extend.hpp +++ b/otherarch/sdcpp/src/core/ggml_extend.hpp @@ -25,7 +25,6 @@ #include "core/ggml_extend_backend.h" #include "core/ggml_graph_cut.h" -#include "core/layer_registry.h" #include "ggml-alloc.h" #include "ggml-backend.h" #include "ggml.h" @@ -36,6 +35,7 @@ #include "core/rng.hpp" #include "core/tensor_ggml.hpp" #include "core/util.h" +#include "weight_manager.h" #define EPS 1e-05f @@ -1655,6 +1655,7 @@ struct GGMLRunnerContext { std::vector>* debug_tensors = nullptr; std::function get_cache_tensor; std::function cache_tensor; + std::function set_backend_tensor_data; void capture_tensor(const std::string& name, ggml_tensor* tensor) { if (debug_tensors == nullptr || tensor == nullptr) { @@ -1680,6 +1681,13 @@ struct GGMLRunnerContext { } cache_tensor(name, tensor); } + + void bind_backend_tensor_data(ggml_tensor* tensor, const void* data) const { + if (!set_backend_tensor_data || tensor == nullptr || data == nullptr) { + return; + } + set_backend_tensor_data(tensor, data); + } }; struct GGMLRunner { @@ -1688,14 +1696,9 @@ protected: using GraphCutSegment = sd::ggml_graph_cut::Segment; using GraphCutPlan = sd::ggml_graph_cut::Plan; - ggml_backend_t params_backend = nullptr; ggml_backend_t runtime_backend = nullptr; - ggml_context* params_ctx = nullptr; - ggml_backend_buffer_t params_buffer = nullptr; - ggml_context* offload_ctx = nullptr; - ggml_backend_buffer_t runtime_params_buffer = nullptr; - bool params_on_runtime_backend = false; + ggml_context* params_ctx = nullptr; ggml_context* cache_ctx = nullptr; ggml_backend_buffer_t cache_buffer = nullptr; @@ -1703,24 +1706,16 @@ protected: ggml_context* compute_ctx = nullptr; ggml_gallocr* compute_allocr = nullptr; - ggml_context* partial_offload_ctx = nullptr; - ggml_backend_buffer_t partial_runtime_params_buffer = nullptr; - std::vector> partial_offload_pairs; - - // Params kept on the runtime backend across streaming segments. - ggml_context* resident_offload_ctx = nullptr; - std::vector> resident_offload_pairs; - ggml_backend_buffer_t resident_runtime_params_buffer = nullptr; - std::unordered_set resident_param_set; - uint64_t resident_state_token = 0; - size_t max_graph_vram_bytes = 0; bool stream_layers_enabled = false; size_t observed_max_effective_budget_ = 0; - sd::layer_registry::LayerRegistry layer_registry_; - std::shared_ptr weight_adapter = nullptr; + std::weak_ptr weight_manager; + std::unordered_set kept_compute_param_tensor_set; + std::vector runner_param_tensors; + std::unordered_set runner_param_tensor_set; + bool params_tensor_set_dirty_ = true; std::vector one_vec = {1.f}; ggml_tensor* one_tensor = nullptr; @@ -1776,10 +1771,7 @@ protected: params_ctx = ggml_init(params); GGML_ASSERT(params_ctx != nullptr); params_tensor_set_.clear(); - if (params_backend != runtime_backend) { - offload_ctx = ggml_init(params); - GGML_ASSERT(offload_ctx != nullptr); - } + params_tensor_set_dirty_ = true; } void free_params_ctx() { @@ -1788,14 +1780,7 @@ protected: params_ctx = nullptr; } params_tensor_set_.clear(); - if (offload_ctx != nullptr) { - ggml_free(offload_ctx); - offload_ctx = nullptr; - } - if (partial_offload_ctx != nullptr) { - ggml_free(partial_offload_ctx); - partial_offload_ctx = nullptr; - } + params_tensor_set_dirty_ = true; } void alloc_cache_ctx() { @@ -1835,6 +1820,9 @@ protected: } void rebuild_params_tensor_set() { + if (!params_tensor_set_dirty_) { + return; + } params_tensor_set_.clear(); if (params_ctx == nullptr) { return; @@ -1842,6 +1830,93 @@ protected: for (ggml_tensor* t = ggml_get_first_tensor(params_ctx); t != nullptr; t = ggml_get_next_tensor(params_ctx, t)) { params_tensor_set_.insert(t); } + params_tensor_set_dirty_ = false; + } + + std::vector collect_used_param_tensors(ggml_cgraph* gf) { + std::vector used_params; + rebuild_params_tensor_set(); + if (gf == nullptr || params_tensor_set_.empty()) { + return used_params; + } + + std::unordered_set seen_params; + const int n_leafs = sd::ggml_graph_cut::leaf_count(gf); + seen_params.reserve(static_cast(n_leafs)); + for (int i = 0; i < n_leafs; ++i) { + ggml_tensor* leaf = sd::ggml_graph_cut::leaf_tensor(gf, i); + ggml_tensor* param_leaf = leaf; + if (param_leaf != nullptr && params_tensor_set_.find(param_leaf) == params_tensor_set_.end()) { + param_leaf = param_leaf->view_src; + } + if (param_leaf != nullptr && + params_tensor_set_.find(param_leaf) != params_tensor_set_.end() && + seen_params.insert(param_leaf).second) { + used_params.push_back(param_leaf); + } + } + return used_params; + } + + bool prepare_execute_graph_weights(ggml_cgraph* gf, + std::vector& graph_param_tensors, + std::vector& params_to_prepare, + bool keep_compute_params) { + graph_param_tensors = collect_used_param_tensors(gf); + params_to_prepare.clear(); + params_to_prepare.reserve(graph_param_tensors.size()); + for (ggml_tensor* param : graph_param_tensors) { + if (param == nullptr) { + continue; + } + if (keep_compute_params && + kept_compute_param_tensor_set.find(param) != kept_compute_param_tensor_set.end()) { + continue; + } + params_to_prepare.push_back(param); + } + auto manager = weight_manager.lock(); + if (manager == nullptr) { + if (!params_to_prepare.empty()) { + LOG_ERROR("%s weight manager is not set for graph params", get_desc().c_str()); + return false; + } + return true; + } + + if (!manager->prepare_params(params_to_prepare)) { + LOG_ERROR("%s prepare graph weights failed", get_desc().c_str()); + return false; + } + for (ggml_tensor* param : params_to_prepare) { + if (param == nullptr) { + continue; + } + if (runner_param_tensor_set.insert(param).second) { + runner_param_tensors.push_back(param); + } + } + return true; + } + + void free_compute_backend_param_tensors(const std::vector& tensors) { + if (tensors.empty()) { + return; + } + auto manager = weight_manager.lock(); + if (manager != nullptr) { + manager->release_compute_backend_params(tensors); + } + } + + void free_params_backend_param_tensors(const std::vector& tensors) { + if (tensors.empty()) { + return; + } + auto manager = weight_manager.lock(); + if (manager != nullptr) { + manager->release_params_backend_params(tensors); + } } void prepare_build_in_tensor_before() { @@ -1932,6 +2007,10 @@ protected: } bool copy_cache_tensors_to_cache_buffer(const std::unordered_set* cache_keep_names = nullptr) { + if (cache_tensor_map.empty() && cache_keep_names == nullptr) { + return true; + } + ggml_context* old_cache_ctx = cache_ctx; ggml_backend_buffer_t old_cache_buffer = cache_buffer; cache_ctx = nullptr; @@ -2109,328 +2188,16 @@ protected: } } - bool offload_all_params() { - restore_partial_params(); - if (params_backend == runtime_backend) { - return true; - } - if (params_on_runtime_backend) { - return true; - } - GGML_ASSERT(runtime_params_buffer == nullptr); - int64_t t0 = ggml_time_ms(); - size_t num_tensors = ggml_tensor_num(offload_ctx); - if (num_tensors == 0) { - for (ggml_tensor* t = ggml_get_first_tensor(params_ctx); t != nullptr; t = ggml_get_next_tensor(params_ctx, t)) { - GGML_ASSERT(t->view_src == nullptr); - ggml_dup_tensor(offload_ctx, t); - } - } - num_tensors = ggml_tensor_num(offload_ctx); - GGML_ASSERT(num_tensors == ggml_tensor_num(params_ctx)); - - runtime_params_buffer = ggml_backend_alloc_ctx_tensors(offload_ctx, runtime_backend); - - if (runtime_params_buffer == nullptr) { - LOG_ERROR("%s alloc runtime params backend buffer failed, num_tensors = %i", - get_desc().c_str(), - num_tensors); - return false; - } - ggml_backend_buffer_set_usage(runtime_params_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - ggml_tensor* t = ggml_get_first_tensor(params_ctx); - ggml_tensor* offload_t = ggml_get_first_tensor(offload_ctx); - - while (t != nullptr && offload_t != nullptr) { - ggml_backend_tensor_copy(t, offload_t); - std::swap(t->buffer, offload_t->buffer); - std::swap(t->data, offload_t->data); - std::swap(t->extra, offload_t->extra); - - t = ggml_get_next_tensor(params_ctx, t); - offload_t = ggml_get_next_tensor(offload_ctx, offload_t); - } - - int64_t t1 = ggml_time_ms(); - - size_t params_buffer_size = ggml_backend_buffer_get_size(runtime_params_buffer); - LOG_INFO("%s offload params (%6.2f MB, %i tensors) to runtime backend (%s), taking %.2fs", - get_desc().c_str(), - params_buffer_size / (1024.f * 1024.f), - num_tensors, - ggml_backend_name(runtime_backend), - (t1 - t0) * 1.0f / 1000); - - params_on_runtime_backend = true; - - return true; - } - - bool offload_partial_params(const std::vector& tensors) { - restore_partial_params(); - if (params_backend == runtime_backend) { - return true; - } - if (tensors.empty()) { - return true; - } - GGML_ASSERT(!params_on_runtime_backend); - GGML_ASSERT(partial_runtime_params_buffer == nullptr); - - std::vector unique_tensors; - std::unordered_set seen_tensors; - unique_tensors.reserve(tensors.size()); - seen_tensors.reserve(tensors.size()); - for (ggml_tensor* tensor : tensors) { - if (tensor == nullptr) { - continue; - } - if (resident_param_set.find(tensor) != resident_param_set.end()) { - continue; - } - if (seen_tensors.insert(tensor).second) { - unique_tensors.push_back(tensor); - } - } - if (unique_tensors.empty()) { - return true; - } - - ggml_init_params params; - params.mem_size = std::max(1, unique_tensors.size()) * ggml_tensor_overhead(); - params.mem_buffer = nullptr; - params.no_alloc = true; - - partial_offload_ctx = ggml_init(params); - GGML_ASSERT(partial_offload_ctx != nullptr); - - partial_offload_pairs.clear(); - partial_offload_pairs.reserve(unique_tensors.size()); - - for (ggml_tensor* tensor : unique_tensors) { - GGML_ASSERT(tensor->view_src == nullptr); - ggml_tensor* offload_tensor = ggml_dup_tensor(partial_offload_ctx, tensor); - ggml_set_name(offload_tensor, tensor->name); - partial_offload_pairs.push_back({tensor, offload_tensor}); - } - - partial_runtime_params_buffer = ggml_backend_alloc_ctx_tensors(partial_offload_ctx, runtime_backend); - if (partial_runtime_params_buffer == nullptr) { - LOG_ERROR("%s alloc partial runtime params backend buffer failed, num_tensors = %zu", - get_desc().c_str(), - partial_offload_pairs.size()); - ggml_free(partial_offload_ctx); - partial_offload_ctx = nullptr; - partial_offload_pairs.clear(); - return false; - } - ggml_backend_buffer_set_usage(partial_runtime_params_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - for (auto& pair : partial_offload_pairs) { - ggml_tensor* tensor = pair.first; - ggml_tensor* offload_tensor = pair.second; - - ggml_backend_tensor_copy(tensor, offload_tensor); - std::swap(tensor->buffer, offload_tensor->buffer); - std::swap(tensor->data, offload_tensor->data); - std::swap(tensor->extra, offload_tensor->extra); - } - - size_t params_buffer_size = ggml_backend_buffer_get_size(partial_runtime_params_buffer); - LOG_DEBUG("%s offload partial params (%6.2f MB, %zu tensors) to runtime backend (%s)", - get_desc().c_str(), - params_buffer_size / (1024.f * 1024.f), - partial_offload_pairs.size(), - ggml_backend_name(runtime_backend)); - - return true; - } - - void restore_all_params() { - restore_partial_params(); - if (!params_on_runtime_backend) { - return; - } - ggml_tensor* t = ggml_get_first_tensor(params_ctx); - ggml_tensor* offload_t = ggml_get_first_tensor(offload_ctx); - - while (t != nullptr && offload_t != nullptr) { - t->buffer = offload_t->buffer; - t->data = offload_t->data; - t->extra = offload_t->extra; - offload_t->buffer = nullptr; - offload_t->data = nullptr; - offload_t->extra = nullptr; - - t = ggml_get_next_tensor(params_ctx, t); - offload_t = ggml_get_next_tensor(offload_ctx, offload_t); - } - - if (runtime_params_buffer != nullptr) { - ggml_backend_buffer_free(runtime_params_buffer); - runtime_params_buffer = nullptr; - } - params_on_runtime_backend = false; - } - - void restore_partial_params() { - if (partial_offload_pairs.empty()) { - if (partial_runtime_params_buffer != nullptr) { - ggml_backend_buffer_free(partial_runtime_params_buffer); - partial_runtime_params_buffer = nullptr; - } - if (partial_offload_ctx != nullptr) { - ggml_free(partial_offload_ctx); - partial_offload_ctx = nullptr; - } - return; - } - - for (auto& pair : partial_offload_pairs) { - ggml_tensor* tensor = pair.first; - ggml_tensor* offload_tensor = pair.second; - - tensor->buffer = offload_tensor->buffer; - tensor->data = offload_tensor->data; - tensor->extra = offload_tensor->extra; - offload_tensor->buffer = nullptr; - offload_tensor->data = nullptr; - offload_tensor->extra = nullptr; - } - - if (partial_runtime_params_buffer != nullptr) { - ggml_backend_buffer_free(partial_runtime_params_buffer); - partial_runtime_params_buffer = nullptr; - } - partial_offload_pairs.clear(); - - if (partial_offload_ctx != nullptr) { - ggml_free(partial_offload_ctx); - partial_offload_ctx = nullptr; - } - } - - bool offload_resident_params(const std::vector& tensors) { - if (params_backend == runtime_backend) { - return true; - } - if (tensors.empty()) { - return true; - } - GGML_ASSERT(resident_runtime_params_buffer == nullptr); - GGML_ASSERT(resident_offload_ctx == nullptr); - GGML_ASSERT(resident_offload_pairs.empty()); - GGML_ASSERT(resident_param_set.empty()); - - std::vector unique_tensors; - std::unordered_set seen; - unique_tensors.reserve(tensors.size()); - seen.reserve(tensors.size()); - for (ggml_tensor* t : tensors) { - if (t == nullptr) - continue; - if (seen.insert(t).second) - unique_tensors.push_back(t); - } - if (unique_tensors.empty()) - return true; - - ggml_init_params init = {}; - init.mem_size = std::max(1, unique_tensors.size()) * ggml_tensor_overhead(); - init.mem_buffer = nullptr; - init.no_alloc = true; - resident_offload_ctx = ggml_init(init); - GGML_ASSERT(resident_offload_ctx != nullptr); - - resident_offload_pairs.reserve(unique_tensors.size()); - for (ggml_tensor* t : unique_tensors) { - GGML_ASSERT(t->view_src == nullptr); - ggml_tensor* twin = ggml_dup_tensor(resident_offload_ctx, t); - ggml_set_name(twin, t->name); - resident_offload_pairs.push_back({t, twin}); - } - - resident_runtime_params_buffer = ggml_backend_alloc_ctx_tensors(resident_offload_ctx, runtime_backend); - if (resident_runtime_params_buffer == nullptr) { - LOG_ERROR("%s alloc resident runtime params backend buffer failed, num_tensors = %zu", - get_desc().c_str(), resident_offload_pairs.size()); - ggml_free(resident_offload_ctx); - resident_offload_ctx = nullptr; - resident_offload_pairs.clear(); - return false; - } - ggml_backend_buffer_set_usage(resident_runtime_params_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - - for (auto& pair : resident_offload_pairs) { - ggml_tensor* t = pair.first; - ggml_tensor* twin = pair.second; - ggml_backend_tensor_copy(t, twin); - std::swap(t->buffer, twin->buffer); - std::swap(t->data, twin->data); - std::swap(t->extra, twin->extra); - resident_param_set.insert(t); - } - ggml_backend_synchronize(runtime_backend); - - size_t sz = ggml_backend_buffer_get_size(resident_runtime_params_buffer); - LOG_INFO("%s offload resident params (%6.2f MB, %zu tensors) to runtime backend (%s)", - get_desc().c_str(), - sz / (1024.f * 1024.f), - resident_offload_pairs.size(), - ggml_backend_name(runtime_backend)); - return true; - } - - void restore_resident_params() { - if (resident_offload_pairs.empty()) { - if (resident_runtime_params_buffer != nullptr) { - ggml_backend_buffer_free(resident_runtime_params_buffer); - resident_runtime_params_buffer = nullptr; - } - if (resident_offload_ctx != nullptr) { - ggml_free(resident_offload_ctx); - resident_offload_ctx = nullptr; - } - resident_param_set.clear(); - resident_state_token = 0; - return; - } - for (auto& pair : resident_offload_pairs) { - ggml_tensor* t = pair.first; - ggml_tensor* twin = pair.second; - t->buffer = twin->buffer; - t->data = twin->data; - t->extra = twin->extra; - twin->buffer = nullptr; - twin->data = nullptr; - twin->extra = nullptr; - } - if (resident_runtime_params_buffer != nullptr) { - ggml_backend_buffer_free(resident_runtime_params_buffer); - resident_runtime_params_buffer = nullptr; - } - resident_offload_pairs.clear(); - if (resident_offload_ctx != nullptr) { - ggml_free(resident_offload_ctx); - resident_offload_ctx = nullptr; - } - resident_param_set.clear(); - resident_state_token = 0; - } - bool should_use_graph_cut_segmented_compute(const GraphCutPlan& plan) { return plan.has_cuts && plan.valid && max_graph_vram_bytes > 0 && plan.segments.size() > 1 && - params_backend != runtime_backend && !sd_backend_is_cpu(runtime_backend); } bool can_attempt_graph_cut_segmented_compute() const { return max_graph_vram_bytes > 0 && - params_backend != runtime_backend && !sd_backend_is_cpu(runtime_backend); } @@ -2440,18 +2207,12 @@ protected: GGML_ASSERT(plan_out != nullptr); GGML_ASSERT(gf != nullptr); - // Keep the plan and resident params under the same live-VRAM cap. - // Add back our own resident buffer so we don't see chunk-K's - // allocation as "taken" VRAM and shrink the budget on every step. size_t effective_budget = max_graph_vram_bytes; if (stream_layers_enabled && max_graph_vram_bytes > 0 && runtime_backend != nullptr) { ggml_backend_dev_t dev = ggml_backend_get_device(runtime_backend); if (dev != nullptr && ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_CPU) { size_t free_vram = 0, total_vram = 0; ggml_backend_dev_memory(dev, &free_vram, &total_vram); - if (resident_runtime_params_buffer != nullptr) { - free_vram += ggml_backend_buffer_get_size(resident_runtime_params_buffer); - } constexpr size_t safety_margin = 512ull * 1024 * 1024; size_t free_clamp = (free_vram > safety_margin) ? (free_vram - safety_margin) : 0; if (free_clamp < effective_budget) { @@ -2500,6 +2261,9 @@ protected: planner_budget, params_tensor_set_, get_desc().c_str()); + if (stream_layers_enabled) { + sd::ggml_graph_cut::annotate_residency(*plan_out, effective_budget); + } if (stream_layers_enabled) { if (budget_increased) { LOG_INFO("%s streaming budget = %.2f MB", @@ -2635,310 +2399,173 @@ protected: template std::optional> execute_graph(ggml_cgraph* gf, int n_threads, - bool free_compute_buffer_immediately, - const std::vector& runtime_param_tensors, + bool free_compute_buffer, + bool free_compute_params, bool preserve_backend_tensor_data_map, bool no_return = false, const std::unordered_set* cache_keep_names = nullptr) { - int64_t t_execute_begin = ggml_time_ms(); - const bool use_partial_param_offload = !runtime_param_tensors.empty(); - int64_t t_offload_begin = ggml_time_ms(); - if (use_partial_param_offload) { - if (!offload_partial_params(runtime_param_tensors)) { - LOG_ERROR("%s offload partial params to runtime backend failed", get_desc().c_str()); - return std::nullopt; - } - } else { - if (!offload_all_params()) { - LOG_ERROR("%s offload params to runtime backend failed", get_desc().c_str()); - return std::nullopt; - } - } - int64_t t_offload_end = ggml_time_ms(); - - int64_t t_alloc_begin = ggml_time_ms(); - if (!alloc_compute_buffer(gf)) { - LOG_ERROR("%s alloc compute buffer failed", get_desc().c_str()); - if (use_partial_param_offload) { - restore_partial_params(); - } + std::vector graph_param_tensors; + std::vector params_to_prepare; + if (!prepare_execute_graph_weights(gf, graph_param_tensors, params_to_prepare, !free_compute_params)) { return std::nullopt; } + struct GraphWeightDoneGuard { + GraphWeightDoneGuard(GGMLRunner* runner, const std::vector* tensors) + : runner(runner), + tensors(tensors) {} + + GGMLRunner* runner = nullptr; + const std::vector* tensors = nullptr; + bool enabled = true; + + ~GraphWeightDoneGuard() { + if (enabled && runner != nullptr && tensors != nullptr) { + runner->free_compute_backend_param_tensors(*tensors); + } + } + + void dismiss() { enabled = false; } + + GraphWeightDoneGuard(const GraphWeightDoneGuard&) = delete; + GraphWeightDoneGuard& operator=(const GraphWeightDoneGuard&) = delete; + }; + GraphWeightDoneGuard graph_weight_done_guard(this, ¶ms_to_prepare); + + if (!alloc_compute_buffer(gf)) { + LOG_ERROR("%s alloc compute buffer failed", get_desc().c_str()); + return std::nullopt; + } + struct ComputeBufferGuard { + ComputeBufferGuard(GGMLRunner* runner, bool enabled) + : runner(runner), + enabled(enabled) {} + + GGMLRunner* runner = nullptr; + bool enabled = false; + + ~ComputeBufferGuard() { + if (enabled && runner != nullptr) { + runner->free_compute_buffer(); + } + } + + ComputeBufferGuard(const ComputeBufferGuard&) = delete; + ComputeBufferGuard& operator=(const ComputeBufferGuard&) = delete; + }; + ComputeBufferGuard compute_buffer_guard(this, free_compute_buffer); if (!ggml_gallocr_alloc_graph(compute_allocr, gf)) { LOG_ERROR("%s alloc compute graph failed", get_desc().c_str()); - if (free_compute_buffer_immediately) { - free_compute_buffer(); - } else if (use_partial_param_offload) { - restore_partial_params(); - } return std::nullopt; } - int64_t t_alloc_end = ggml_time_ms(); - int64_t t_copy_begin = ggml_time_ms(); copy_data_to_backend_tensor(gf, !preserve_backend_tensor_data_map); - int64_t t_copy_end = ggml_time_ms(); if (sd_backend_is_cpu(runtime_backend)) { sd_backend_cpu_set_n_threads(runtime_backend, n_threads); } - int64_t t_compute_begin = ggml_time_ms(); - ggml_status status = ggml_backend_graph_compute(runtime_backend, gf); - int64_t t_compute_end = ggml_time_ms(); + ggml_status status = ggml_backend_graph_compute(runtime_backend, gf); if (status != GGML_STATUS_SUCCESS) { LOG_ERROR("%s compute failed: %s", get_desc().c_str(), ggml_status_to_string(status)); - if (free_compute_buffer_immediately) { - free_compute_buffer(); - } else if (use_partial_param_offload) { - restore_partial_params(); - } return std::nullopt; } - std::unordered_set debug_graph_tensor_set; - const int n_debug_leafs = sd::ggml_graph_cut::leaf_count(gf); - const int n_debug_nodes = ggml_graph_n_nodes(gf); - debug_graph_tensor_set.reserve(static_cast(n_debug_leafs + n_debug_nodes)); - for (int i = 0; i < n_debug_leafs; ++i) { - debug_graph_tensor_set.insert(sd::ggml_graph_cut::leaf_tensor(gf, i)); - } - for (int i = 0; i < n_debug_nodes; ++i) { - debug_graph_tensor_set.insert(ggml_graph_node(gf, i)); + if (!debug_tensors.empty()) { + std::unordered_set debug_graph_tensor_set; + const int n_debug_leafs = sd::ggml_graph_cut::leaf_count(gf); + const int n_debug_nodes = ggml_graph_n_nodes(gf); + debug_graph_tensor_set.reserve(static_cast(n_debug_leafs + n_debug_nodes)); + for (int i = 0; i < n_debug_leafs; ++i) { + debug_graph_tensor_set.insert(sd::ggml_graph_cut::leaf_tensor(gf, i)); + } + for (int i = 0; i < n_debug_nodes; ++i) { + debug_graph_tensor_set.insert(ggml_graph_node(gf, i)); + } + + for (const auto& entry : debug_tensors) { + auto tensor = entry.first; + if (tensor == nullptr) { + continue; + } + if (debug_graph_tensor_set.find(tensor) == debug_graph_tensor_set.end()) { + continue; + } + ggml_backend_buffer_t tensor_buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; + if (tensor_buf == nullptr) { + LOG_WARN("%s skip debug tensor '%s': tensor buffer not set", + get_desc().c_str(), + entry.second.c_str()); + continue; + } + if (tensor->type != GGML_TYPE_F32) { + LOG_WARN("%s skip debug tensor '%s': only GGML_TYPE_F32 is supported, got %s", + get_desc().c_str(), + entry.second.c_str(), + ggml_type_name(tensor->type)); + continue; + } + auto debug_tensor = sd::make_sd_tensor_from_ggml(tensor); + print_sd_tensor(debug_tensor, false, entry.second.c_str()); + } } - for (const auto& entry : debug_tensors) { - auto tensor = entry.first; - if (tensor == nullptr) { - continue; - } - if (debug_graph_tensor_set.find(tensor) == debug_graph_tensor_set.end()) { - continue; - } - ggml_backend_buffer_t tensor_buf = tensor->view_src ? tensor->view_src->buffer : tensor->buffer; - if (tensor_buf == nullptr) { - LOG_WARN("%s skip debug tensor '%s': tensor buffer not set", - get_desc().c_str(), - entry.second.c_str()); - continue; - } - if (tensor->type != GGML_TYPE_F32) { - LOG_WARN("%s skip debug tensor '%s': only GGML_TYPE_F32 is supported, got %s", - get_desc().c_str(), - entry.second.c_str(), - ggml_type_name(tensor->type)); - continue; - } - auto debug_tensor = sd::make_sd_tensor_from_ggml(tensor); - print_sd_tensor(debug_tensor, false, entry.second.c_str()); - } - - int64_t t_cache_begin = ggml_time_ms(); if (!copy_cache_tensors_to_cache_buffer(cache_keep_names)) { - if (free_compute_buffer_immediately) { - free_compute_buffer(); - } else if (use_partial_param_offload) { - restore_partial_params(); - } return std::nullopt; } - int64_t t_cache_end = ggml_time_ms(); - auto result = ggml_get_tensor(compute_ctx, final_result_name.c_str()); + auto result = ggml_get_tensor(compute_ctx, final_result_name.c_str()); std::optional> output; if (!no_return) { output = read_graph_tensor(result, "output"); if (!output.has_value()) { - if (free_compute_buffer_immediately) { - free_compute_buffer(); - } else if (use_partial_param_offload) { - restore_partial_params(); - } return std::nullopt; } } else { output = sd::Tensor(); } - if (free_compute_buffer_immediately) { - free_compute_buffer(); - } else if (use_partial_param_offload) { - restore_partial_params(); - } - if (use_partial_param_offload) { - LOG_DEBUG("%s execute_graph timing: offload=%lld ms alloc=%lld ms copy_in=%lld ms compute=%lld ms cache=%lld ms total=%lld ms", - get_desc().c_str(), - t_offload_end - t_offload_begin, - t_alloc_end - t_alloc_begin, - t_copy_end - t_copy_begin, - t_compute_end - t_compute_begin, - t_cache_end - t_cache_begin, - ggml_time_ms() - t_execute_begin); - } - return output; - } - - template - std::optional> compute_with_graph_cuts(ggml_cgraph* gf, - const GraphCutPlan& plan, - int n_threads, - bool free_compute_buffer_immediately, - bool no_return = false) { - GGML_ASSERT(gf != nullptr); - - free_compute_buffer(); - free_cache_ctx_and_buffer(); - - std::unordered_map persistent_externals; - snapshot_persistent_externals(plan, gf, persistent_externals); - - std::optional> output = sd::Tensor(); - for (size_t seg_idx = 0; seg_idx < plan.segments.size(); ++seg_idx) { - int64_t t_segment_begin = ggml_time_ms(); - const auto& segment = plan.segments[seg_idx]; - auto future_cut_names = sd::ggml_graph_cut::collect_future_input_names(gf, plan, seg_idx); - LOG_DEBUG("%s graph cut executing segment %zu/%zu: %s", - get_desc().c_str(), - seg_idx + 1, - plan.segments.size(), - segment.group_name.c_str()); - - reset_segment_runtime_tensors(segment, gf, &persistent_externals); - if (!bind_segment_cached_inputs(gf, segment)) { - free_cache_ctx_and_buffer(); - free_compute_buffer(); - free_compute_ctx(); - return std::nullopt; - } - - const bool is_last_segment = seg_idx + 1 == plan.segments.size(); - if (!is_last_segment) { - for (size_t output_idx = 0; output_idx < segment.output_node_indices.size(); ++output_idx) { - ggml_tensor* output_tensor = sd::ggml_graph_cut::output_tensor(gf, segment, output_idx); - if (output_tensor != nullptr && - sd::ggml_graph_cut::is_graph_cut_tensor(output_tensor) && - future_cut_names.find(output_tensor->name) != future_cut_names.end()) { - cache(output_tensor->name, output_tensor); - } + if (!free_compute_params) { + for (ggml_tensor* param : params_to_prepare) { + if (param == nullptr) { + continue; } + kept_compute_param_tensor_set.insert(param); } - - ggml_context* segment_graph_ctx = nullptr; - ggml_cgraph* segment_graph = sd::ggml_graph_cut::build_segment_graph(gf, segment, &segment_graph_ctx); - auto segment_output = execute_graph(segment_graph, - n_threads, - true, - sd::ggml_graph_cut::runtime_param_tensors(gf, segment, get_desc().c_str()), - true, - !is_last_segment || no_return, - &future_cut_names); - ggml_free(segment_graph_ctx); - if (!segment_output.has_value()) { - free_cache_ctx_and_buffer(); - free_compute_buffer(); - free_compute_ctx(); - return std::nullopt; - } - output = std::move(segment_output); + graph_weight_done_guard.dismiss(); } - - backend_tensor_data_map.clear(); - free_cache_ctx_and_buffer(); - free_compute_ctx(); return output; } -public: - void release_streaming_residency() { - restore_resident_params(); - } - template - std::optional> compute_streaming_segments(ggml_cgraph* gf, + std::optional> compute_graph_cut_segments(ggml_cgraph* gf, const GraphCutPlan& plan, - size_t residency_budget_bytes, int n_threads, - bool free_compute_buffer_immediately, + bool log_residency, bool no_return = false) { GGML_ASSERT(gf != nullptr); - // Runtime LoRA composes `weight + diff` in the compute graph via - // ggml_add; the resident weight tensor's data is never mutated, so - // chunk-K residency stays valid across sampling steps. - // Reserve room for the worst merged segment so chunk-K can't grow - // large enough to starve later partial-param allocations. - size_t worst_merged_segment_footprint = 0; - for (const auto& seg : plan.segments) { - const size_t fp = seg.input_param_bytes + - seg.compute_buffer_size + - seg.output_bytes + - seg.input_previous_cut_bytes + - seg.input_external_bytes; - if (fp > worst_merged_segment_footprint) { - worst_merged_segment_footprint = fp; - } - } - const size_t residency_budget_for_annotate = - residency_budget_bytes > worst_merged_segment_footprint - ? residency_budget_bytes - worst_merged_segment_footprint - : 0; - - sd::ggml_graph_cut::Plan& base_plan = graph_cut_plan_cache_.graph_cut_plan; - if (base_plan.available) { - sd::ggml_graph_cut::annotate_residency(base_plan, residency_budget_for_annotate); - - std::vector resident_params; - uint64_t token = 0; - for (const auto& segment : base_plan.segments) { - if (segment.residency != sd::ggml_graph_cut::SegmentResidency::RESIDENT) { - continue; - } - auto seg_params = sd::ggml_graph_cut::param_tensors(gf, segment); - for (ggml_tensor* t : seg_params) { - if (t == nullptr) - continue; - resident_params.push_back(t); - token ^= reinterpret_cast(t) * 0x9E3779B97F4A7C15ull; - } - } - if (token != resident_state_token) { - restore_resident_params(); - if (!resident_params.empty()) { - if (offload_resident_params(resident_params)) { - resident_state_token = token; - } else { - LOG_ERROR("%s chunk-K: resident offload failed; continuing with per-segment streaming", - get_desc().c_str()); - restore_resident_params(); - } - } - } - } - free_compute_buffer(); free_cache_ctx_and_buffer(); - layer_registry_.move_layer_to_gpu("_global"); - std::unordered_map persistent_externals; snapshot_persistent_externals(plan, gf, persistent_externals); std::optional> output = sd::Tensor(); for (size_t seg_idx = 0; seg_idx < plan.segments.size(); ++seg_idx) { - int64_t t_segment_begin = ggml_time_ms(); - const auto& segment = plan.segments[seg_idx]; - const bool is_last = seg_idx + 1 == plan.segments.size(); - auto future_cut_names = sd::ggml_graph_cut::collect_future_input_names(gf, plan, seg_idx); - - LOG_DEBUG("%s streaming-cut executing segment %zu/%zu: %s (residency=%s)", - get_desc().c_str(), - seg_idx + 1, - plan.segments.size(), - segment.group_name.c_str(), - segment.residency == sd::ggml_graph_cut::SegmentResidency::RESIDENT ? "RESIDENT" : "STREAMED"); - - if (!layer_registry_.move_layer_to_gpu(segment.group_name)) { - LOG_DEBUG("%s streaming: no registry entry for group '%s' (using upstream offload path)", + const auto& segment = plan.segments[seg_idx]; + const bool is_last = seg_idx + 1 == plan.segments.size(); + auto future_cut_names = sd::ggml_graph_cut::collect_future_input_names(gf, plan, seg_idx); + if (log_residency) { + LOG_DEBUG("%s graph cut executing segment %zu/%zu: %s (residency=%s)", get_desc().c_str(), + seg_idx + 1, + plan.segments.size(), + segment.group_name.c_str(), + segment.residency == sd::ggml_graph_cut::SegmentResidency::RESIDENT ? "RESIDENT" : "STREAMED"); + } else { + LOG_DEBUG("%s graph cut executing segment %zu/%zu: %s", + get_desc().c_str(), + seg_idx + 1, + plan.segments.size(), segment.group_name.c_str()); } @@ -2952,23 +2579,24 @@ public: if (!is_last) { for (size_t output_idx = 0; output_idx < segment.output_node_indices.size(); ++output_idx) { - ggml_tensor* out_tensor = sd::ggml_graph_cut::output_tensor(gf, segment, output_idx); - if (out_tensor != nullptr && - sd::ggml_graph_cut::is_graph_cut_tensor(out_tensor) && - future_cut_names.find(out_tensor->name) != future_cut_names.end()) { - cache(out_tensor->name, out_tensor); + ggml_tensor* output_tensor = sd::ggml_graph_cut::output_tensor(gf, segment, output_idx); + if (output_tensor != nullptr && + sd::ggml_graph_cut::is_graph_cut_tensor(output_tensor) && + future_cut_names.find(output_tensor->name) != future_cut_names.end()) { + cache(output_tensor->name, output_tensor); } } } ggml_context* segment_graph_ctx = nullptr; ggml_cgraph* segment_graph = sd::ggml_graph_cut::build_segment_graph(gf, segment, &segment_graph_ctx); + const bool keep_segment_params = segment.residency == sd::ggml_graph_cut::SegmentResidency::RESIDENT; auto segment_output = execute_graph(segment_graph, n_threads, - /*free_compute_buffer_immediately=*/true, - sd::ggml_graph_cut::runtime_param_tensors(gf, segment, get_desc().c_str()), - /*preserve_backend_tensor_data_map=*/true, - /*no_return=*/!is_last || no_return, + true, + !keep_segment_params, + true, + !is_last || no_return, &future_cut_names); ggml_free(segment_graph_ctx); if (!segment_output.has_value()) { @@ -2978,11 +2606,6 @@ public: return std::nullopt; } output = std::move(segment_output); - - if (segment.residency == sd::ggml_graph_cut::SegmentResidency::STREAMED) { - layer_registry_.move_layer_to_cpu(segment.group_name); - } - (void)t_segment_begin; } backend_tensor_data_map.clear(); @@ -2991,21 +2614,29 @@ public: return output; } +public: + void runner_done() { + free_compute_buffer(); + std::vector tensors_to_release = std::move(this->runner_param_tensors); + this->runner_param_tensors.clear(); + runner_param_tensor_set.clear(); + kept_compute_param_tensor_set.clear(); + free_compute_backend_param_tensors(tensors_to_release); + free_params_backend_param_tensors(tensors_to_release); + } + public: virtual std::string get_desc() = 0; - GGMLRunner(ggml_backend_t backend, ggml_backend_t params_backend) - : params_backend(params_backend), - runtime_backend(backend) { + GGMLRunner(ggml_backend_t backend, + std::shared_ptr manager = nullptr) + : runtime_backend(backend), + weight_manager(manager) { GGML_ASSERT(runtime_backend != nullptr); - GGML_ASSERT(params_backend != nullptr); alloc_params_ctx(); - layer_registry_.set_backends(runtime_backend, params_backend); } virtual ~GGMLRunner() { - restore_resident_params(); - free_params_buffer(); free_compute_buffer(); free_params_ctx(); free_compute_ctx(); @@ -3028,6 +2659,9 @@ public: runner_ctx.cache_tensor = [this](const std::string& name, ggml_tensor* tensor) { this->cache(name, tensor); }; + runner_ctx.set_backend_tensor_data = [this](ggml_tensor* tensor, const void* data) { + this->set_backend_tensor_data(tensor, data); + }; return runner_ctx; } @@ -3036,74 +2670,7 @@ public: alloc_compute_ctx(); } - bool alloc_params_buffer() { - size_t num_tensors = ggml_tensor_num(params_ctx); - if (num_tensors > 0) { - // ggml_backend_alloc_ctx_tensors fails when all tensors are already allocated - // (typical for memory-mapped weights). See ggml-alloc.c n_buffers==0 branch. - bool all_have_data = true; - for (ggml_tensor* t = ggml_get_first_tensor(params_ctx); t != nullptr; t = ggml_get_next_tensor(params_ctx, t)) { - if (t->data == nullptr) { - all_have_data = false; - break; - } - } - if (all_have_data) { - LOG_DEBUG("%s all params already mmap-allocated (no separate buffer needed)", get_desc().c_str()); - params_buffer = nullptr; - rebuild_params_tensor_set(); - return true; - } - } else { - LOG_DEBUG("%s skipping params allocation (no tensors)", get_desc().c_str()); - return true; - } - // Pinned host buffer when CPU-offloaded for DMA-direct H2D. - ggml_backend_buffer_type_t params_buft = nullptr; - if (params_backend != runtime_backend) { - ggml_backend_dev_t runtime_dev = ggml_backend_get_device(runtime_backend); - if (runtime_dev != nullptr) { - params_buft = ggml_backend_dev_host_buffer_type(runtime_dev); - } - } - if (params_buft == nullptr) { - params_buft = ggml_backend_get_default_buffer_type(params_backend); - } - params_buffer = ggml_backend_alloc_ctx_tensors_from_buft(params_ctx, params_buft); - if (params_buffer == nullptr) { - LOG_ERROR("%s alloc params backend buffer failed, num_tensors = %i", - get_desc().c_str(), - num_tensors); - return false; - } - rebuild_params_tensor_set(); - ggml_backend_buffer_set_usage(params_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); - size_t params_buffer_size = ggml_backend_buffer_get_size(params_buffer); - LOG_DEBUG("%s params backend buffer size = % 6.2f MB(%s) (%i tensors)", - get_desc().c_str(), - params_buffer_size / (1024.f * 1024.f), - sd_backend_is_cpu(params_backend) ? "RAM" : "VRAM", - num_tensors); - return true; - } - - void free_params_buffer() { - // Restore swapped resident params before freeing their backing buffer. - restore_resident_params(); - if (params_buffer != nullptr) { - ggml_backend_buffer_free(params_buffer); - params_buffer = nullptr; - } - observed_max_effective_budget_ = 0; - } - - size_t get_params_buffer_size() { - if (params_buffer != nullptr) { - return ggml_backend_buffer_get_size(params_buffer); - } - return 0; - } - +public: void free_cache_ctx_and_buffer() { free_cache_buffer(); free_cache_ctx(); @@ -3114,8 +2681,6 @@ public: ggml_gallocr_free(compute_allocr); compute_allocr = nullptr; } - restore_partial_params(); - restore_all_params(); } // do copy after alloc graph @@ -3180,47 +2745,57 @@ public: template std::optional> compute(get_graph_cb_t get_graph, int n_threads, - bool free_compute_buffer_immediately, - bool no_return = false) { + bool auto_free = true, + bool free_compute_buffer = true, + bool free_compute_params = true, + bool no_return = false) { + struct RunnerDoneGuard { + RunnerDoneGuard(GGMLRunner* runner, bool enabled) + : runner(runner), + enabled(enabled) {} + + ~RunnerDoneGuard() { + if (enabled && runner != nullptr) { + runner->runner_done(); + } + } + + RunnerDoneGuard(const RunnerDoneGuard&) = delete; + RunnerDoneGuard& operator=(const RunnerDoneGuard&) = delete; + + GGMLRunner* runner = nullptr; + bool enabled = false; + }; + RunnerDoneGuard runner_done_guard(this, auto_free); + ggml_cgraph* gf = nullptr; if (!prepare_compute_graph(get_graph, &gf)) { return std::nullopt; } GGML_ASSERT(gf != nullptr); + rebuild_params_tensor_set(); if (can_attempt_graph_cut_segmented_compute()) { GraphCutPlan plan; - size_t effective_graph_vram_bytes = 0; - if (!resolve_graph_cut_plan(gf, &plan, &effective_graph_vram_bytes)) { + if (!resolve_graph_cut_plan(gf, &plan)) { free_compute_ctx(); return std::nullopt; } if (should_use_graph_cut_segmented_compute(plan)) { - if (stream_layers_enabled) { - return compute_streaming_segments(gf, - plan, - effective_graph_vram_bytes, - n_threads, - free_compute_buffer_immediately, - no_return); - } - return compute_with_graph_cuts(gf, - plan, - n_threads, - free_compute_buffer_immediately, - no_return); + return compute_graph_cut_segments(gf, + plan, + n_threads, + stream_layers_enabled, + no_return); } } - if (!alloc_compute_buffer(gf)) { - LOG_ERROR("%s alloc compute buffer failed", get_desc().c_str()); - return std::nullopt; - } return execute_graph(gf, n_threads, - free_compute_buffer_immediately, - {}, + free_compute_buffer, + free_compute_params, false, - no_return); + no_return, + nullptr); } void set_flash_attention_enabled(bool enabled) { @@ -3247,16 +2822,6 @@ public: void set_stream_layers_enabled(bool enabled) { stream_layers_enabled = enabled; } - - sd::layer_registry::LayerRegistry& get_layer_registry() { return layer_registry_; } - - ggml_backend_t get_runtime_backend() { - return runtime_backend; - } - - ggml_backend_t get_params_backend() { - return params_backend; - } }; class GGMLBlock { diff --git a/otherarch/sdcpp/src/core/ggml_extend_backend.cpp b/otherarch/sdcpp/src/core/ggml_extend_backend.cpp index d085129db..f3e2cceba 100644 --- a/otherarch/sdcpp/src/core/ggml_extend_backend.cpp +++ b/otherarch/sdcpp/src/core/ggml_extend_backend.cpp @@ -45,6 +45,10 @@ static bool is_default_backend_token(const std::string& name) { return lower.empty() || lower == "default" || lower == "auto"; } +static bool is_disk_backend_token(const std::string& name) { + return lower_copy(trim_copy(name)) == "disk"; +} + static bool parse_backend_module(const std::string& raw_name, SDBackendModule* module) { std::string name = lower_copy(trim_copy(raw_name)); name.erase(std::remove(name.begin(), name.end(), '-'), name.end()); @@ -200,6 +204,36 @@ void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value } } +bool add_rpc_devices(const std::string& servers) { + const std::string in = trim_copy(servers); + if (in.empty()) { + return true; + } + auto rpc_servers = split_copy(in, ','); + if (rpc_servers.empty()) { + LOG_ERROR("invalid RPC servers specification: '%s'", servers.c_str()); + return false; + } + ggml_backend_reg_t rpc_reg = ggml_backend_reg_by_name("RPC"); + if (!rpc_reg) { + LOG_ERROR("RPC backend not found, cannot add RPC servers"); + return false; + } + typedef ggml_backend_reg_t (*ggml_backend_rpc_add_server_t)(const char* endpoint); + ggml_backend_rpc_add_server_t ggml_backend_rpc_add_server_fn = (ggml_backend_rpc_add_server_t)ggml_backend_reg_get_proc_address(rpc_reg, "ggml_backend_rpc_add_server"); + if (!ggml_backend_rpc_add_server_fn) { + LOG_ERROR("RPC backend does not have ggml_backend_rpc_add_server function, cannot add RPC servers"); + return false; + } + for (const auto& server : rpc_servers) { + LOG_INFO("Adding RPC server: %s", server.c_str()); + auto reg = ggml_backend_rpc_add_server_fn(server.c_str()); + // no return value to check for success but should print errors from the RPC backend if it fails to add the server + ggml_backend_register(reg); + } + return true; +} + static void ggml_backend_load_all_once() { // If the registry already has devices and the CPU backend is present, // assume either static registration or explicit host-side preloading has @@ -246,7 +280,7 @@ static std::string get_default_backend_name() { return resolve_first_device_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); } -static std::string sd_resolve_backend_name(const std::string& name) { +std::string sd_backend_resolve_name(const std::string& name) { ggml_backend_load_all_once(); std::string requested = trim_copy(name); std::string lower = lower_copy(requested); @@ -284,7 +318,7 @@ static std::string sd_resolve_backend_name(const std::string& name) { } static bool backend_name_exists(const std::string& name) { - return !sd_resolve_backend_name(name).empty(); + return !sd_backend_resolve_name(name).empty(); } static ggml_backend_t init_named_backend(const std::string& name) { @@ -294,7 +328,7 @@ static ggml_backend_t init_named_backend(const std::string& name) { return ggml_backend_init_best(); } - std::string resolved = sd_resolve_backend_name(name); + std::string resolved = sd_backend_resolve_name(name); if (resolved.empty()) { return nullptr; } @@ -504,6 +538,9 @@ ggml_backend_t SDBackendManager::params_backend(SDBackendModule module) { if (name.empty()) { return runtime_backend(module); } + if (is_disk_backend_token(name)) { + return runtime_backend(module); + } return init_cached_backend(name); } @@ -515,6 +552,10 @@ bool SDBackendManager::params_backend_is_cpu(SDBackendModule module) { return sd_backend_is_cpu(params_backend(module)); } +bool SDBackendManager::params_backend_is_disk(SDBackendModule module) const { + return is_disk_backend_token(params_assignment_.get(module)); +} + bool SDBackendManager::runtime_backend_supports_host_buffer(SDBackendModule module) { ggml_backend_t backend = runtime_backend(module); if (backend == nullptr) { @@ -534,10 +575,6 @@ bool SDBackendManager::runtime_backend_supports_host_buffer(SDBackendModule modu bool SDBackendManager::init(const char* backend_spec, const char* params_backend_spec, - bool offload_params_to_cpu, - bool keep_clip_on_cpu, - bool keep_vae_on_cpu, - bool keep_control_net_on_cpu, std::string* error) { reset(); @@ -548,31 +585,21 @@ bool SDBackendManager::init(const char* backend_spec, return false; } - if (runtime_assignment_.empty()) { - if (keep_clip_on_cpu) { - runtime_assignment_.set_module(SDBackendModule::TE, "cpu"); - } - if (keep_vae_on_cpu) { - runtime_assignment_.set_module(SDBackendModule::VAE, "cpu"); - } - if (keep_control_net_on_cpu) { - runtime_assignment_.set_module(SDBackendModule::CONTROL_NET, "cpu"); - } - } - - if (params_assignment_.empty() && offload_params_to_cpu) { - params_assignment_.set_default("cpu"); - } - return validate(error); } bool SDBackendManager::validate(std::string* error) const { - auto validate_name = [&](const std::string& name) -> bool { + auto validate_runtime_name = [&](const std::string& name) -> bool { if (is_default_backend_token(name)) { return true; } - if (!sd_resolve_backend_name(name).empty()) { + if (is_disk_backend_token(name)) { + if (error != nullptr) { + *error = "backend 'disk' is only supported by params_backend"; + } + return false; + } + if (!sd_backend_resolve_name(name).empty()) { return true; } if (error != nullptr) { @@ -580,18 +607,24 @@ bool SDBackendManager::validate(std::string* error) const { } return false; }; + auto validate_params_name = [&](const std::string& name) -> bool { + if (is_disk_backend_token(name)) { + return true; + } + return validate_runtime_name(name); + }; - if (!validate_name(runtime_assignment_.default_name) || - !validate_name(params_assignment_.default_name)) { + if (!validate_runtime_name(runtime_assignment_.default_name) || + !validate_params_name(params_assignment_.default_name)) { return false; } for (const auto& kv : runtime_assignment_.module_names) { - if (!validate_name(kv.second)) { + if (!validate_runtime_name(kv.second)) { return false; } } for (const auto& kv : params_assignment_.module_names) { - if (!validate_name(kv.second)) { + if (!validate_params_name(kv.second)) { return false; } } @@ -599,7 +632,7 @@ bool SDBackendManager::validate(std::string* error) const { } ggml_backend_t SDBackendManager::init_cached_backend(const std::string& name) { - std::string resolved = sd_resolve_backend_name(name); + std::string resolved = sd_backend_resolve_name(name); std::string key = lower_copy(resolved); ggml_backend_t backend = nullptr; diff --git a/otherarch/sdcpp/src/core/ggml_extend_backend.h b/otherarch/sdcpp/src/core/ggml_extend_backend.h index fc071ffda..9aecf97c0 100644 --- a/otherarch/sdcpp/src/core/ggml_extend_backend.h +++ b/otherarch/sdcpp/src/core/ggml_extend_backend.h @@ -51,10 +51,6 @@ public: bool init(const char* backend_spec, const char* params_backend_spec, - bool offload_params_to_cpu, - bool keep_clip_on_cpu, - bool keep_vae_on_cpu, - bool keep_control_net_on_cpu, std::string* error); void reset(); @@ -63,6 +59,7 @@ public: bool runtime_backend_is_cpu(SDBackendModule module); bool params_backend_is_cpu(SDBackendModule module); + bool params_backend_is_disk(SDBackendModule module) const; bool runtime_backend_supports_host_buffer(SDBackendModule module); private: @@ -74,6 +71,8 @@ bool sd_backend_is(ggml_backend_t backend, const std::string& name); bool sd_backend_is_cpu(ggml_backend_t backend); ggml_backend_t sd_backend_cpu_init(); bool sd_backend_cpu_set_n_threads(ggml_backend_t backend_cpu, int n_threads); +std::string sd_backend_resolve_name(const std::string& name); const char* sd_backend_module_name(SDBackendModule module); void ggml_ext_im_set_f32_1d(const struct ggml_tensor* tensor, int i, float value); +bool add_rpc_devices(const std::string& servers); #endif // __SD_CORE_GGML_EXTEND_BACKEND_H__ diff --git a/otherarch/sdcpp/src/core/ggml_graph_cut.cpp b/otherarch/sdcpp/src/core/ggml_graph_cut.cpp index c77a45448..d4874b054 100644 --- a/otherarch/sdcpp/src/core/ggml_graph_cut.cpp +++ b/otherarch/sdcpp/src/core/ggml_graph_cut.cpp @@ -1,6 +1,8 @@ #include "core/ggml_graph_cut.h" #include +#include +#include #include #include #include @@ -8,6 +10,7 @@ #include #include +#include "core/ggml_extend_backend.h" #include "core/util.h" #include "ggml-alloc.h" #include "ggml-backend.h" @@ -44,7 +47,9 @@ namespace sd::ggml_graph_cut { if (tensor == nullptr) { return false; } - return params_tensor_set.find(tensor) != params_tensor_set.end(); + return params_tensor_set.find(tensor) != params_tensor_set.end() || + (tensor->view_src != nullptr && + params_tensor_set.find(tensor->view_src) != params_tensor_set.end()); } static int graph_node_index_by_name(ggml_cgraph* gf, const char* name) { @@ -81,6 +86,157 @@ namespace sd::ggml_graph_cut { segment.output_bytes; } + static std::string lower_ascii_copy(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return value; + } + + static std::string normalize_backend_budget_key(const std::string& value) { + return lower_ascii_copy(trim(value)); + } + + static bool is_default_max_vram_key(const std::string& key) { + std::string normalized = normalize_backend_budget_key(key); + return normalized == "all" || normalized == "default" || normalized == "*"; + } + + static bool parse_max_vram_budget_value(const std::string& text, float* value, std::string* error) { + float parsed = 0.f; + if (!parse_strict_float(text, parsed) || !std::isfinite(parsed)) { + if (error != nullptr) { + *error = "invalid --max-vram value '" + text + "'"; + } + return false; + } + *value = parsed; + return true; + } + + static std::vector backend_budget_keys(ggml_backend_t backend) { + std::vector keys; + if (backend == nullptr) { + return keys; + } + + ggml_backend_dev_t dev = ggml_backend_get_device(backend); + if (dev != nullptr) { + keys.push_back(normalize_backend_budget_key(ggml_backend_dev_name(dev))); + } + const char* backend_name = ggml_backend_name(backend); + if (backend_name != nullptr) { + keys.push_back(normalize_backend_budget_key(backend_name)); + } + return keys; + } + + void MaxVramAssignment::reset(float fallback_gib) { + default_gib = fallback_gib; + backend_gib.clear(); + resolved_backend_bytes.clear(); + } + + bool MaxVramAssignment::parse(const std::string& raw_spec, std::string* error) { + const std::string in = trim(raw_spec); + if (in.empty()) { + return true; + } + + for (const std::string& raw_part : split_string(in, ',')) { + const std::string part = trim(raw_part); + if (part.empty()) { + continue; + } + + const size_t eq = part.find('='); + if (eq == std::string::npos) { + float value = 0.f; + if (!parse_max_vram_budget_value(part, &value, error)) { + return false; + } + default_gib = value; + continue; + } + + const std::string key = trim(part.substr(0, eq)); + const std::string value_text = trim(part.substr(eq + 1)); + if (key.empty() || value_text.empty()) { + if (error != nullptr) { + *error = "invalid --max-vram assignment '" + part + "'"; + } + return false; + } + + float value = 0.f; + if (!parse_max_vram_budget_value(value_text, &value, error)) { + return false; + } + + if (is_default_max_vram_key(key)) { + default_gib = value; + continue; + } + + const std::string backend_key = trim(key); + if (backend_key.empty()) { + if (error != nullptr) { + *error = "invalid --max-vram backend key in '" + part + "'"; + } + return false; + } + backend_gib[backend_key] = value; + } + resolved_backend_bytes.clear(); + return true; + } + + bool MaxVramAssignment::canonicalize_backend_keys(std::string* error) { + if (backend_gib.empty()) { + return true; + } + + std::unordered_map normalized; + for (const auto& kv : backend_gib) { + std::string resolved = sd_backend_resolve_name(kv.first); + if (resolved.empty()) { + if (error != nullptr) { + *error = "unknown --max-vram backend '" + kv.first + "'"; + } + return false; + } + normalized[normalize_backend_budget_key(resolved)] = kv.second; + } + backend_gib = std::move(normalized); + resolved_backend_bytes.clear(); + return true; + } + + size_t MaxVramAssignment::bytes_for_backend(ggml_backend_t backend) { + std::vector keys = backend_budget_keys(backend); + const std::string cache_key = keys.empty() ? std::string("") : keys.front(); + auto cached = resolved_backend_bytes.find(cache_key); + if (cached != resolved_backend_bytes.end()) { + return cached->second; + } + + float budget_gib = default_gib; + if (!backend_gib.empty()) { + for (const std::string& key : keys) { + auto backend_it = backend_gib.find(key); + if (backend_it != backend_gib.end()) { + budget_gib = backend_it->second; + break; + } + } + } + + const float resolved_gib = resolve_max_vram_gib(budget_gib, backend); + const size_t bytes = max_vram_gib_to_bytes(resolved_gib); + resolved_backend_bytes[cache_key] = bytes; + return bytes; + } + size_t max_vram_gib_to_bytes(float max_vram) { if (max_vram <= 0.f) { return 0; @@ -135,6 +291,24 @@ namespace sd::ggml_graph_cut { return max_vram_bytes_to_gib(resolve_auto_max_vram_bytes(-max_vram, backend)); } + static bool is_segment_output_needed_after(const Plan& plan, + size_t end_segment_index, + int output_node_index) { + if (end_segment_index + 1 >= plan.segments.size()) { + return false; + } + for (size_t seg_idx = end_segment_index + 1; seg_idx < plan.segments.size(); ++seg_idx) { + const auto& segment = plan.segments[seg_idx]; + for (const auto& input_ref : segment.input_refs) { + if (input_ref.type == Segment::INPUT_PREVIOUS_CUT && + input_ref.node_index == output_node_index) { + return true; + } + } + } + return false; + } + static Segment make_segment_seed(const Plan& plan, size_t start_segment_index, size_t end_segment_index) { @@ -147,8 +321,11 @@ namespace sd::ggml_graph_cut { const auto& target_segment = plan.segments[end_segment_index]; std::unordered_set seen_output_node_indices; for (size_t seg_idx = start_segment_index; seg_idx <= end_segment_index; ++seg_idx) { + const bool is_boundary_segment = seg_idx == end_segment_index; for (int output_node_index : plan.segments[seg_idx].output_node_indices) { - if (seen_output_node_indices.insert(output_node_index).second) { + if ((is_boundary_segment || + is_segment_output_needed_after(plan, end_segment_index, output_node_index)) && + seen_output_node_indices.insert(output_node_index).second) { seed.output_node_indices.push_back(output_node_index); } } @@ -400,23 +577,6 @@ namespace sd::ggml_graph_cut { return tensors; } - std::vector runtime_param_tensors(ggml_cgraph* gf, const Segment& segment, const char* log_desc) { - std::vector tensors = param_tensors(gf, segment); - std::vector filtered_tensors; - filtered_tensors.reserve(tensors.size()); - for (ggml_tensor* tensor : tensors) { - if (tensor_buffer(tensor) == nullptr) { - LOG_WARN("%s graph cut skipping param input without buffer: segment=%s tensor=%s", - log_desc == nullptr ? "unknown" : log_desc, - segment.group_name.c_str(), - tensor->name); - continue; - } - filtered_tensors.push_back(tensor); - } - return filtered_tensors; - } - std::unordered_set collect_future_input_names(ggml_cgraph* gf, const Plan& plan, size_t current_segment_index) { @@ -487,6 +647,44 @@ namespace sd::ggml_graph_cut { return 0; } + struct TensorRuntimeBinding { + ggml_backend_buffer_t buffer = nullptr; + void* data = nullptr; + void* extra = nullptr; + }; + std::unordered_map saved_bindings; + auto mark_measurement_external = [&](ggml_tensor* tensor) { + if (tensor == nullptr) { + return; + } + auto save_tensor = [&](ggml_tensor* t) { + if (t == nullptr || saved_bindings.find(t) != saved_bindings.end()) { + return; + } + saved_bindings[t] = {t->buffer, t->data, t->extra}; + // During real execution params and previous-cut inputs already + // have backend/cache buffers, so gallocr must not reserve them. + t->data = reinterpret_cast(static_cast(1)); + }; + save_tensor(tensor); + save_tensor(tensor->view_src); + }; + for (const auto& input : segment.input_refs) { + if (input.type != Segment::INPUT_PARAM && + input.type != Segment::INPUT_PREVIOUS_CUT) { + continue; + } + mark_measurement_external(input_tensor(gf, input)); + } + + std::unordered_map saved_output_flags; + for (int output_node_index : segment.output_node_indices) { + ggml_tensor* output = ggml_graph_node(gf, output_node_index); + if (output != nullptr && saved_output_flags.find(output) == saved_output_flags.end()) { + saved_output_flags[output] = output->flags; + } + } + ggml_context* graph_ctx = nullptr; ggml_cgraph* segment_graph = build_segment_graph(gf, segment, &graph_ctx); ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); @@ -502,6 +700,14 @@ namespace sd::ggml_graph_cut { ggml_gallocr_free(allocr); ggml_free(graph_ctx); + for (const auto& kv : saved_output_flags) { + kv.first->flags = kv.second; + } + for (const auto& kv : saved_bindings) { + kv.first->buffer = kv.second.buffer; + kv.first->data = kv.second.data; + kv.first->extra = kv.second.extra; + } return buffer_size; } @@ -669,7 +875,8 @@ namespace sd::ggml_graph_cut { GGML_ASSERT(!candidate_plan.segments.empty()); const auto& candidate_segment = candidate_plan.segments.back(); - if (graph_cut_segment_vram_bytes(candidate_segment) > max_graph_vram_bytes) { + const size_t candidate_bytes = graph_cut_segment_vram_bytes(candidate_segment); + if (candidate_bytes > max_graph_vram_bytes) { break; } diff --git a/otherarch/sdcpp/src/core/ggml_graph_cut.h b/otherarch/sdcpp/src/core/ggml_graph_cut.h index 7919acadd..17f2f1d7d 100644 --- a/otherarch/sdcpp/src/core/ggml_graph_cut.h +++ b/otherarch/sdcpp/src/core/ggml_graph_cut.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -68,6 +69,17 @@ namespace sd::ggml_graph_cut { static constexpr const char* GGML_RUNNER_CUT_PREFIX = "ggml_runner_cut:"; + struct MaxVramAssignment { + float default_gib = 0.f; + std::unordered_map backend_gib; + std::unordered_map resolved_backend_bytes; + + void reset(float fallback_gib); + bool parse(const std::string& raw_spec, std::string* error); + bool canonicalize_backend_keys(std::string* error); + size_t bytes_for_backend(ggml_backend_t backend); + }; + bool is_graph_cut_tensor(const ggml_tensor* tensor); std::string make_graph_cut_name(const std::string& group, const std::string& output); void mark_graph_cut(ggml_tensor* tensor, const std::string& group, const std::string& output); @@ -80,7 +92,6 @@ namespace sd::ggml_graph_cut { ggml_tensor* output_tensor(ggml_cgraph* gf, const Segment& segment, size_t output_index); ggml_tensor* input_tensor(ggml_cgraph* gf, const Segment::InputRef& input_ref); std::vector param_tensors(ggml_cgraph* gf, const Segment& segment); - std::vector runtime_param_tensors(ggml_cgraph* gf, const Segment& segment, const char* log_desc); std::unordered_set collect_future_input_names(ggml_cgraph* gf, const Plan& plan, size_t current_segment_index); diff --git a/otherarch/sdcpp/src/core/layer_registry.cpp b/otherarch/sdcpp/src/core/layer_registry.cpp deleted file mode 100644 index 65771ee9e..000000000 --- a/otherarch/sdcpp/src/core/layer_registry.cpp +++ /dev/null @@ -1,132 +0,0 @@ -#include "core/layer_registry.h" - -#include - -#include "core/util.h" - -namespace sd::layer_registry { - - void LayerRegistry::register_layer(const std::string& name, ggml_tensor* tensor) { - auto& info = layers_[name]; - info.tensors.push_back(tensor); - info.bytes += ggml_nbytes(tensor); - } - - bool LayerRegistry::move_layer_to_gpu(const std::string& name) { - auto it = layers_.find(name); - if (it == layers_.end()) - return false; - - LayerInfo& info = it->second; - if (info.on_gpu) - return true; - if (gpu_backend_ == nullptr || cpu_backend_ == nullptr) { - LOG_ERROR("layer_registry: backends not set; cannot move '%s' to GPU", - name.c_str()); - return false; - } - if (info.tensors.empty()) { - info.on_gpu = true; - return true; - } - - // 1. Build a no_alloc context big enough to hold one twin tensor per CPU - // tensor, plus a little overhead. - const size_t ctx_size = info.tensors.size() * ggml_tensor_overhead() + 1024; - ggml_init_params ctx_params{ctx_size, /*mem_buffer=*/nullptr, /*no_alloc=*/true}; - ggml_context* twin_ctx = ggml_init(ctx_params); - if (twin_ctx == nullptr) { - LOG_ERROR("layer_registry: failed to allocate twin context for '%s'", - name.c_str()); - return false; - } - - // 2. Create one GPU twin per CPU tensor. The twin shares the original - // name so any name-based lookup keeps working. - std::vector gpu_twins; - gpu_twins.reserve(info.tensors.size()); - for (ggml_tensor* cpu_t : info.tensors) { - ggml_tensor* twin = ggml_dup_tensor(twin_ctx, cpu_t); - if (cpu_t->name[0] != '\0') { - ggml_set_name(twin, cpu_t->name); - } - gpu_twins.push_back(twin); - } - - // 3. Back the twins with a GPU buffer in one alloc call. - ggml_backend_buffer_t gpu_buffer = ggml_backend_alloc_ctx_tensors(twin_ctx, gpu_backend_); - if (gpu_buffer == nullptr) { - LOG_ERROR("layer_registry: failed to allocate GPU buffer for '%s'", - name.c_str()); - ggml_free(twin_ctx); - return false; - } - - // 4. H2D copy + sync. - for (size_t i = 0; i < info.tensors.size(); ++i) { - ggml_backend_tensor_copy(info.tensors[i], gpu_twins[i]); - } - ggml_backend_synchronize(gpu_backend_); - - // 5. Swap buffer/data/extra so the originals now point at GPU memory. - for (size_t i = 0; i < info.tensors.size(); ++i) { - std::swap(info.tensors[i]->buffer, gpu_twins[i]->buffer); - std::swap(info.tensors[i]->data, gpu_twins[i]->data); - std::swap(info.tensors[i]->extra, gpu_twins[i]->extra); - } - - info.gpu_twins = std::move(gpu_twins); - info.twin_ctx = twin_ctx; - info.gpu_buffer = gpu_buffer; - info.on_gpu = true; - return true; - } - - bool LayerRegistry::move_layer_to_cpu(const std::string& name) { - auto it = layers_.find(name); - if (it == layers_.end()) - return false; - - LayerInfo& info = it->second; - if (!info.on_gpu) - return true; - if (info.tensors.size() != info.gpu_twins.size()) { - LOG_ERROR("layer_registry: twin/tensor count mismatch for '%s'", - name.c_str()); - return false; - } - - // 1. Swap back: originals point at CPU memory again. - for (size_t i = 0; i < info.tensors.size(); ++i) { - if (info.gpu_twins[i] == nullptr) - continue; - std::swap(info.tensors[i]->buffer, info.gpu_twins[i]->buffer); - std::swap(info.tensors[i]->data, info.gpu_twins[i]->data); - std::swap(info.tensors[i]->extra, info.gpu_twins[i]->extra); - } - - // 2. Free the GPU buffer + twin context. - if (info.gpu_buffer != nullptr) { - ggml_backend_buffer_free(info.gpu_buffer); - info.gpu_buffer = nullptr; - } - if (info.twin_ctx != nullptr) { - ggml_free(info.twin_ctx); - info.twin_ctx = nullptr; - } - info.gpu_twins.clear(); - info.on_gpu = false; - return true; - } - - bool LayerRegistry::is_layer_on_gpu(const std::string& name) const { - auto it = layers_.find(name); - return it != layers_.end() && it->second.on_gpu; - } - - size_t LayerRegistry::get_layer_size(const std::string& name) const { - auto it = layers_.find(name); - return it != layers_.end() ? it->second.bytes : 0; - } - -} // namespace sd::layer_registry diff --git a/otherarch/sdcpp/src/core/layer_registry.h b/otherarch/sdcpp/src/core/layer_registry.h deleted file mode 100644 index c0b980d0b..000000000 --- a/otherarch/sdcpp/src/core/layer_registry.h +++ /dev/null @@ -1,50 +0,0 @@ -#ifndef __SD_CORE_LAYER_REGISTRY_H__ -#define __SD_CORE_LAYER_REGISTRY_H__ - -#include -#include -#include -#include - -#include "ggml-backend.h" -#include "ggml.h" - -namespace sd::layer_registry { - - struct LayerInfo { - std::vector tensors; - std::vector gpu_twins; - ggml_context* twin_ctx = nullptr; - ggml_backend_buffer_t gpu_buffer = nullptr; - bool on_gpu = false; - size_t bytes = 0; - }; - - class LayerRegistry { - public: - LayerRegistry() = default; - LayerRegistry(ggml_backend_t gpu_backend, ggml_backend_t cpu_backend) - : gpu_backend_(gpu_backend), cpu_backend_(cpu_backend) {} - - void set_backends(ggml_backend_t gpu_backend, ggml_backend_t cpu_backend) { - gpu_backend_ = gpu_backend; - cpu_backend_ = cpu_backend; - } - void register_layer(const std::string& name, ggml_tensor* tensor); - bool move_layer_to_gpu(const std::string& name); - bool move_layer_to_cpu(const std::string& name); - bool is_layer_on_gpu(const std::string& name) const; - size_t get_layer_size(const std::string& name) const; - size_t get_layer_count() const { return layers_.size(); } - - const std::map& layers() const { return layers_; } - - private: - ggml_backend_t gpu_backend_ = nullptr; - ggml_backend_t cpu_backend_ = nullptr; - std::map layers_; - }; - -} // namespace sd::layer_registry - -#endif // __SD_CORE_LAYER_REGISTRY_H__ diff --git a/otherarch/sdcpp/src/core/util.cpp b/otherarch/sdcpp/src/core/util.cpp index 950254431..a70722af2 100644 --- a/otherarch/sdcpp/src/core/util.cpp +++ b/otherarch/sdcpp/src/core/util.cpp @@ -507,7 +507,7 @@ static int sdloglevel = 0; //-1 = hide all, 0 = normal, 1 = showall static bool sdquiet = false; // } kcpp -static std::string build_progress_bar(int step, int steps) { +static std::string build_progress_bar(int step, int steps, char progress_char = '=', bool show_head = true) { std::string progress = " |"; int max_progress = 50; int32_t current = 0; @@ -517,21 +517,21 @@ static std::string build_progress_bar(int step, int steps) { for (int i = 0; i < 50; i++) { if (i > current) { progress += " "; - } else if (i == current && i != max_progress - 1) { + } else if (show_head && i == current && i != max_progress - 1) { progress += ">"; } else { - progress += "="; + progress += progress_char; } } progress += "|"; return progress; } -static void print_progress_line(int step, int steps, const std::string& speed_text) { +static void print_progress_line(int step, int steps, const std::string& speed_text, char progress_char = '=', bool show_head = true) { if (step == 0) { return; } - std::string progress = build_progress_bar(step, steps); + std::string progress = build_progress_bar(step, steps, progress_char, show_head); const char* lf = (step == steps ? "\n" : ""); printf("\r%s %i/%i - %s\033[K%s", progress.c_str(), step, steps, speed_text.c_str(), lf); fflush(stdout); // for linux @@ -574,9 +574,9 @@ void pretty_bytes_progress(int step, int steps, uint64_t bytes_processed, float double speed_mb = bytes_per_second / (1024.0 * 1024.0); if (speed_mb >= 1024.0) { - print_progress_line(step, steps, sd_format("%.2fGB/s", speed_mb / 1024.0)); + print_progress_line(step, steps, sd_format("%.2fGB/s", speed_mb / 1024.0), '#', false); } else { - print_progress_line(step, steps, sd_format("%.2fMB/s", speed_mb)); + print_progress_line(step, steps, sd_format("%.2fMB/s", speed_mb), '#', false); } } diff --git a/otherarch/sdcpp/src/extensions/generation_extension.h b/otherarch/sdcpp/src/extensions/generation_extension.h index 1e6d1341e..67085c157 100644 --- a/otherarch/sdcpp/src/extensions/generation_extension.h +++ b/otherarch/sdcpp/src/extensions/generation_extension.h @@ -6,10 +6,13 @@ #include #include #include +#include #include "conditioning/conditioner.hpp" #include "core/ggml_extend_backend.h" +#include "model/diffusion/model.hpp" #include "model_loader.h" +#include "model_manager.h" #include "stable-diffusion.h" struct GenerationExtensionInitContext { @@ -17,27 +20,20 @@ struct GenerationExtensionInitContext { SDVersion version; const String2TensorStorage& tensor_storage_map; ModelLoader& model_loader; + std::shared_ptr model_manager; int n_threads; std::function ensure_backend_pair; std::function backend_for; std::function params_backend_for; }; -struct GenerationExtensionTensorContext { - std::map& tensors; - std::map& mmap_able_tensors; - std::function module_can_mmap; -}; - struct GenerationExtensionConditionContext { Conditioner* conditioner; ConditionerParams& condition_params; const sd_pm_params_t& pm_params; - std::map& tensors; - SDVersion version; + const sd_pulid_params_t& pulid_params; int n_threads; int total_steps; - bool free_params_immediately; }; struct GenerationExtension { @@ -50,14 +46,10 @@ struct GenerationExtension { virtual bool init(const GenerationExtensionInitContext&) { return true; } - virtual void collect_param_tensors(GenerationExtensionTensorContext&) {} + virtual void get_param_tensors(std::map&) {} + virtual void collect_loras(std::vector&) {} virtual void add_ignore_tensors(std::set&) const {} - virtual bool alloc_params_buffer() { - return true; - } - virtual size_t get_params_buffer_size() const { - return 0; - } + virtual void runner_done() {} virtual void reset_runtime_condition() {} virtual bool prepare_condition(GenerationExtensionConditionContext&) { return false; @@ -66,8 +58,20 @@ struct GenerationExtension { const SDCondition& condition) const { return condition; } + + // Called in the denoise loop for each enabled extension, after the per-step + // DiffusionParams (including its version-specific `extra`) has been built, + // but before diffusion_model->compute(). Lets an extension feed data into + // the diffusion forward that the conditioning-side hooks can't reach -- it + // can set/override fields on `params` (typically the architecture-specific + // `params.extra`, e.g. a guidance tensor, control payload, or an identity + // embedding for an adapter that injects inside the model's blocks). The + // extension targets whichever `extra` variant matches the active model. + // Mutates `params` only, never the extension. Default no-op. + virtual void before_diffusion(DiffusionParams& /*params*/, int /*step*/) const {} }; std::shared_ptr create_photomaker_extension(); +std::shared_ptr create_pulid_extension(); #endif diff --git a/otherarch/sdcpp/src/extensions/photomaker_extension.cpp b/otherarch/sdcpp/src/extensions/photomaker_extension.cpp index ac3949a11..78c5cdb9b 100644 --- a/otherarch/sdcpp/src/extensions/photomaker_extension.cpp +++ b/otherarch/sdcpp/src/extensions/photomaker_extension.cpp @@ -7,7 +7,6 @@ #include "core/tensor_ggml.hpp" #include "core/util.h" -#include "model/adapter/lora.hpp" #include "model/adapter/pmid.hpp" static std::tuple, std::vector, std::vector> @@ -103,7 +102,6 @@ static std::string remove_photomaker_trigger_from_prompt(FrozenCLIPEmbedderWithC struct PhotoMakerExtension : public GenerationExtension { std::shared_ptr pmid_model; - std::shared_ptr pmid_lora; bool enabled = false; std::string model_path; std::string trigger_word = "img"; @@ -129,54 +127,45 @@ struct PhotoMakerExtension : public GenerationExtension { } PMVersion pm_version = std::strstr(model_path.c_str(), "v2") != nullptr ? PM_VERSION_2 : PM_VERSION_1; - pmid_model = std::make_shared(ctx.backend_for(SDBackendModule::PHOTOMAKER), - ctx.params_backend_for(SDBackendModule::PHOTOMAKER), - ctx.tensor_storage_map, - "pmid", - ctx.version, - pm_version); - if (pm_version == PM_VERSION_2) { - LOG_INFO("using PhotoMaker Version 2"); - } - - pmid_lora = std::make_shared("pmid", - ctx.backend_for(SDBackendModule::PHOTOMAKER), - ctx.params_backend_for(SDBackendModule::PHOTOMAKER), - model_path, - "", - ctx.version); - auto lora_tensor_filter = [&](const std::string& tensor_name) { - return starts_with(tensor_name, "lora.model"); - }; - if (!pmid_lora->load_from_file(ctx.n_threads, lora_tensor_filter)) { - LOG_WARN("load photomaker lora tensors from %s failed", model_path.c_str()); - return false; - } - LOG_INFO("loading stacked ID embedding (PHOTOMAKER) model file from '%s'", model_path.c_str()); if (!ctx.model_loader.init_from_file_and_convert_name(model_path, "pmid.")) { LOG_WARN("loading stacked ID embedding from '%s' failed", model_path.c_str()); return true; } + pmid_model = std::make_shared(ctx.backend_for(SDBackendModule::PHOTOMAKER), + ctx.tensor_storage_map, + "pmid", + ctx.version, + pm_version, + 20.f, + ctx.model_manager); + if (pm_version == PM_VERSION_2) { + LOG_INFO("using PhotoMaker Version 2"); + } + enabled = true; return true; } - void collect_param_tensors(GenerationExtensionTensorContext& ctx) override { + void get_param_tensors(std::map& tensors) override { if (!enabled || pmid_model == nullptr) { return; } - std::map temp; - pmid_model->get_param_tensors(temp, "pmid"); - bool do_mmap = ctx.module_can_mmap(SDBackendModule::PHOTOMAKER); - for (const auto& [key, tensor] : temp) { - ctx.tensors[key] = tensor; - if (do_mmap) { - ctx.mmap_able_tensors[key] = tensor; - } + pmid_model->get_param_tensors(tensors, "pmid"); + } + + void collect_loras(std::vector& loras) override { + if (!enabled || model_path.empty()) { + return; } + ModelManager::LoraSpec lora; + lora.path = model_path; + lora.multiplier = 1.0f; + lora.tensor_name_prefix_filter = "lora.model"; + lora.required = true; + loras.push_back(std::move(lora)); } void add_ignore_tensors(std::set& ignore_tensors) const override { @@ -186,18 +175,10 @@ struct PhotoMakerExtension : public GenerationExtension { ignore_tensors.insert("pmid.unet."); } - bool alloc_params_buffer() override { - if (!enabled || pmid_model == nullptr) { - return true; + void runner_done() override { + if (pmid_model != nullptr) { + pmid_model->runner_done(); } - return pmid_model->alloc_params_buffer(); - } - - size_t get_params_buffer_size() const override { - if (!enabled || pmid_model == nullptr) { - return 0; - } - return pmid_model->get_params_buffer_size(); } void reset_runtime_condition() override { @@ -207,21 +188,10 @@ struct PhotoMakerExtension : public GenerationExtension { bool prepare_condition(GenerationExtensionConditionContext& ctx) override { reset_runtime_condition(); - if (!enabled || pmid_model == nullptr || pmid_lora == nullptr) { + if (!enabled || pmid_model == nullptr) { return false; } - if (!pmid_lora->applied) { - int64_t t0 = ggml_time_ms(); - pmid_lora->apply(ctx.tensors, ctx.version, ctx.n_threads); - int64_t t1 = ggml_time_ms(); - pmid_lora->applied = true; - LOG_INFO("pmid_lora apply completed, taking %.2fs", (t1 - t0) * 1.0f / 1000); - if (ctx.free_params_immediately) { - pmid_lora->free_params_buffer(); - } - } - bool pmv2 = pmid_model->get_version() == PM_VERSION_2; if (ctx.pm_params.id_images_count <= 0 || ctx.pm_params.id_images == nullptr) { LOG_WARN("Provided PhotoMaker model file, but NO input ID images"); @@ -305,9 +275,6 @@ struct PhotoMakerExtension : public GenerationExtension { LOG_INFO("Photomaker ID Stacking, taking %" PRId64 " ms", t1 - t0); LOG_INFO("PHOTOMAKER: start_merge_step: %d", start_merge_step); - if (ctx.free_params_immediately) { - pmid_model->free_params_buffer(); - } return true; } diff --git a/otherarch/sdcpp/src/extensions/pulid_extension.cpp b/otherarch/sdcpp/src/extensions/pulid_extension.cpp new file mode 100644 index 000000000..d529e5710 --- /dev/null +++ b/otherarch/sdcpp/src/extensions/pulid_extension.cpp @@ -0,0 +1,123 @@ +#include "extensions/generation_extension.h" + +#include +#include + +#include "core/tensor_ggml.hpp" +#include "core/util.h" +#include "gguf.h" + +static sd::Tensor load_pulid_id_embedding(const char* path) { + sd::Tensor empty; + if (path == nullptr || strlen(path) == 0) { + return empty; + } + + struct ggml_context* ctx_data = nullptr; + struct gguf_init_params gp = {/*.no_alloc =*/false, /*.ctx =*/&ctx_data}; + struct gguf_context* gguf_ctx = gguf_init_from_file(path, gp); + if (gguf_ctx == nullptr || ctx_data == nullptr) { + LOG_WARN("PuLID id-embedding: cannot read gguf '%s'", path); + if (gguf_ctx != nullptr) + gguf_free(gguf_ctx); + if (ctx_data != nullptr) + ggml_free(ctx_data); + return empty; + } + + struct ggml_tensor* t = ggml_get_tensor(ctx_data, "pulid_id"); + if (t == nullptr) { + LOG_WARN("PuLID id-embedding: no 'pulid_id' tensor in '%s'", path); + gguf_free(gguf_ctx); + ggml_free(ctx_data); + return empty; + } + + const int64_t token_dim = t->ne[0]; + const int64_t num_tokens = t->ne[1]; + if (token_dim <= 0 || num_tokens <= 0 || token_dim > 65536 || num_tokens > 1024 || + t->ne[2] != 1 || t->ne[3] != 1) { + LOG_WARN("PuLID id-embedding: implausible shape [%lld, %lld] in '%s'", + (long long)token_dim, (long long)num_tokens, path); + gguf_free(gguf_ctx); + ggml_free(ctx_data); + return empty; + } + + const size_t n_elem = (size_t)token_dim * (size_t)num_tokens; + sd::Tensor out({token_dim, num_tokens, 1}); + float* dst = out.data(); + if (t->type == GGML_TYPE_F32) { + memcpy(dst, t->data, n_elem * sizeof(float)); + } else if (t->type == GGML_TYPE_F16) { + const ggml_fp16_t* src = reinterpret_cast(t->data); + for (size_t i = 0; i < n_elem; i++) { + dst[i] = ggml_fp16_to_fp32(src[i]); + } + } else if (t->type == GGML_TYPE_BF16) { + const ggml_bf16_t* src = reinterpret_cast(t->data); + for (size_t i = 0; i < n_elem; i++) { + dst[i] = ggml_bf16_to_fp32(src[i]); + } + } else { + LOG_WARN("PuLID id-embedding: unsupported tensor type %s in '%s'", + ggml_type_name(t->type), path); + gguf_free(gguf_ctx); + ggml_free(ctx_data); + return empty; + } + + LOG_INFO("PuLID id-embedding: loaded [%lld, %lld] type=%s from '%s'", + (long long)token_dim, (long long)num_tokens, ggml_type_name(t->type), path); + gguf_free(gguf_ctx); + ggml_free(ctx_data); + return out; +} + +struct PuLIDExtension : public GenerationExtension { + bool enabled = false; + sd::Tensor id_embedding; + float id_weight = 1.0f; + + const char* name() const override { + return "pulid"; + } + + bool is_enabled() const override { + return enabled; + } + + bool init(const GenerationExtensionInitContext& ctx) override { + enabled = strlen(SAFE_STR(ctx.params->pulid_weights_path)) > 0; + return true; + } + + void reset_runtime_condition() override { + id_embedding = {}; + id_weight = 1.0f; + } + + bool prepare_condition(GenerationExtensionConditionContext& ctx) override { + reset_runtime_condition(); + if (!enabled) { + return false; + } + id_embedding = load_pulid_id_embedding(ctx.pulid_params.id_embedding_path); + id_weight = ctx.pulid_params.id_weight; + return false; // PuLID does not modify the conditioning + } + + void before_diffusion(DiffusionParams& params, int /*step*/) const override { + if (!enabled || id_embedding.empty()) { + return; + } + if (auto* flux_extra = std::get_if(¶ms.extra)) { + flux_extra->pulid_id = &id_embedding; + flux_extra->pulid_id_weight = id_weight; + } + } +}; + +std::shared_ptr create_pulid_extension() { + return std::make_shared(); +} diff --git a/otherarch/sdcpp/src/model.h b/otherarch/sdcpp/src/model.h index d037705e7..a62c4d1bf 100644 --- a/otherarch/sdcpp/src/model.h +++ b/otherarch/sdcpp/src/model.h @@ -48,6 +48,7 @@ enum SDVersion { VERSION_LONGCAT, VERSION_PID, VERSION_IDEOGRAM4, + VERSION_ESRGAN, VERSION_COUNT, }; diff --git a/otherarch/sdcpp/src/model/adapter/lora.hpp b/otherarch/sdcpp/src/model/adapter/lora.hpp index 2945b32af..77b26ef99 100644 --- a/otherarch/sdcpp/src/model/adapter/lora.hpp +++ b/otherarch/sdcpp/src/model/adapter/lora.hpp @@ -4,6 +4,7 @@ #include #include "core/ggml_extend.hpp" #include "model_loader.h" +#include "model_manager.h" #define LORA_GRAPH_BASE_SIZE 10240 @@ -14,22 +15,24 @@ struct LoraModel : public GGMLRunner { std::map original_tensor_to_final_tensor; std::set applied_lora_tensors; std::string file_path; - ModelLoader model_loader; - bool load_failed = false; - bool applied = false; - bool tensor_preprocessed = false; + std::shared_ptr model_manager; + ggml_backend_t params_backend = nullptr; + bool load_failed = false; + bool applied = false; + bool tensor_preprocessed = false; typedef std::function filter_t; LoraModel(const std::string& lora_id, ggml_backend_t backend, - ggml_backend_t params_backend, - const std::string& file_path = "", - std::string prefix = "", - SDVersion version = VERSION_COUNT) - : lora_id(lora_id), file_path(file_path), GGMLRunner(backend, params_backend) { + ggml_backend_t params_backend_, + const std::string& file_path = "", + std::string prefix = "", + SDVersion version = VERSION_COUNT, + std::shared_ptr manager = std::make_shared()) + : GGMLRunner(backend, manager), lora_id(lora_id), file_path(file_path), model_manager(std::move(manager)), params_backend(params_backend_) { prefix = "lora." + prefix; - if (!model_loader.init_from_file_and_convert_name(file_path, prefix, version)) { + if (model_manager == nullptr || !model_manager->loader().init_from_file_and_convert_name(file_path, prefix, version)) { load_failed = true; } } @@ -71,7 +74,11 @@ struct LoraModel : public GGMLRunner { return true; }; - model_loader.load_tensors(on_new_tensor_cb, n_threads); + if (model_manager != nullptr) { + model_manager->set_n_threads(n_threads); + } + ModelLoader& model_loader = model_manager->loader(); + model_loader.load_tensors(on_new_tensor_cb); if (tensors_to_create.empty()) { return true; @@ -87,25 +94,64 @@ struct LoraModel : public GGMLRunner { lora_tensors[name] = real; } - if (!alloc_params_buffer()) { - LOG_ERROR("lora model buffer allocation failed"); + std::map tensors; + for (const auto& pair : lora_tensors) { + tensors[pair.first] = pair.second; + } + if (model_manager == nullptr || + !model_manager->register_param_tensors("LoRA", + std::move(tensors), + ModelManager::ResidencyMode::ParamBackend, + runtime_backend, + params_backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("lora model manager registration failed"); + return false; + } + std::vector lora_params; + lora_params.reserve(lora_tensors.size()); + for (const auto& pair : lora_tensors) { + lora_params.push_back(pair.second); + } + if (!model_manager->prepare_params(lora_params)) { + LOG_ERROR("lora model manager prepare params failed"); return false; } - - dry_run = false; - model_loader.load_tensors(on_new_tensor_cb, n_threads); LOG_DEBUG("finished loaded lora"); return true; } - void preprocess_lora_tensors(const std::map& model_tensors) { + void release_loaded_tensors() { + runner_done(); + free_compute_buffer(); + model_manager.reset(); + free_params_ctx(); + alloc_params_ctx(); + model_manager = std::make_shared(); + weight_manager = model_manager; + lora_tensors.clear(); + original_tensor_to_final_tensor.clear(); + applied_lora_tensors.clear(); + applied = false; + tensor_preprocessed = false; + } + + static std::set tensor_names(const std::map& model_tensors) { + std::set names; + for (const auto& item : model_tensors) { + names.insert(item.first); + } + return names; + } + + void preprocess_lora_tensors(const std::set& model_tensor_names) { if (tensor_preprocessed) { return; } tensor_preprocessed = true; // I really hate these hardcoded processes. - if (model_tensors.find("cond_stage_model.1.transformer.text_model.encoder.layers.0.self_attn.in_proj.weight") != model_tensors.end()) { + if (model_tensor_names.find("cond_stage_model.1.transformer.text_model.encoder.layers.0.self_attn.in_proj.weight") != model_tensor_names.end()) { std::unordered_map new_lora_tensors; for (auto& [old_name, tensor] : lora_tensors) { std::string new_name = old_name; @@ -612,7 +658,7 @@ struct LoraModel : public GGMLRunner { if (lokr_w2) applied_lora_tensors.insert(lokr_w2_name); if (lokr_w2_a) - applied_lora_tensors.insert(lokr_w2_name); + applied_lora_tensors.insert(lokr_w2_a_name); if (lokr_w2_b) applied_lora_tensors.insert(lokr_w2_b_name); applied_lora_tensors.insert(alpha_name); @@ -753,11 +799,13 @@ struct LoraModel : public GGMLRunner { return out_diff; } - ggml_cgraph* build_lora_graph(const std::map& model_tensors, SDVersion version) { + ggml_cgraph* build_lora_graph(const std::map& model_tensors, + const std::set& model_tensor_names, + SDVersion version) { size_t lora_graph_size = LORA_GRAPH_BASE_SIZE + lora_tensors.size() * 10; ggml_cgraph* gf = ggml_new_graph_custom(compute_ctx, lora_graph_size, false); - preprocess_lora_tensors(model_tensors); + preprocess_lora_tensors(model_tensor_names); original_tensor_to_final_tensor.clear(); applied_lora_tensors.clear(); @@ -794,12 +842,16 @@ struct LoraModel : public GGMLRunner { return gf; } - void apply(std::map model_tensors, SDVersion version, int n_threads) { + void apply(std::map model_tensors, + const std::set& model_tensor_names, + SDVersion version, + int n_threads, + bool warn_unused = true) { auto get_graph = [&]() -> ggml_cgraph* { - return build_lora_graph(model_tensors, version); + return build_lora_graph(model_tensors, model_tensor_names, version); }; - GGMLRunner::compute(get_graph, n_threads, false, true); - stat(); + GGMLRunner::compute(get_graph, n_threads, false, false, false, true); + stat(!warn_unused); for (auto item : original_tensor_to_final_tensor) { ggml_tensor* original_tensor = item.first; ggml_tensor* final_tensor = item.second; @@ -810,6 +862,10 @@ struct LoraModel : public GGMLRunner { GGMLRunner::free_compute_buffer(); } + void apply(std::map model_tensors, SDVersion version, int n_threads, bool warn_unused = true) { + apply(model_tensors, tensor_names(model_tensors), version, n_threads, warn_unused); + } + void stat(bool at_runntime = false) { size_t total_lora_tensors_count = 0; size_t applied_lora_tensors_count = 0; diff --git a/otherarch/sdcpp/src/model/adapter/pmid.hpp b/otherarch/sdcpp/src/model/adapter/pmid.hpp index 3cf59a470..8f7d4dbde 100644 --- a/otherarch/sdcpp/src/model/adapter/pmid.hpp +++ b/otherarch/sdcpp/src/model/adapter/pmid.hpp @@ -413,13 +413,13 @@ public: public: PhotoMakerIDEncoder(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - SDVersion version = VERSION_SDXL, - PMVersion pm_v = PM_VERSION_1, - float sty = 20.f) - : GGMLRunner(backend, params_backend), + SDVersion version = VERSION_SDXL, + PMVersion pm_v = PM_VERSION_1, + float sty = 20.f, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), version(version), pm_version(pm_v), style_strength(sty) { @@ -558,24 +558,25 @@ public: return build_graph(id_pixel_values, prompt_embeds, class_tokens_mask, id_embeds); }; - return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true)); + return take_or_empty(GGMLRunner::compute(get_graph, n_threads, true, true, true)); } }; struct PhotoMakerIDEmbed : public GGMLRunner { std::map tensors; std::string file_path; - ModelLoader* model_loader; - bool load_failed = false; - bool applied = false; + std::shared_ptr model_manager; + ggml_backend_t params_backend = nullptr; + bool load_failed = false; + bool applied = false; PhotoMakerIDEmbed(ggml_backend_t backend, - ggml_backend_t params_backend, - ModelLoader* ml, - const std::string& file_path = "", - const std::string& prefix = "") - : file_path(file_path), GGMLRunner(backend, params_backend), model_loader(ml) { - if (!model_loader->init_from_file_and_convert_name(file_path, prefix)) { + ggml_backend_t params_backend_, + std::shared_ptr manager = std::make_shared(), + const std::string& file_path = "", + const std::string& prefix = "") + : GGMLRunner(backend, manager), file_path(file_path), model_manager(std::move(manager)), params_backend(params_backend_) { + if (model_manager == nullptr || !model_manager->loader().init_from_file_and_convert_name(file_path, prefix)) { load_failed = true; } } @@ -616,14 +617,27 @@ struct PhotoMakerIDEmbed : public GGMLRunner { return true; }; - model_loader->load_tensors(on_new_tensor_cb, n_threads); - if (!alloc_params_buffer()) { - LOG_ERROR("PhotoMaker ID embeds buffer allocation failed"); + model_manager->set_n_threads(n_threads); + ModelLoader& model_loader = model_manager->loader(); + model_loader.load_tensors(on_new_tensor_cb); + if (!model_manager->register_param_tensors("PhotoMaker ID embeds", + tensors, + ModelManager::ResidencyMode::ParamBackend, + runtime_backend, + params_backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("PhotoMaker ID embeds model manager registration failed"); + return false; + } + std::vector id_embed_params; + id_embed_params.reserve(tensors.size()); + for (const auto& pair : tensors) { + id_embed_params.push_back(pair.second); + } + if (!model_manager->prepare_params(id_embed_params)) { + LOG_ERROR("PhotoMaker ID embeds model manager prepare params failed"); return false; } - - dry_run = false; - model_loader->load_tensors(on_new_tensor_cb, n_threads); LOG_DEBUG("finished loading PhotoMaker ID Embeds "); return true; diff --git a/otherarch/sdcpp/src/model/adapter/pulid.hpp b/otherarch/sdcpp/src/model/adapter/pulid.hpp new file mode 100644 index 000000000..442c5b8b2 --- /dev/null +++ b/otherarch/sdcpp/src/model/adapter/pulid.hpp @@ -0,0 +1,76 @@ +#ifndef __PULID_HPP__ +#define __PULID_HPP__ + +#include "core/ggml_extend.hpp" +#include "model/common/block.hpp" + +class PuLIDPerceiverAttentionCA : public GGMLBlock { +public: + static constexpr int64_t DEFAULT_DIM = 3072; // Flux hidden size + static constexpr int64_t DEFAULT_DIM_HEAD = 128; + static constexpr int64_t DEFAULT_HEADS = 16; + static constexpr int64_t DEFAULT_KV_DIM = 2048; // PuLID ID-embedding dim + +protected: + int64_t dim; + int64_t dim_head; + int64_t heads; + int64_t kv_dim; + int64_t inner_dim; + +public: + PuLIDPerceiverAttentionCA(int64_t dim = DEFAULT_DIM, + int64_t dim_head = DEFAULT_DIM_HEAD, + int64_t heads = DEFAULT_HEADS, + int64_t kv_dim = DEFAULT_KV_DIM) + : dim(dim), + dim_head(dim_head), + heads(heads), + kv_dim(kv_dim), + inner_dim(dim_head * heads) { + blocks["norm1"] = std::shared_ptr(new LayerNorm(kv_dim)); + blocks["norm2"] = std::shared_ptr(new LayerNorm(dim)); + blocks["to_q"] = std::shared_ptr(new Linear(dim, inner_dim, /*bias=*/false)); + blocks["to_kv"] = std::shared_ptr(new Linear(kv_dim, inner_dim * 2, /*bias=*/false)); + blocks["to_out"] = std::shared_ptr(new Linear(inner_dim, dim, /*bias=*/false)); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, + ggml_tensor* id_embedding, + ggml_tensor* image_tokens) { + auto norm1 = std::dynamic_pointer_cast(blocks["norm1"]); + auto norm2 = std::dynamic_pointer_cast(blocks["norm2"]); + auto to_q = std::dynamic_pointer_cast(blocks["to_q"]); + auto to_kv = std::dynamic_pointer_cast(blocks["to_kv"]); + auto to_out = std::dynamic_pointer_cast(blocks["to_out"]); + + ggml_tensor* x_normed = norm1->forward(ctx, id_embedding); + ggml_tensor* lat_normed = norm2->forward(ctx, image_tokens); + + ggml_tensor* q = to_q->forward(ctx, lat_normed); // [N, T_img, 2048] + ggml_tensor* kv = to_kv->forward(ctx, x_normed); // [N, T_img, 3072] + + ggml_tensor* k = ggml_view_3d(ctx->ggml_ctx, kv, + inner_dim, kv->ne[1], kv->ne[2], + kv->nb[1], kv->nb[2], + /*offset=*/0); + ggml_tensor* v = ggml_view_3d(ctx->ggml_ctx, kv, + inner_dim, kv->ne[1], kv->ne[2], + kv->nb[1], kv->nb[2], + /*offset=*/inner_dim * ggml_element_size(kv)); + k = ggml_cont(ctx->ggml_ctx, k); + v = ggml_cont(ctx->ggml_ctx, v); + + ggml_tensor* attn_out = ggml_ext_attention_ext( + ctx->ggml_ctx, ctx->backend, + q, k, v, + heads, + /*mask=*/nullptr, + /*diag_mask_inf=*/false); + + ggml_tensor* out = to_out->forward(ctx, attn_out); + return out; + } +}; + +#endif // __PULID_HPP__ diff --git a/otherarch/sdcpp/src/model/common/block.hpp b/otherarch/sdcpp/src/model/common/block.hpp index 69db0a906..15bfa3767 100644 --- a/otherarch/sdcpp/src/model/common/block.hpp +++ b/otherarch/sdcpp/src/model/common/block.hpp @@ -560,11 +560,11 @@ protected: params["mix_factor"] = ggml_new_tensor_1d(ctx, wtype, 1); } - float get_alpha() { + ggml_tensor* get_alpha(GGMLRunnerContext* ctx) { // image_only_indicator is always tensor([0.]) and since mix_factor.shape is [1,] // so learned_with_images is same as learned - float alpha = ggml_ext_backend_tensor_get_f32(params["mix_factor"]); - return sigmoid(alpha); + auto mix_factor = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, params["mix_factor"]); + return ggml_sigmoid(ctx->ggml_ctx, mix_factor); } public: @@ -578,11 +578,12 @@ public: ggml_tensor* x_spatial, ggml_tensor* x_temporal) { // image_only_indicator is always tensor([0.]) - float alpha = get_alpha(); - auto x = ggml_add(ctx->ggml_ctx, - ggml_ext_scale(ctx->ggml_ctx, x_spatial, alpha), - ggml_ext_scale(ctx->ggml_ctx, x_temporal, 1.0f - alpha)); - return x; + auto alpha = get_alpha(ctx); + return ggml_add(ctx->ggml_ctx, + x_temporal, + ggml_mul(ctx->ggml_ctx, + ggml_sub(ctx->ggml_ctx, x_spatial, x_temporal), + alpha)); } }; diff --git a/otherarch/sdcpp/src/model/diffusion/anima.hpp b/otherarch/sdcpp/src/model/diffusion/anima.hpp index 49c2e45ae..6042516a9 100644 --- a/otherarch/sdcpp/src/model/diffusion/anima.hpp +++ b/otherarch/sdcpp/src/model/diffusion/anima.hpp @@ -561,10 +561,10 @@ namespace Anima { AnimaNet net; AnimaRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "model.diffusion_model") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "model.diffusion_model", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(AnimaConfig::detect_from_weights(tensor_storage_map, prefix + ".net")) { net = AnimaNet(config); net.init(params_ctx, tensor_storage_map, prefix + ".net"); @@ -697,7 +697,7 @@ namespace Anima { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x, timesteps, context, t5_ids, t5_weights); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/control.hpp b/otherarch/sdcpp/src/model/diffusion/control.hpp index 2f5eb5742..57e3616f2 100644 --- a/otherarch/sdcpp/src/model/diffusion/control.hpp +++ b/otherarch/sdcpp/src/model/diffusion/control.hpp @@ -1,8 +1,9 @@ -#ifndef __SD_MODEL_DIFFUSION_CONTROL_HPP__ +#ifndef __SD_MODEL_DIFFUSION_CONTROL_HPP__ #define __SD_MODEL_DIFFUSION_CONTROL_HPP__ #include "model/common/block.hpp" #include "model_loader.h" +#include "model_manager.h" #define CONTROL_NET_GRAPH_SIZE 1536 @@ -309,73 +310,47 @@ public: struct ControlNet : public GGMLRunner { SDVersion version = VERSION_SD1; ControlNetBlock control_net; + std::string weight_prefix; - ggml_backend_buffer_t control_buffer = nullptr; - ggml_context* control_ctx = nullptr; std::vector control_outputs_ggml; ggml_tensor* guided_hint_output_ggml = nullptr; std::vector> controls; - sd::Tensor guided_hint; bool guided_hint_cached = false; + std::shared_ptr owned_model_manager; + ggml_backend_t params_backend = nullptr; + + static const char* guided_hint_cache_name() { + return "controlnet.guided_hint"; + } ControlNet(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - SDVersion version = VERSION_SD1) - : GGMLRunner(backend, params_backend), control_net(version) { - control_net.init(params_ctx, tensor_storage_map, ""); + ggml_backend_t params_backend_, + const String2TensorStorage& tensor_storage_map = {}, + SDVersion version = VERSION_SD1, + const std::string& prefix = "", + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), version(version), control_net(version), weight_prefix(prefix), params_backend(params_backend_) { + control_net.init(params_ctx, tensor_storage_map, prefix); } ~ControlNet() override { free_control_ctx(); } - void alloc_control_ctx(std::vector outs) { - ggml_init_params params; - params.mem_size = static_cast(outs.size() * ggml_tensor_overhead()) + 1024 * 1024; - params.mem_buffer = nullptr; - params.no_alloc = true; - control_ctx = ggml_init(params); - - control_outputs_ggml.resize(outs.size() - 1); - - size_t control_buffer_size = 0; - - guided_hint_output_ggml = ggml_dup_tensor(control_ctx, outs[0]); - control_buffer_size += ggml_nbytes(guided_hint_output_ggml); - - for (int i = 0; i < outs.size() - 1; i++) { - control_outputs_ggml[i] = ggml_dup_tensor(control_ctx, outs[i + 1]); - control_buffer_size += ggml_nbytes(control_outputs_ggml[i]); - } - - control_buffer = ggml_backend_alloc_ctx_tensors(control_ctx, runtime_backend); - - LOG_DEBUG("control buffer size %.2fMB", control_buffer_size * 1.f / 1024.f / 1024.f); - } - void free_control_ctx() { - if (control_buffer != nullptr) { - ggml_backend_buffer_free(control_buffer); - control_buffer = nullptr; - } - if (control_ctx != nullptr) { - ggml_free(control_ctx); - control_ctx = nullptr; - } guided_hint_output_ggml = nullptr; guided_hint_cached = false; - guided_hint = {}; control_outputs_ggml.clear(); controls.clear(); + free_cache_ctx_and_buffer(); } std::string get_desc() override { return "control_net"; } - void get_param_tensors(std::map& tensors, const std::string prefix) { - control_net.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) { + control_net.get_param_tensors(tensors, weight_prefix); } ggml_cgraph* build_graph(const sd::Tensor& x_tensor, @@ -391,11 +366,17 @@ struct ControlNet : public GGMLRunner { ggml_tensor* context = make_optional_input(context_tensor); ggml_tensor* y = make_optional_input(y_tensor); + guided_hint_output_ggml = nullptr; + control_outputs_ggml.clear(); + ggml_tensor* guided_hint_input = nullptr; - if (guided_hint_cached && !guided_hint.empty()) { - guided_hint_input = make_input(guided_hint); - hint = nullptr; - } else { + if (guided_hint_cached) { + guided_hint_input = get_cache_tensor_by_name(guided_hint_cache_name()); + if (guided_hint_input == nullptr) { + guided_hint_cached = false; + } + } + if (guided_hint_input == nullptr) { hint = make_input(hint_tensor); } @@ -409,13 +390,19 @@ struct ControlNet : public GGMLRunner { context, y); - if (control_ctx == nullptr) { - alloc_control_ctx(outs); + if (guided_hint_input == nullptr && !outs.empty()) { + guided_hint_output_ggml = outs[0]; + ggml_set_output(guided_hint_output_ggml); + cache(guided_hint_cache_name(), guided_hint_output_ggml); + ggml_build_forward_expand(gf, guided_hint_output_ggml); } - ggml_build_forward_expand(gf, ggml_cpy(compute_ctx, outs[0], guided_hint_output_ggml)); - for (int i = 0; i < outs.size() - 1; i++) { - ggml_build_forward_expand(gf, ggml_cpy(compute_ctx, outs[i + 1], control_outputs_ggml[i])); + control_outputs_ggml.reserve(outs.size() > 0 ? outs.size() - 1 : 0); + for (size_t i = 1; i < outs.size(); i++) { + ggml_tensor* control_output = outs[i]; + ggml_set_output(control_output); + ggml_build_forward_expand(gf, control_output); + control_outputs_ggml.push_back(control_output); } return gf; @@ -435,15 +422,12 @@ struct ControlNet : public GGMLRunner { return build_graph(x, hint, timesteps, context, y); }; - auto compute_result = GGMLRunner::compute(get_graph, n_threads, false); + auto compute_result = GGMLRunner::compute(get_graph, n_threads, false, false, false, true); if (!compute_result.has_value()) { return std::nullopt; } - if (guided_hint_output_ggml != nullptr) { - guided_hint = restore_trailing_singleton_dims(sd::make_sd_tensor_from_ggml(guided_hint_output_ggml), - 4); - } + guided_hint_cached = get_cache_tensor_by_name(guided_hint_cache_name()) != nullptr; controls.clear(); controls.reserve(control_outputs_ggml.size()); for (ggml_tensor* control : control_outputs_ggml) { @@ -451,36 +435,40 @@ struct ControlNet : public GGMLRunner { GGML_ASSERT(!control_host.empty()); controls.push_back(std::move(control_host)); } - guided_hint_cached = true; return controls; } bool load_from_file(const std::string& file_path, int n_threads) { LOG_INFO("loading control net from '%s'", file_path.c_str()); - if (!alloc_params_buffer()) { - LOG_ERROR("control net model buffer allocation failed"); - return false; - } - std::map tensors; control_net.get_param_tensors(tensors); - std::set ignore_tensors; - ModelLoader model_loader; + auto manager = std::dynamic_pointer_cast(weight_manager.lock()); + if (manager == nullptr) { + owned_model_manager = std::make_shared(); + weight_manager = owned_model_manager; + manager = owned_model_manager; + } + + ModelLoader& model_loader = manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path)) { LOG_ERROR("init control net model loader from file failed: '%s'", file_path.c_str()); return false; } - bool success = model_loader.load_tensors(tensors, ignore_tensors, n_threads); - - if (!success) { - LOG_ERROR("load control net tensors from model loader failed"); + manager->set_n_threads(n_threads); + if (!manager->register_param_tensors("ControlNet", + std::move(tensors), + ModelManager::ResidencyMode::ParamBackend, + runtime_backend, + params_backend) || + !manager->validate_registered_tensors()) { + LOG_ERROR("register control net tensors with model manager failed"); return false; } LOG_INFO("control net model loaded"); - return success; + return true; } }; diff --git a/otherarch/sdcpp/src/model/diffusion/ernie_image.hpp b/otherarch/sdcpp/src/model/diffusion/ernie_image.hpp index 09bcba3b1..12fcada59 100644 --- a/otherarch/sdcpp/src/model/diffusion/ernie_image.hpp +++ b/otherarch/sdcpp/src/model/diffusion/ernie_image.hpp @@ -387,10 +387,10 @@ namespace ErnieImage { std::vector pe_vec; ErnieImageRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(ErnieImageConfig::detect_from_weights(tensor_storage_map, prefix)) { ernie_image = ErnieImageModel(config); ernie_image.init(params_ctx, tensor_storage_map, prefix); @@ -440,7 +440,7 @@ namespace ErnieImage { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x, timesteps, context); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/flux.hpp b/otherarch/sdcpp/src/model/diffusion/flux.hpp index 1d01041b7..b5e6c63bf 100644 --- a/otherarch/sdcpp/src/model/diffusion/flux.hpp +++ b/otherarch/sdcpp/src/model/diffusion/flux.hpp @@ -4,6 +4,7 @@ #include #include +#include "model/adapter/pulid.hpp" #include "model/common/rope.hpp" #include "model/diffusion/dit.hpp" #include "model/diffusion/model.hpp" @@ -49,6 +50,10 @@ namespace Flux { float ref_index_scale = 1.f; ChromaRadianceConfig chroma_radiance_params; + bool pulid_enabled = false; + int pulid_double_interval = 2; + int pulid_single_interval = 4; + static FluxConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, const std::string& prefix, SDVersion version = VERSION_FLUX) { @@ -138,6 +143,9 @@ namespace Flux { if (ends_with(name, "double_blocks.0.txt_attn.norm.key_norm.scale")) { head_dim = tensor_storage.ne[0]; } + if (name.find("pulid_ca.") != std::string::npos) { + config.pulid_enabled = true; + } } if (actual_radiance_patch_size > 0 && actual_radiance_patch_size != config.patch_size) { GGML_ASSERT(config.patch_size == 2 * actual_radiance_patch_size); @@ -957,6 +965,20 @@ namespace Flux { blocks["double_stream_modulation_txt"] = std::make_shared(config.hidden_size, true, !config.disable_bias); blocks["single_stream_modulation"] = std::make_shared(config.hidden_size, false, !config.disable_bias); } + + if (config.pulid_enabled) { + int num_double_ca = (config.depth + config.pulid_double_interval - 1) / config.pulid_double_interval; + int num_single_ca = (config.depth_single_blocks + config.pulid_single_interval - 1) / config.pulid_single_interval; + int num_ca = num_double_ca + num_single_ca; + for (int i = 0; i < num_ca; i++) { + blocks["pulid_ca." + std::to_string(i)] = + std::shared_ptr(new PuLIDPerceiverAttentionCA( + /*dim=*/config.hidden_size, + /*dim_head=*/PuLIDPerceiverAttentionCA::DEFAULT_DIM_HEAD, + /*heads=*/PuLIDPerceiverAttentionCA::DEFAULT_HEADS, + /*kv_dim=*/PuLIDPerceiverAttentionCA::DEFAULT_KV_DIM)); + } + } } ggml_tensor* forward_orig(GGMLRunnerContext* ctx, @@ -967,7 +989,9 @@ namespace Flux { ggml_tensor* guidance, ggml_tensor* pe, ggml_tensor* mod_index_arange = nullptr, - std::vector skip_layers = {}) { + std::vector skip_layers = {}, + ggml_tensor* pulid_id = nullptr, + float pulid_id_weight = 1.0f) { auto img_in = std::dynamic_pointer_cast(blocks["img_in"]); auto txt_in = std::dynamic_pointer_cast(blocks["txt_in"]); auto final_layer = std::dynamic_pointer_cast(blocks["final_layer"]); @@ -1044,6 +1068,13 @@ namespace Flux { sd::ggml_graph_cut::mark_graph_cut(txt, "flux.prelude", "txt"); sd::ggml_graph_cut::mark_graph_cut(vec, "flux.prelude", "vec"); + const bool pulid_active = config.pulid_enabled && pulid_id != nullptr; + if (pulid_active && !skip_layers.empty()) { + LOG_WARN("PuLID + skip_layers is not supported; disabling PuLID for this generation."); + } + const bool pulid_run = pulid_active && skip_layers.empty(); + int ca_idx = 0; + for (int i = 0; i < config.depth; i++) { if (skip_layers.size() > 0 && std::find(skip_layers.begin(), skip_layers.end(), i) != skip_layers.end()) { continue; @@ -1056,9 +1087,19 @@ namespace Flux { txt = img_txt.second; // [N, n_txt_token, hidden_size] sd::ggml_graph_cut::mark_graph_cut(img, "flux.double_blocks." + std::to_string(i), "img"); sd::ggml_graph_cut::mark_graph_cut(txt, "flux.double_blocks." + std::to_string(i), "txt"); + + if (pulid_run && (i % config.pulid_double_interval == 0)) { + auto pulid_ca = std::dynamic_pointer_cast( + blocks["pulid_ca." + std::to_string(ca_idx)]); + ggml_tensor* ca_out = pulid_ca->forward(ctx, pulid_id, img); // [N, n_img_token, hidden_size] + img = ggml_add(ctx->ggml_ctx, img, ggml_scale(ctx->ggml_ctx, ca_out, pulid_id_weight)); + sd::ggml_graph_cut::mark_graph_cut(img, "flux.pulid_ca." + std::to_string(ca_idx), "img"); + ca_idx++; + } } - auto txt_img = ggml_concat(ctx->ggml_ctx, txt, img, 1); // [N, n_txt_token + n_img_token, hidden_size] + auto txt_img = ggml_concat(ctx->ggml_ctx, txt, img, 1); // [N, n_txt_token + n_img_token, hidden_size] + const int64_t n_txt_tok = txt->ne[1]; for (int i = 0; i < config.depth_single_blocks; i++) { if (skip_layers.size() > 0 && std::find(skip_layers.begin(), skip_layers.end(), i + config.depth) != skip_layers.end()) { continue; @@ -1067,6 +1108,29 @@ namespace Flux { txt_img = block->forward(ctx, txt_img, vec, pe, txt_img_mask, ss_mods); sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.single_blocks." + std::to_string(i), "txt_img"); + + if (pulid_run && (i % config.pulid_single_interval == 0)) { + auto pulid_ca = std::dynamic_pointer_cast( + blocks["pulid_ca." + std::to_string(ca_idx)]); + ggml_tensor* txt_part = ggml_view_3d(ctx->ggml_ctx, txt_img, + txt_img->ne[0], n_txt_tok, txt_img->ne[2], + txt_img->nb[1], txt_img->nb[2], + 0); + ggml_tensor* img_part = ggml_view_3d(ctx->ggml_ctx, txt_img, + txt_img->ne[0], + txt_img->ne[1] - n_txt_tok, + txt_img->ne[2], + txt_img->nb[1], + txt_img->nb[2], + n_txt_tok * txt_img->nb[1]); + txt_part = ggml_cont(ctx->ggml_ctx, txt_part); + img_part = ggml_cont(ctx->ggml_ctx, img_part); + ggml_tensor* ca_out = pulid_ca->forward(ctx, pulid_id, img_part); + img_part = ggml_add(ctx->ggml_ctx, img_part, ggml_scale(ctx->ggml_ctx, ca_out, pulid_id_weight)); + txt_img = ggml_concat(ctx->ggml_ctx, txt_part, img_part, 1); + sd::ggml_graph_cut::mark_graph_cut(txt_img, "flux.pulid_ca." + std::to_string(ca_idx), "txt_img"); + ca_idx++; + } } img = ggml_view_3d(ctx->ggml_ctx, @@ -1105,7 +1169,9 @@ namespace Flux { ggml_tensor* mod_index_arange = nullptr, ggml_tensor* dct = nullptr, std::vector ref_latents = {}, - std::vector skip_layers = {}) { + std::vector skip_layers = {}, + ggml_tensor* pulid_id = nullptr, + float pulid_id_weight = 1.0f) { GGML_ASSERT(x->ne[3] == 1); int64_t W = x->ne[0]; @@ -1131,7 +1197,8 @@ namespace Flux { img = ggml_reshape_3d(ctx->ggml_ctx, img, img->ne[0] * img->ne[1], img->ne[2], img->ne[3]); // [N, hidden_size, H/patch_size*W/patch_size] img = ggml_cont(ctx->ggml_ctx, ggml_ext_torch_permute(ctx->ggml_ctx, img, 1, 0, 2, 3)); // [N, H/patch_size*W/patch_size, hidden_size] - auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, mod_index_arange, skip_layers); // [N, n_img_token, hidden_size] + auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, mod_index_arange, skip_layers, + pulid_id, pulid_id_weight); // [N, n_img_token, hidden_size] // nerf decode auto nerf_image_embedder = std::dynamic_pointer_cast(blocks["nerf_image_embedder"]); @@ -1179,7 +1246,9 @@ namespace Flux { ggml_tensor* mod_index_arange = nullptr, ggml_tensor* dct = nullptr, std::vector ref_latents = {}, - std::vector skip_layers = {}) { + std::vector skip_layers = {}, + ggml_tensor* pulid_id = nullptr, + float pulid_id_weight = 1.0f) { GGML_ASSERT(x->ne[3] == 1); int64_t W = x->ne[0]; @@ -1226,7 +1295,8 @@ namespace Flux { } } - auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, mod_index_arange, skip_layers); // [N, num_tokens, C * patch_size * patch_size] + auto out = forward_orig(ctx, img, context, timestep, y, guidance, pe, mod_index_arange, skip_layers, + pulid_id, pulid_id_weight); // [N, num_tokens, C * patch_size * patch_size] if (out->ne[1] > img_tokens) { out = ggml_view_3d(ctx->ggml_ctx, out, out->ne[0], img_tokens, out->ne[2], out->nb[1], out->nb[2], 0); @@ -1248,7 +1318,9 @@ namespace Flux { ggml_tensor* mod_index_arange = nullptr, ggml_tensor* dct = nullptr, std::vector ref_latents = {}, - std::vector skip_layers = {}) { + std::vector skip_layers = {}, + ggml_tensor* pulid_id = nullptr, + float pulid_id_weight = 1.0f) { // Forward pass of DiT. // x: (N, C, H, W) tensor of spatial inputs (images or latent representations of images) // timestep: (N,) tensor of diffusion timesteps @@ -1271,7 +1343,9 @@ namespace Flux { mod_index_arange, dct, ref_latents, - skip_layers); + skip_layers, + pulid_id, + pulid_id_weight); } else { return forward_flux_chroma(ctx, x, @@ -1284,7 +1358,9 @@ namespace Flux { mod_index_arange, dct, ref_latents, - skip_layers); + skip_layers, + pulid_id, + pulid_id_weight); } } }; @@ -1301,12 +1377,12 @@ namespace Flux { bool use_mask = false; FluxRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - SDVersion version = VERSION_FLUX, - bool use_mask = false) - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + SDVersion version = VERSION_FLUX, + bool use_mask = false, + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(FluxConfig::detect_from_weights(tensor_storage_map, prefix, version)), version(version), use_mask(use_mask) { @@ -1384,7 +1460,9 @@ namespace Flux { const sd::Tensor& guidance_tensor = {}, const std::vector>& ref_latents_tensor = {}, bool increase_ref_index = false, - std::vector skip_layers = {}) { + std::vector skip_layers = {}, + const sd::Tensor& pulid_id_tensor = {}, + float pulid_id_weight = 1.0f) { ggml_tensor* x = make_input(x_tensor); ggml_tensor* timesteps = make_input(timesteps_tensor); ggml_tensor* context = make_optional_input(context_tensor); @@ -1461,6 +1539,10 @@ namespace Flux { set_backend_tensor_data(dct, dct_vec.data()); } + ggml_tensor* pulid_id = pulid_id_tensor.empty() + ? nullptr + : make_input(pulid_id_tensor); + auto runner_ctx = get_context(); ggml_tensor* out = flux.forward(&runner_ctx, @@ -1474,7 +1556,9 @@ namespace Flux { mod_index_arange, dct, ref_latents, - skip_layers); + skip_layers, + pulid_id, + pulid_id_weight); ggml_build_forward_expand(gf, out); @@ -1490,17 +1574,20 @@ namespace Flux { const sd::Tensor& guidance = {}, const std::vector>& ref_latents = {}, bool increase_ref_index = false, - std::vector skip_layers = std::vector()) { + std::vector skip_layers = std::vector(), + const sd::Tensor& pulid_id = {}, + float pulid_id_weight = 1.0f) { // x: [N, in_channels, h, w] // timesteps: [N, ] // context: [N, max_position, hidden_size] // y: [N, adm_in_channels] or [1, adm_in_channels] // guidance: [N, ] + // pulid_id: empty (no injection) or [N, num_id_tokens=32, kv_dim=2048] auto get_graph = [&]() -> ggml_cgraph* { - return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, increase_ref_index, skip_layers); + return build_graph(x, timesteps, context, c_concat, y, guidance, ref_latents, increase_ref_index, skip_layers, pulid_id, pulid_id_weight); }; - auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); return result; } @@ -1520,7 +1607,9 @@ namespace Flux { tensor_or_empty(extra->guidance), diffusion_params.ref_latents ? *diffusion_params.ref_latents : empty_ref_latents, diffusion_params.increase_ref_index, - extra->skip_layers ? *extra->skip_layers : empty_skip_layers); + extra->skip_layers ? *extra->skip_layers : empty_skip_layers, + tensor_or_empty(extra->pulid_id), + extra->pulid_id_weight); } void test() { @@ -1583,7 +1672,8 @@ namespace Flux { ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_COUNT; - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path, "model.diffusion_model.")) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; @@ -1599,24 +1689,20 @@ namespace Flux { } std::shared_ptr flux = std::make_shared(backend, - backend, tensor_storage_map, "model.diffusion_model", VERSION_FLUX2, - false); + false, + model_manager); - if (!flux->alloc_params_buffer()) { - LOG_ERROR("flux model allocation failed"); - return; - } - - std::map tensors; - flux->get_param_tensors(tensors, "model.diffusion_model"); - - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("Flux test", + *flux, + "model.diffusion_model", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register flux tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/diffusion/hidream_o1.hpp b/otherarch/sdcpp/src/model/diffusion/hidream_o1.hpp index 3d384def2..559f61bcf 100644 --- a/otherarch/sdcpp/src/model/diffusion/hidream_o1.hpp +++ b/otherarch/sdcpp/src/model/diffusion/hidream_o1.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_MODEL_DIFFUSION_HIDREAM_O1_HPP__ +#ifndef __SD_MODEL_DIFFUSION_HIDREAM_O1_HPP__ #define __SD_MODEL_DIFFUSION_HIDREAM_O1_HPP__ #include @@ -282,10 +282,10 @@ namespace HiDreamO1 { std::array, 4> pos_embed_weight_data_; HiDreamO1VisionRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string& prefix = "model.visual") - : GGMLRunner(backend, params_backend), + const String2TensorStorage& tensor_storage_map = {}, + const std::string& prefix = "model.visual", + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), config(HiDreamO1Config::detect_from_weights(tensor_storage_map, prefix)), model(std::make_shared(false, config.llm.vision)) { model->init(params_ctx, tensor_storage_map, prefix); @@ -323,11 +323,15 @@ namespace HiDreamO1 { return gf; } - sd::Tensor compute(int n_threads, const sd::Tensor& image) { + sd::Tensor compute(int n_threads, + const sd::Tensor& image, + bool auto_free = true, + bool free_compute_buffer = true, + bool free_compute_params = true) { auto get_graph = [&]() { return build_graph(image); }; - auto output = GGMLRunner::compute(get_graph, n_threads, false); + auto output = GGMLRunner::compute(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params); return output.has_value() ? std::move(output.value()) : sd::Tensor(); } }; @@ -339,10 +343,10 @@ namespace HiDreamO1 { std::vector attention_mask_vec; HiDreamO1Runner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string& prefix = "model") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string& prefix = "model", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(HiDreamO1Config::detect_from_weights(tensor_storage_map, prefix)) { model = HiDreamO1Model(config); model.init(params_ctx, tensor_storage_map, prefix); @@ -455,7 +459,7 @@ namespace HiDreamO1 { auto get_graph = [&]() { return build_graph(x, timestep, input_ids, input_pos, token_types, vinput_mask, image_embeds, ref_images); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, @@ -486,29 +490,14 @@ namespace HiDreamO1 { std::shared_ptr vision_runner; HiDreamO1Conditioner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}) - : vision_runner(std::make_shared(backend, params_backend, tensor_storage_map)) {} + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) + : vision_runner(std::make_shared(backend, tensor_storage_map, "model.visual", weight_manager)) {} void get_param_tensors(std::map& tensors) override { vision_runner->get_param_tensors(tensors); } - bool alloc_params_buffer() override { - if (!vision_runner->alloc_params_buffer()) { - return false; - } - return true; - } - - void free_params_buffer() override { - vision_runner->free_params_buffer(); - } - - size_t get_params_buffer_size() override { - return vision_runner->get_params_buffer_size(); - } - void set_max_graph_vram_bytes(size_t max_graph_vram_bytes) override { vision_runner->set_max_graph_vram_bytes(max_graph_vram_bytes); } @@ -521,6 +510,10 @@ namespace HiDreamO1 { vision_runner->set_weight_adapter(adapter); } + void runner_done() override { + vision_runner->runner_done(); + } + SDCondition get_learned_condition(int n_threads, const ConditionerParams& conditioner_params) override { SDCondition result; @@ -666,7 +659,7 @@ namespace HiDreamO1 { result.c_vinput_mask = sd::Tensor(vinput_mask_shape, std::move(vinput_mask)); result.c_image_embeds.reserve(vlm_images.size()); for (const auto& vlm_image : vlm_images) { - auto image_embed = vision_runner->compute(n_threads, vlm_image.second); + auto image_embed = vision_runner->compute(n_threads, vlm_image.second, false, true, true); if (image_embed.empty()) { LOG_ERROR("hidream_o1 conditioner: encode VLM image failed"); return SDCondition(); diff --git a/otherarch/sdcpp/src/model/diffusion/ideogram4.hpp b/otherarch/sdcpp/src/model/diffusion/ideogram4.hpp index 2f53c787c..bfa2f86a4 100644 --- a/otherarch/sdcpp/src/model/diffusion/ideogram4.hpp +++ b/otherarch/sdcpp/src/model/diffusion/ideogram4.hpp @@ -189,11 +189,11 @@ namespace Ideogram4 { } return Rope::embed_interleaved_mrope(ids, - bs, - static_cast(rope_theta), - head_dim, - mrope_section, - axis_wrap_dims); + bs, + static_cast(rope_theta), + head_dim, + mrope_section, + axis_wrap_dims); } class Ideogram4Attention : public GGMLBlock { @@ -449,10 +449,10 @@ namespace Ideogram4 { std::vector image_indicator_vec; Ideogram4Runner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(Ideogram4Config::detect_from_weights(tensor_storage_map, prefix)), uncond_prefix(prefix + ".uncond") { model = Ideogram4Transformer(config); @@ -505,16 +505,16 @@ namespace Ideogram4 { int64_t head_dim = config.emb_dim / config.num_heads; auto runner_ctx = get_context(); - pe_vec = gen_ideogram4_pe(static_cast(grid_h), - static_cast(grid_w), - static_cast(x->ne[3]), - static_cast(context_len), - static_cast(head_dim), - static_cast(config.rope_theta), - config.mrope_section, - runner_ctx.circular_x_enabled, - runner_ctx.circular_y_enabled); - auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len); + pe_vec = gen_ideogram4_pe(static_cast(grid_h), + static_cast(grid_w), + static_cast(x->ne[3]), + static_cast(context_len), + static_cast(head_dim), + static_cast(config.rope_theta), + config.mrope_section, + runner_ctx.circular_x_enabled, + runner_ctx.circular_y_enabled); + auto pe = ggml_new_tensor_4d(compute_ctx, GGML_TYPE_F32, 2, 2, head_dim / 2, pos_len); set_backend_tensor_data(pe, pe_vec.data()); image_indicator_vec.assign(static_cast(pos_len), 1); @@ -537,7 +537,7 @@ namespace Ideogram4 { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x, timesteps, context, use_uncond_model); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/lens.hpp b/otherarch/sdcpp/src/model/diffusion/lens.hpp index 740ff8249..931a8527c 100644 --- a/otherarch/sdcpp/src/model/diffusion/lens.hpp +++ b/otherarch/sdcpp/src/model/diffusion/lens.hpp @@ -356,10 +356,10 @@ namespace Lens { std::vector pe_vec; LensRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(LensConfig::detect_from_weights(tensor_storage_map, prefix)) { lens = LensModel(config); lens.init(params_ctx, tensor_storage_map, prefix); @@ -408,7 +408,7 @@ namespace Lens { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x, timesteps, context); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/ltxv.hpp b/otherarch/sdcpp/src/model/diffusion/ltxv.hpp index a86b4cf50..b89ff32c6 100644 --- a/otherarch/sdcpp/src/model/diffusion/ltxv.hpp +++ b/otherarch/sdcpp/src/model/diffusion/ltxv.hpp @@ -1686,10 +1686,10 @@ namespace LTXV { sd::Tensor ax_input_cache; LTXAVRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string& prefix = "model.diffusion_model") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string& prefix = "model.diffusion_model", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(LTXAVConfig::detect_from_weights(tensor_storage_map, prefix)), model(config) { model.init(params_ctx, tensor_storage_map, prefix); @@ -1939,7 +1939,7 @@ namespace LTXV { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x, timesteps, context, audio_x, audio_timesteps, audio_length, frame_rate, video_positions); }; - auto out = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + auto out = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); return out; } @@ -2025,7 +2025,8 @@ namespace LTXV { ggml_backend_t backend = sd_backend_cpu_init(); LOG_INFO("loading ltxav from '%s'", model_path.c_str()); - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(model_path, "model.diffusion_model.")) { LOG_ERROR("init model loader from file failed: '%s'", model_path.c_str()); return; @@ -2040,19 +2041,18 @@ namespace LTXV { auto& tensor_storage_map = model_loader.get_tensor_storage_map(); std::shared_ptr ltxav = std::make_shared(backend, - backend, tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); - if (!ltxav->alloc_params_buffer()) { - LOG_ERROR("ltxav buffer allocation failed"); - return; - } - std::map tensors; - ltxav->get_param_tensors(tensors, "model.diffusion_model"); - - if (!model_loader.load_tensors(tensors)) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("LTXAV test", + *ltxav, + "model.diffusion_model", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register ltxav tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/diffusion/mmdit.hpp b/otherarch/sdcpp/src/model/diffusion/mmdit.hpp index 844339452..d8e76dfb9 100644 --- a/otherarch/sdcpp/src/model/diffusion/mmdit.hpp +++ b/otherarch/sdcpp/src/model/diffusion/mmdit.hpp @@ -879,10 +879,10 @@ struct MMDiTRunner : public DiffusionModelRunner { MMDiT mmdit; MMDiTRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "") - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(MMDiTConfig::detect_from_weights(tensor_storage_map, prefix)), mmdit(config) { mmdit.init(params_ctx, tensor_storage_map, prefix); @@ -935,7 +935,7 @@ struct MMDiTRunner : public DiffusionModelRunner { return build_graph(x, timesteps, context, y, skip_layers); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, @@ -1001,28 +1001,25 @@ struct MMDiTRunner : public DiffusionModelRunner { // ggml_backend_t backend = ggml_backend_cuda_init(0); ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_F16; - std::shared_ptr mmdit = std::make_shared(backend, backend); + auto model_manager = std::make_shared(); + std::shared_ptr mmdit = std::make_shared(backend, String2TensorStorage{}, "", model_manager); { LOG_INFO("loading from '%s'", file_path.c_str()); - if (!mmdit->alloc_params_buffer()) { - LOG_ERROR("mmdit embeds buffer allocation failed"); - return; - } - - std::map tensors; - mmdit->get_param_tensors(tensors, "model.diffusion_model"); - - ModelLoader model_loader; + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path)) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; } - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("MMDiT test", + *mmdit, + "model.diffusion_model", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register mmdit tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/diffusion/model.hpp b/otherarch/sdcpp/src/model/diffusion/model.hpp index b386711b7..67f0fee02 100644 --- a/otherarch/sdcpp/src/model/diffusion/model.hpp +++ b/otherarch/sdcpp/src/model/diffusion/model.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_MODEL_DIFFUSION_MODEL_HPP__ +#ifndef __SD_MODEL_DIFFUSION_MODEL_HPP__ #define __SD_MODEL_DIFFUSION_MODEL_HPP__ #include @@ -7,6 +7,7 @@ #include "core/ggml_extend.hpp" #include "core/tensor_ggml.hpp" +#include "model_manager.h" struct UNetDiffusionExtra { int num_video_frames = -1; @@ -21,6 +22,8 @@ struct SkipLayerDiffusionExtra { struct FluxDiffusionExtra { const sd::Tensor* guidance = nullptr; const std::vector* skip_layers = nullptr; + const sd::Tensor* pulid_id = nullptr; + float pulid_id_weight = 1.0f; }; struct AnimaDiffusionExtra { @@ -88,9 +91,9 @@ protected: public: DiffusionModelRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const std::string& prefix) - : GGMLRunner(backend, params_backend), + const std::string& prefix, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), prefix(prefix) {} virtual sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/pid.hpp b/otherarch/sdcpp/src/model/diffusion/pid.hpp index ac5851516..68dca00f8 100644 --- a/otherarch/sdcpp/src/model/diffusion/pid.hpp +++ b/otherarch/sdcpp/src/model/diffusion/pid.hpp @@ -710,10 +710,10 @@ namespace Pid { std::vector pixel_pos_comp_vec; PiDRunner(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, - const std::string prefix = "model.diffusion_model") - : DiffusionModelRunner(backend, params_backend, prefix), + const std::string prefix = "model.diffusion_model", + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(PixelDiTConfig::detect_from_weights(tensor_storage_map, prefix)) { model = PixelDiT(config); model.init(params_ctx, tensor_storage_map, prefix); @@ -823,7 +823,7 @@ namespace Pid { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x, timesteps, context, lq_latent, degrade_sigma); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/qwen_image.hpp b/otherarch/sdcpp/src/model/diffusion/qwen_image.hpp index 678c34675..aecd8bcef 100644 --- a/otherarch/sdcpp/src/model/diffusion/qwen_image.hpp +++ b/otherarch/sdcpp/src/model/diffusion/qwen_image.hpp @@ -518,12 +518,12 @@ namespace Qwen { SDVersion version; QwenImageRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - SDVersion version = VERSION_QWEN_IMAGE, - bool zero_cond_t = false) - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + SDVersion version = VERSION_QWEN_IMAGE, + bool zero_cond_t = false, + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(QwenImageConfig::detect_from_weights(tensor_storage_map, prefix)) { config.zero_cond_t = config.zero_cond_t || zero_cond_t; qwen_image = QwenImageModel(config); @@ -627,7 +627,7 @@ namespace Qwen { return build_graph(x, timesteps, context, ref_latents, increase_ref_index); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, @@ -691,7 +691,8 @@ namespace Qwen { ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_Q8_0; - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path, "model.diffusion_model.")) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; @@ -705,23 +706,20 @@ namespace Qwen { } std::shared_ptr qwen_image = std::make_shared(backend, - backend, tensor_storage_map, "model.diffusion_model", - VERSION_QWEN_IMAGE); + VERSION_QWEN_IMAGE, + false, + model_manager); - if (!qwen_image->alloc_params_buffer()) { - LOG_ERROR("qwen_image buffer allocation failed"); - return; - } - - std::map tensors; - qwen_image->get_param_tensors(tensors, "model.diffusion_model"); - - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("Qwen image test", + *qwen_image, + "model.diffusion_model", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register qwen_image tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/diffusion/unet.hpp b/otherarch/sdcpp/src/model/diffusion/unet.hpp index f1a96f9d3..253b3b4b2 100644 --- a/otherarch/sdcpp/src/model/diffusion/unet.hpp +++ b/otherarch/sdcpp/src/model/diffusion/unet.hpp @@ -694,11 +694,11 @@ struct UNetModelRunner : public DiffusionModelRunner { UnetModelBlock unet; UNetModelRunner(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - SDVersion version = VERSION_SD1) - : DiffusionModelRunner(backend, params_backend, prefix), + SDVersion version = VERSION_SD1, + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(UNetConfig::detect_from_weights(tensor_storage_map, prefix, version)), unet(config) { unet.init(params_ctx, tensor_storage_map, prefix); @@ -772,7 +772,7 @@ struct UNetModelRunner : public DiffusionModelRunner { return build_graph(x, timesteps, context, c_concat, y, num_video_frames, controls, control_strength); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, diff --git a/otherarch/sdcpp/src/model/diffusion/wan.hpp b/otherarch/sdcpp/src/model/diffusion/wan.hpp index 92f49dc21..9a907dcfa 100644 --- a/otherarch/sdcpp/src/model/diffusion/wan.hpp +++ b/otherarch/sdcpp/src/model/diffusion/wan.hpp @@ -799,11 +799,11 @@ namespace WAN { SDVersion version; WanRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - SDVersion version = VERSION_WAN2) - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + SDVersion version = VERSION_WAN2, + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(WanConfig::detect_from_weights(tensor_storage_map, prefix)) { if (config.num_layers == 30) { if (version == VERSION_WAN2_2_TI2V) { @@ -950,7 +950,7 @@ namespace WAN { return build_graph(x, timesteps, context, clip_fea, c_concat, time_dim_concat, vace_context, vace_strength); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, @@ -1017,7 +1017,8 @@ namespace WAN { ggml_type model_data_type = GGML_TYPE_F16; LOG_INFO("loading from '%s'", file_path.c_str()); - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path, "model.diffusion_model.")) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; @@ -1031,23 +1032,19 @@ namespace WAN { } std::shared_ptr wan = std::make_shared(backend, - backend, tensor_storage_map, "model.diffusion_model", - VERSION_WAN2_2_TI2V); + VERSION_WAN2_2_TI2V, + model_manager); - if (!wan->alloc_params_buffer()) { - LOG_ERROR("wan buffer allocation failed"); - return; - } - - std::map tensors; - wan->get_param_tensors(tensors, "model.diffusion_model"); - - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("Wan test", + *wan, + "model.diffusion_model", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register wan tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/diffusion/z_image.hpp b/otherarch/sdcpp/src/model/diffusion/z_image.hpp index c35c4495d..d23c28563 100644 --- a/otherarch/sdcpp/src/model/diffusion/z_image.hpp +++ b/otherarch/sdcpp/src/model/diffusion/z_image.hpp @@ -553,11 +553,11 @@ namespace ZImage { SDVersion version; ZImageRunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - SDVersion version = VERSION_Z_IMAGE) - : DiffusionModelRunner(backend, params_backend, prefix), + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + SDVersion version = VERSION_Z_IMAGE, + std::shared_ptr weight_manager = nullptr) + : DiffusionModelRunner(backend, prefix, weight_manager), config(ZImageConfig::detect_from_weights(tensor_storage_map, prefix)) { z_image = ZImageModel(config); z_image.init(params_ctx, tensor_storage_map, prefix); @@ -634,7 +634,7 @@ namespace ZImage { return build_graph(x, timesteps, context, ref_latents, increase_ref_index); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); } sd::Tensor compute(int n_threads, @@ -698,7 +698,8 @@ namespace ZImage { ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_Q8_0; - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path, "model.diffusion_model.")) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; @@ -714,22 +715,19 @@ namespace ZImage { } std::shared_ptr z_image = std::make_shared(backend, - backend, tensor_storage_map, "model.diffusion_model", - VERSION_QWEN_IMAGE); + VERSION_QWEN_IMAGE, + model_manager); - if (!z_image->alloc_params_buffer()) { - LOG_ERROR("z_image buffer allocation failed"); - return; - } - std::map tensors; - z_image->get_param_tensors(tensors, "model.diffusion_model"); - - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("ZImage test", + *z_image, + "model.diffusion_model", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register z_image tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/te/clip.hpp b/otherarch/sdcpp/src/model/te/clip.hpp index 7b7c883e0..6dc8a947b 100644 --- a/otherarch/sdcpp/src/model/te/clip.hpp +++ b/otherarch/sdcpp/src/model/te/clip.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_MODEL_TE_CLIP_HPP__ +#ifndef __SD_MODEL_TE_CLIP_HPP__ #define __SD_MODEL_TE_CLIP_HPP__ #include "core/ggml_extend.hpp" @@ -469,13 +469,13 @@ struct CLIPTextModelRunner : public GGMLRunner { std::vector attention_mask_vec; CLIPTextModelRunner(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - CLIPVersion version = OPENAI_CLIP_VIT_L_14, - bool with_final_ln = true, - bool force_clip_f32 = false) - : GGMLRunner(backend, params_backend) { + CLIPVersion version = OPENAI_CLIP_VIT_L_14, + bool with_final_ln = true, + bool force_clip_f32 = false, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager) { bool proj_in = false; for (const auto& [name, tensor_storage] : tensor_storage_map) { if (!starts_with(name, prefix)) { @@ -567,11 +567,14 @@ struct CLIPTextModelRunner : public GGMLRunner { void* custom_embeddings_data, size_t max_token_idx, bool return_pooled, - int clip_skip) { + int clip_skip, + bool auto_free = true, + bool free_compute_buffer = true, + bool free_compute_params = true) { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(input_ids, num_custom_embeddings, custom_embeddings_data, max_token_idx, return_pooled, clip_skip); }; - auto result = GGMLRunner::compute(get_graph, n_threads, true); + auto result = GGMLRunner::compute(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params); if (return_pooled) { return take_or_empty(std::move(result)); } diff --git a/otherarch/sdcpp/src/model/te/llm.hpp b/otherarch/sdcpp/src/model/te/llm.hpp index 3a22e881a..74dc232e5 100644 --- a/otherarch/sdcpp/src/model/te/llm.hpp +++ b/otherarch/sdcpp/src/model/te/llm.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_MODEL_TE_LLM_HPP__ +#ifndef __SD_MODEL_TE_LLM_HPP__ #define __SD_MODEL_TE_LLM_HPP__ #include @@ -22,6 +22,7 @@ #include "json.hpp" #include "model/common/rope.hpp" #include "model_loader.h" +#include "model_manager.h" #include "tokenizers/bpe_tokenizer.h" #include "tokenizers/gemma_tokenizer.h" #include "tokenizers/gpt_oss_tokenizer.h" @@ -1571,11 +1572,11 @@ namespace LLM { public: LLMRunner(LLMArch arch, ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - bool enable_vision_ = false) - : GGMLRunner(backend, params_backend), + bool enable_vision_ = false, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), config(LLMConfig::detect_from_weights(tensor_storage_map, prefix, arch)), enable_vision(enable_vision_) { if (enable_vision && !config.have_vision_weight) { @@ -1733,7 +1734,10 @@ namespace LLM { const sd::Tensor& attention_mask, const std::vector>>& image_embeds, std::set out_layers, - bool return_all_hidden_states = false) { + bool return_all_hidden_states = false, + bool auto_free = true, + bool free_compute_buffer = true, + bool free_compute_params = true) { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(input_ids, attention_mask, @@ -1741,7 +1745,7 @@ namespace LLM { out_layers, return_all_hidden_states); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, true), + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params), input_ids.dim() + 1); } @@ -1802,11 +1806,14 @@ namespace LLM { } sd::Tensor encode_image(const int n_threads, - const sd::Tensor& image) { + const sd::Tensor& image, + bool auto_free = false, + bool free_compute_buffer = false, + bool free_compute_params = false) { auto get_graph = [&]() -> ggml_cgraph* { return build_encode_image_graph(image); }; - return take_or_empty(GGMLRunner::compute(get_graph, n_threads, false)); + return take_or_empty(GGMLRunner::compute(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params)); } }; @@ -1816,11 +1823,11 @@ namespace LLM { LLMEmbedder(LLMArch arch, ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - bool enable_vision = false) - : model(arch, backend, params_backend, tensor_storage_map, prefix, enable_vision) { + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + bool enable_vision = false, + std::shared_ptr weight_manager = nullptr) + : model(arch, backend, tensor_storage_map, prefix, enable_vision, weight_manager) { if (arch == LLMArch::MISTRAL_SMALL_3_2 || arch == LLMArch::MINISTRAL_3_3B) { tokenizer = std::make_shared(); } else if (arch == LLMArch::GPT_OSS_20B) { @@ -1834,13 +1841,6 @@ namespace LLM { model.get_param_tensors(tensors, prefix); } - bool alloc_params_buffer() { - if (!model.alloc_params_buffer()) { - return false; - } - return true; - } - std::tuple, std::vector> tokenize(std::string text, std::pair attn_range, size_t max_length = 0, @@ -2056,7 +2056,8 @@ namespace LLM { ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_COUNT; - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path, "text_encoders.llm.")) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; @@ -2074,24 +2075,20 @@ namespace LLM { LLMArch arch = LLMArch::QWEN3; std::shared_ptr llm = std::make_shared(arch, - backend, backend, tensor_storage_map, "text_encoders.llm", - true); + true, + model_manager); - if (!llm->alloc_params_buffer()) { - LOG_ERROR("llm model allocation failed"); - return; - } - - std::map tensors; - llm->get_param_tensors(tensors, "text_encoders.llm"); - - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("LLM test", + *llm, + "text_encoders.llm", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register llm tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/te/t5.hpp b/otherarch/sdcpp/src/model/te/t5.hpp index 41d9978e5..23da08222 100644 --- a/otherarch/sdcpp/src/model/te/t5.hpp +++ b/otherarch/sdcpp/src/model/te/t5.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_MODEL_TE_T5_HPP__ +#ifndef __SD_MODEL_TE_T5_HPP__ #define __SD_MODEL_TE_T5_HPP__ #include @@ -12,6 +12,7 @@ #include "core/ggml_extend.hpp" #include "model_loader.h" +#include "model_manager.h" #include "tokenizers/t5_unigram_tokenizer.h" struct T5Config { @@ -334,11 +335,11 @@ struct T5Runner : public GGMLRunner { std::vector relative_position_bucket_vec; T5Runner(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - bool is_umt5 = false) - : GGMLRunner(backend, params_backend), + bool is_umt5 = false, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), config(T5Config::detect_from_weights(tensor_storage_map, prefix, is_umt5)) { model = T5(config); model.init(params_ctx, tensor_storage_map, prefix); @@ -394,11 +395,14 @@ struct T5Runner : public GGMLRunner { sd::Tensor compute(const int n_threads, const sd::Tensor& input_ids, - const sd::Tensor& attention_mask) { + const sd::Tensor& attention_mask, + bool auto_free = true, + bool free_compute_buffer = true, + bool free_compute_params = true) { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(input_ids, attention_mask); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, true), 3); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, auto_free, free_compute_buffer, free_compute_params), 3); } static std::vector _relative_position_bucket(const std::vector& relative_position, @@ -474,24 +478,17 @@ struct T5Embedder { T5Runner model; T5Embedder(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - bool is_umt5 = false) - : model(backend, params_backend, tensor_storage_map, prefix, is_umt5), tokenizer(is_umt5) { + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + bool is_umt5 = false, + std::shared_ptr weight_manager = nullptr) + : model(backend, tensor_storage_map, prefix, is_umt5, weight_manager), tokenizer(is_umt5) { } void get_param_tensors(std::map& tensors, const std::string prefix) { model.get_param_tensors(tensors, prefix); } - bool alloc_params_buffer() { - if (!model.alloc_params_buffer()) { - return false; - } - return true; - } - std::tuple, std::vector, std::vector> tokenize(std::string text, size_t max_length = 0, bool padding = false) { @@ -576,7 +573,8 @@ struct T5Embedder { ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_F16; - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path)) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; @@ -589,19 +587,16 @@ struct T5Embedder { } } - std::shared_ptr t5 = std::make_shared(backend, backend, tensor_storage_map, "", true); + std::shared_ptr t5 = std::make_shared(backend, tensor_storage_map, "", true, model_manager); - if (!t5->alloc_params_buffer()) { - LOG_ERROR("t5 params buffer allocation failed"); - return; - } - std::map tensors; - t5->get_param_tensors(tensors, ""); - - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("T5 test", + *t5, + "", + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register t5 tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/upscaler/esrgan.hpp b/otherarch/sdcpp/src/model/upscaler/esrgan.hpp index a56ebfe57..4afbab07d 100644 --- a/otherarch/sdcpp/src/model/upscaler/esrgan.hpp +++ b/otherarch/sdcpp/src/model/upscaler/esrgan.hpp @@ -1,8 +1,14 @@ -#ifndef __SD_MODEL_UPSCALER_ESRGAN_HPP__ +#ifndef __SD_MODEL_UPSCALER_ESRGAN_HPP__ #define __SD_MODEL_UPSCALER_ESRGAN_HPP__ +#include +#include +#include +#include +#include + #include "core/ggml_extend.hpp" -#include "model_loader.h" +#include "core/util.h" /* =================================== ESRGAN =================================== @@ -12,6 +18,74 @@ */ +struct ESRGANConfig { + int scale = 4; + int num_block = 23; + int num_in_ch = 3; + int num_out_ch = 3; + int num_feat = 64; + int num_grow_ch = 32; + + static ESRGANConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, + const std::string& prefix = "") { + ESRGANConfig config; + auto find_weight = [&](const std::string& suffix) -> const TensorStorage* { + std::string name = prefix.empty() ? suffix : prefix + "." + suffix; + auto iter = tensor_storage_map.find(name); + if (iter == tensor_storage_map.end()) { + return nullptr; + } + return &iter->second; + }; + + int detected_num_block = 0; + const std::string body_prefix = prefix.empty() ? "body." : prefix + ".body."; + for (const auto& [name, _] : tensor_storage_map) { + if (!starts_with(name, body_prefix)) { + continue; + } + size_t pos = name.find('.', body_prefix.size()); + if (pos == std::string::npos) { + continue; + } + try { + int idx = std::stoi(name.substr(body_prefix.size(), pos - body_prefix.size())); + detected_num_block = std::max(detected_num_block, idx + 1); + } catch (...) { + } + } + if (detected_num_block > 0) { + config.num_block = detected_num_block; + } + + bool has_conv_up2 = find_weight("conv_up2.weight") != nullptr; + bool has_conv_up1 = find_weight("conv_up1.weight") != nullptr; + bool has_model_tensor = + detected_num_block > 0 || + find_weight("conv_first.weight") != nullptr || + find_weight("conv_hr.weight") != nullptr || + find_weight("conv_last.weight") != nullptr; + if (has_conv_up2) { + config.scale = 4; + } else if (has_conv_up1) { + config.scale = 2; + } else if (has_model_tensor) { + config.scale = 1; + } + + if (has_model_tensor || has_conv_up1 || has_conv_up2) { + LOG_DEBUG("esrgan: scale = %d, num_block = %d, num_in_ch = %d, num_out_ch = %d, num_feat = %d, num_grow_ch = %d", + config.scale, + config.num_block, + config.num_in_ch, + config.num_out_ch, + config.num_feat, + config.num_grow_ch); + } + return config; + } +}; + class ResidualDenseBlock : public GGMLBlock { protected: int num_feat; @@ -83,34 +157,29 @@ public: class RRDBNet : public GGMLBlock { protected: - int scale = 4; - int num_block = 23; - int num_in_ch = 3; - int num_out_ch = 3; - int num_feat = 64; - int num_grow_ch = 32; + ESRGANConfig config; public: - RRDBNet(int scale, int num_block, int num_in_ch, int num_out_ch, int num_feat, int num_grow_ch) - : scale(scale), num_block(num_block), num_in_ch(num_in_ch), num_out_ch(num_out_ch), num_feat(num_feat), num_grow_ch(num_grow_ch) { - blocks["conv_first"] = std::shared_ptr(new Conv2d(num_in_ch, num_feat, {3, 3}, {1, 1}, {1, 1})); - for (int i = 0; i < num_block; i++) { + explicit RRDBNet(ESRGANConfig config) + : config(std::move(config)) { + blocks["conv_first"] = std::shared_ptr(new Conv2d(this->config.num_in_ch, this->config.num_feat, {3, 3}, {1, 1}, {1, 1})); + for (int i = 0; i < this->config.num_block; i++) { std::string name = "body." + std::to_string(i); - blocks[name] = std::shared_ptr(new RRDB(num_feat, num_grow_ch)); + blocks[name] = std::shared_ptr(new RRDB(this->config.num_feat, this->config.num_grow_ch)); } - blocks["conv_body"] = std::shared_ptr(new Conv2d(num_feat, num_feat, {3, 3}, {1, 1}, {1, 1})); - if (scale >= 2) { - blocks["conv_up1"] = std::shared_ptr(new Conv2d(num_feat, num_feat, {3, 3}, {1, 1}, {1, 1})); + blocks["conv_body"] = std::shared_ptr(new Conv2d(this->config.num_feat, this->config.num_feat, {3, 3}, {1, 1}, {1, 1})); + if (this->config.scale >= 2) { + blocks["conv_up1"] = std::shared_ptr(new Conv2d(this->config.num_feat, this->config.num_feat, {3, 3}, {1, 1}, {1, 1})); } - if (scale == 4) { - blocks["conv_up2"] = std::shared_ptr(new Conv2d(num_feat, num_feat, {3, 3}, {1, 1}, {1, 1})); + if (this->config.scale == 4) { + blocks["conv_up2"] = std::shared_ptr(new Conv2d(this->config.num_feat, this->config.num_feat, {3, 3}, {1, 1}, {1, 1})); } - blocks["conv_hr"] = std::shared_ptr(new Conv2d(num_feat, num_feat, {3, 3}, {1, 1}, {1, 1})); - blocks["conv_last"] = std::shared_ptr(new Conv2d(num_feat, num_out_ch, {3, 3}, {1, 1}, {1, 1})); + blocks["conv_hr"] = std::shared_ptr(new Conv2d(this->config.num_feat, this->config.num_feat, {3, 3}, {1, 1}, {1, 1})); + blocks["conv_last"] = std::shared_ptr(new Conv2d(this->config.num_feat, this->config.num_out_ch, {3, 3}, {1, 1}, {1, 1})); } - int get_scale() { return scale; } - int get_num_block() { return num_block; } + int get_scale() { return config.scale; } + int get_num_block() { return config.num_block; } ggml_tensor* lrelu(GGMLRunnerContext* ctx, ggml_tensor* x) { return ggml_leaky_relu(ctx->ggml_ctx, x, 0.2f, true); @@ -127,7 +196,7 @@ public: auto feat = conv_first->forward(ctx, x); sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.prelude", "feat"); auto body_feat = feat; - for (int i = 0; i < num_block; i++) { + for (int i = 0; i < config.num_block; i++) { std::string name = "body." + std::to_string(i); auto block = std::dynamic_pointer_cast(blocks[name]); @@ -138,11 +207,11 @@ public: feat = ggml_add(ctx->ggml_ctx, feat, body_feat); sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.body.out", "feat"); // upsample - if (scale >= 2) { + if (config.scale >= 2) { auto conv_up1 = std::dynamic_pointer_cast(blocks["conv_up1"]); feat = lrelu(ctx, conv_up1->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST))); sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.up1", "feat"); - if (scale == 4) { + if (config.scale == 4) { auto conv_up2 = std::dynamic_pointer_cast(blocks["conv_up2"]); feat = lrelu(ctx, conv_up2->forward(ctx, ggml_upscale(ctx->ggml_ctx, feat, 2, GGML_SCALE_MODE_NEAREST))); sd::ggml_graph_cut::mark_graph_cut(feat, "esrgan.up2", "feat"); @@ -156,199 +225,28 @@ public: }; struct ESRGAN : public GGMLRunner { + ESRGANConfig config; std::unique_ptr rrdb_net; - int scale = 4; - int tile_size = 128; // avoid cuda OOM for 4gb VRAM ESRGAN(ggml_backend_t backend, - ggml_backend_t params_backend, - int tile_size = 128, - const String2TensorStorage& tensor_storage_map = {}) - : GGMLRunner(backend, params_backend) { - this->tile_size = tile_size; + const String2TensorStorage& tensor_storage_map = {}, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), + config(ESRGANConfig::detect_from_weights(tensor_storage_map)), + rrdb_net(std::make_unique(config)) { + rrdb_net->init(params_ctx, tensor_storage_map, ""); } std::string get_desc() override { return "esrgan"; } - bool load_from_file(const std::string& file_path, int n_threads) { - LOG_INFO("loading esrgan from '%s'", file_path.c_str()); - - ModelLoader model_loader; - if (!model_loader.init_from_file_and_convert_name(file_path)) { - LOG_ERROR("init esrgan model loader from file failed: '%s'", file_path.c_str()); - return false; + void get_param_tensors(std::map& tensors) { + if (!rrdb_net) { + return; } - // Get tensor names - auto tensor_names = model_loader.get_tensor_names(); - - // Detect if it's ESRGAN format - bool is_ESRGAN = std::find(tensor_names.begin(), tensor_names.end(), "model.0.weight") != tensor_names.end(); - - // Detect parameters from tensor names - int detected_num_block = 0; - if (is_ESRGAN) { - for (const auto& name : tensor_names) { - if (name.find("model.1.sub.") == 0) { - size_t first_dot = name.find('.', 12); - if (first_dot != std::string::npos) { - size_t second_dot = name.find('.', first_dot + 1); - if (second_dot != std::string::npos && name.substr(first_dot + 1, 3) == "RDB") { - try { - int idx = std::stoi(name.substr(12, first_dot - 12)); - detected_num_block = std::max(detected_num_block, idx + 1); - } catch (...) { - } - } - } - } - } - } else { - // Original format - for (const auto& name : tensor_names) { - if (name.find("body.") == 0) { - size_t pos = name.find('.', 5); - if (pos != std::string::npos) { - try { - int idx = std::stoi(name.substr(5, pos - 5)); - detected_num_block = std::max(detected_num_block, idx + 1); - } catch (...) { - } - } - } - } - } - - int detected_scale = 4; // default - if (is_ESRGAN) { - // For ESRGAN format, detect scale by highest model number - int max_model_num = 0; - for (const auto& name : tensor_names) { - if (name.find("model.") == 0) { - size_t dot_pos = name.find('.', 6); - if (dot_pos != std::string::npos) { - try { - int num = std::stoi(name.substr(6, dot_pos - 6)); - max_model_num = std::max(max_model_num, num); - } catch (...) { - } - } - } - } - if (max_model_num <= 4) { - detected_scale = 1; - } else if (max_model_num <= 7) { - detected_scale = 2; - } else { - detected_scale = 4; - } - } else { - // Original format - bool has_conv_up2 = std::any_of(tensor_names.begin(), tensor_names.end(), [](const std::string& name) { - return name == "conv_up2.weight"; - }); - bool has_conv_up1 = std::any_of(tensor_names.begin(), tensor_names.end(), [](const std::string& name) { - return name == "conv_up1.weight"; - }); - if (has_conv_up2) { - detected_scale = 4; - } else if (has_conv_up1) { - detected_scale = 2; - } else { - detected_scale = 1; - } - } - - int detected_num_in_ch = 3; - int detected_num_out_ch = 3; - int detected_num_feat = 64; - int detected_num_grow_ch = 32; - - // Create RRDBNet with detected parameters - rrdb_net = std::make_unique(detected_scale, detected_num_block, detected_num_in_ch, detected_num_out_ch, detected_num_feat, detected_num_grow_ch); - rrdb_net->init(params_ctx, {}, ""); - - if (!alloc_params_buffer()) { - LOG_ERROR("esrgan model buffer allocation failed"); - return false; - } - - std::map esrgan_tensors; - rrdb_net->get_param_tensors(esrgan_tensors); - - bool success; - if (is_ESRGAN) { - // Build name mapping for ESRGAN format - std::map expected_to_model; - expected_to_model["conv_first.weight"] = "model.0.weight"; - expected_to_model["conv_first.bias"] = "model.0.bias"; - - for (int i = 0; i < detected_num_block; i++) { - for (int j = 1; j <= 3; j++) { - for (int k = 1; k <= 5; k++) { - std::string expected_weight = "body." + std::to_string(i) + ".rdb" + std::to_string(j) + ".conv" + std::to_string(k) + ".weight"; - std::string model_weight = "model.1.sub." + std::to_string(i) + ".RDB" + std::to_string(j) + ".conv" + std::to_string(k) + ".0.weight"; - expected_to_model[expected_weight] = model_weight; - - std::string expected_bias = "body." + std::to_string(i) + ".rdb" + std::to_string(j) + ".conv" + std::to_string(k) + ".bias"; - std::string model_bias = "model.1.sub." + std::to_string(i) + ".RDB" + std::to_string(j) + ".conv" + std::to_string(k) + ".0.bias"; - expected_to_model[expected_bias] = model_bias; - } - } - } - - if (detected_scale == 1) { - expected_to_model["conv_body.weight"] = "model.1.sub." + std::to_string(detected_num_block) + ".weight"; - expected_to_model["conv_body.bias"] = "model.1.sub." + std::to_string(detected_num_block) + ".bias"; - expected_to_model["conv_hr.weight"] = "model.2.weight"; - expected_to_model["conv_hr.bias"] = "model.2.bias"; - expected_to_model["conv_last.weight"] = "model.4.weight"; - expected_to_model["conv_last.bias"] = "model.4.bias"; - } else { - expected_to_model["conv_body.weight"] = "model.1.sub." + std::to_string(detected_num_block) + ".weight"; - expected_to_model["conv_body.bias"] = "model.1.sub." + std::to_string(detected_num_block) + ".bias"; - if (detected_scale >= 2) { - expected_to_model["conv_up1.weight"] = "model.3.weight"; - expected_to_model["conv_up1.bias"] = "model.3.bias"; - } - if (detected_scale == 4) { - expected_to_model["conv_up2.weight"] = "model.6.weight"; - expected_to_model["conv_up2.bias"] = "model.6.bias"; - expected_to_model["conv_hr.weight"] = "model.8.weight"; - expected_to_model["conv_hr.bias"] = "model.8.bias"; - expected_to_model["conv_last.weight"] = "model.10.weight"; - expected_to_model["conv_last.bias"] = "model.10.bias"; - } else if (detected_scale == 2) { - expected_to_model["conv_hr.weight"] = "model.5.weight"; - expected_to_model["conv_hr.bias"] = "model.5.bias"; - expected_to_model["conv_last.weight"] = "model.7.weight"; - expected_to_model["conv_last.bias"] = "model.7.bias"; - } - } - - std::map model_tensors; - for (auto& p : esrgan_tensors) { - auto it = expected_to_model.find(p.first); - if (it != expected_to_model.end()) { - model_tensors[it->second] = p.second; - } - } - - success = model_loader.load_tensors(model_tensors, {}, n_threads); - } else { - success = model_loader.load_tensors(esrgan_tensors, {}, n_threads); - } - - if (!success) { - LOG_ERROR("load esrgan tensors from model loader failed"); - return false; - } - - scale = rrdb_net->get_scale(); - LOG_INFO("esrgan model loaded with scale=%d, num_block=%d", scale, detected_num_block); - return success; + rrdb_net->get_param_tensors(tensors); } ggml_cgraph* build_graph(const sd::Tensor& x_tensor) { @@ -367,7 +265,7 @@ struct ESRGAN : public GGMLRunner { sd::Tensor compute(const int n_threads, const sd::Tensor& x) { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); }; - auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), x.dim()); + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), x.dim()); return result; } }; diff --git a/otherarch/sdcpp/src/model/upscaler/ltx_latent_upscaler.hpp b/otherarch/sdcpp/src/model/upscaler/ltx_latent_upscaler.hpp index b411e8aaf..5343ad036 100644 --- a/otherarch/sdcpp/src/model/upscaler/ltx_latent_upscaler.hpp +++ b/otherarch/sdcpp/src/model/upscaler/ltx_latent_upscaler.hpp @@ -1,9 +1,9 @@ -#ifndef __SD_MODEL_UPSCALER_LTX_LATENT_UPSCALER_HPP__ +#ifndef __SD_MODEL_UPSCALER_LTX_LATENT_UPSCALER_HPP__ #define __SD_MODEL_UPSCALER_LTX_LATENT_UPSCALER_HPP__ +#include #include #include -#include #include #include #include @@ -32,90 +32,100 @@ namespace LTXVUpsampler { int spatial_up_num = 2; int spatial_down_den = 1; int temporal_up_factor = 1; - }; - static inline bool has_tensor(const String2TensorStorage& tensor_storage_map, - const std::string& name) { - return tensor_storage_map.find(name) != tensor_storage_map.end(); - } + static LatentUpsamplerConfig detect_from_weights(const String2TensorStorage& tensor_storage_map, + const std::string& prefix = "") { + LatentUpsamplerConfig config; + auto find_weight = [&](const std::string& suffix) -> const TensorStorage* { + std::string name = prefix.empty() ? suffix : prefix + "." + suffix; + auto iter = tensor_storage_map.find(name); + if (iter == tensor_storage_map.end()) { + return nullptr; + } + return &iter->second; + }; - static inline int64_t get_tensor_ne(const String2TensorStorage& tensor_storage_map, - const std::string& name, - int axis, - int64_t fallback) { - auto it = tensor_storage_map.find(name); - if (it == tensor_storage_map.end() || axis < 0 || axis >= GGML_MAX_DIMS) { - return fallback; - } - return it->second.ne[axis]; - } + bool inferred = false; - static inline int64_t get_tensor_ne0(const String2TensorStorage& tensor_storage_map, - const std::string& name, - int64_t fallback) { - return get_tensor_ne(tensor_storage_map, name, 0, fallback); - } - - static inline int count_module_blocks(const String2TensorStorage& tensor_storage_map, - const std::string& module_name) { - int max_block = -1; - const std::string prefix = module_name + "."; - for (const auto& pair : tensor_storage_map) { - const std::string& name = pair.first; - if (name.find(prefix) != 0) { - continue; + const TensorStorage* initial_norm = find_weight("initial_norm.weight"); + if (initial_norm != nullptr) { + config.mid_channels = initial_norm->ne[0]; + inferred = true; } - size_t begin = prefix.size(); - size_t end = name.find('.', begin); - if (end == std::string::npos) { - continue; - } - int index = atoi(name.substr(begin, end - begin).c_str()); - max_block = std::max(max_block, index); - } - return max_block + 1; - } - static inline LatentUpsamplerConfig detect_config_from_weights(const String2TensorStorage& tensor_storage_map) { - LatentUpsamplerConfig config; - config.mid_channels = get_tensor_ne0(tensor_storage_map, "initial_norm.weight", config.mid_channels); - config.in_channels = get_tensor_ne0(tensor_storage_map, "final_conv.bias", config.in_channels); - int detected_blocks = count_module_blocks(tensor_storage_map, "res_blocks"); - if (detected_blocks > 0) { - config.num_blocks_per_stage = detected_blocks; - } - config.rational_resampler = has_tensor(tensor_storage_map, "upsampler.conv.weight"); - int64_t upsampler_out_channels = get_tensor_ne0(tensor_storage_map, "upsampler.0.bias", 0); - config.spatial_upsample = config.rational_resampler || upsampler_out_channels == 4 * config.mid_channels; - config.temporal_upsample = upsampler_out_channels == 2 * config.mid_channels; - if (config.temporal_upsample) { - config.temporal_up_factor = 2; - } - if (config.rational_resampler) { - int64_t out_channels = get_tensor_ne(tensor_storage_map, - "upsampler.conv.weight", - 3, - config.mid_channels * 9); - if (config.mid_channels > 0 && out_channels % config.mid_channels == 0) { - int64_t ratio = out_channels / config.mid_channels; - int num = static_cast(std::round(std::sqrt(static_cast(ratio)))); - if (num > 0 && static_cast(num) * num == ratio) { - config.spatial_up_num = num; + const TensorStorage* final_conv = find_weight("final_conv.bias"); + if (final_conv != nullptr) { + config.in_channels = final_conv->ne[0]; + inferred = true; + } + + int detected_blocks = 0; + const std::string res_blocks_prefix = prefix.empty() ? "res_blocks." : prefix + ".res_blocks."; + for (const auto& [name, _] : tensor_storage_map) { + if (!starts_with(name, res_blocks_prefix)) { + continue; + } + size_t begin = res_blocks_prefix.size(); + size_t end = name.find('.', begin); + if (end == std::string::npos) { + continue; + } + try { + int idx = std::stoi(name.substr(begin, end - begin)); + detected_blocks = std::max(detected_blocks, idx + 1); + } catch (...) { } } - if (config.spatial_up_num == 3) { - config.spatial_down_den = 2; - config.spatial_scale = 1.5f; - } else if (config.spatial_up_num == 4) { - config.spatial_down_den = 1; - config.spatial_scale = 4.f; - } else { - config.spatial_down_den = 1; - config.spatial_scale = static_cast(config.spatial_up_num); + if (detected_blocks > 0) { + config.num_blocks_per_stage = detected_blocks; + inferred = true; } + + const TensorStorage* rational_upsampler_weight = find_weight("upsampler.conv.weight"); + const TensorStorage* upsampler_bias = find_weight("upsampler.0.bias"); + config.rational_resampler = rational_upsampler_weight != nullptr; + int64_t upsampler_out_channels = upsampler_bias == nullptr ? 0 : upsampler_bias->ne[0]; + config.spatial_upsample = config.rational_resampler || upsampler_out_channels == 4 * config.mid_channels; + config.temporal_upsample = upsampler_out_channels == 2 * config.mid_channels; + if (config.rational_resampler || upsampler_out_channels > 0) { + inferred = true; + } + if (config.temporal_upsample) { + config.temporal_up_factor = 2; + } + if (rational_upsampler_weight != nullptr) { + int64_t out_channels = rational_upsampler_weight->ne[3]; + if (config.mid_channels > 0 && out_channels % config.mid_channels == 0) { + int64_t ratio = out_channels / config.mid_channels; + int num = static_cast(std::round(std::sqrt(static_cast(ratio)))); + if (num > 0 && static_cast(num) * num == ratio) { + config.spatial_up_num = num; + } + } + if (config.spatial_up_num == 3) { + config.spatial_down_den = 2; + config.spatial_scale = 1.5f; + } else if (config.spatial_up_num == 4) { + config.spatial_down_den = 1; + config.spatial_scale = 4.f; + } else { + config.spatial_down_den = 1; + config.spatial_scale = static_cast(config.spatial_up_num); + } + } + + if (inferred) { + LOG_DEBUG("ltx latent upsampler: in_channels = %" PRId64 ", mid_channels = %" PRId64 ", num_blocks_per_stage = %d, spatial_scale = %.3f, temporal_up_factor = %d, rational_resampler = %d", + config.in_channels, + config.mid_channels, + config.num_blocks_per_stage, + config.spatial_scale, + config.temporal_up_factor, + config.rational_resampler); + } + return config; } - return config; - } + }; class VideoGroupNorm : public GGMLBlock { protected: @@ -240,20 +250,25 @@ namespace LTXVUpsampler { protected: int64_t channels; int stride; - ggml_tensor* kernel = nullptr; std::vector kernel_data; + std::string kernel_name; void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { + SD_UNUSED(ctx); SD_UNUSED(tensor_storage_map); if (stride == 1) { return; } - kernel = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 5, 5, 1, channels); - std::string name = prefix + "kernel"; - ggml_set_name(kernel, name.c_str()); + kernel_name = prefix + "kernel"; + } + public: + BlurDownsample(int64_t channels, int stride) + : channels(channels), + stride(stride) { + GGML_ASSERT(stride >= 1); static const float binomial[5] = {1.f, 4.f, 6.f, 4.f, 1.f}; kernel_data.resize(static_cast(5 * 5 * channels)); for (int64_t c = 0; c < channels; ++c) { @@ -266,26 +281,16 @@ namespace LTXVUpsampler { } } - public: - BlurDownsample(int64_t channels, int stride) - : channels(channels), - stride(stride) { - GGML_ASSERT(stride >= 1); - } - - void load_fixed_tensors() { - if (kernel == nullptr || kernel_data.empty()) { - return; - } - ggml_backend_tensor_set(kernel, kernel_data.data(), 0, kernel_data.size() * sizeof(float)); - } - ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { if (stride == 1) { return x; } - GGML_ASSERT(kernel != nullptr); + GGML_ASSERT(ctx != nullptr); + GGML_ASSERT(!kernel_data.empty()); GGML_ASSERT(x->ne[2] == channels); + ggml_tensor* kernel = ggml_new_tensor_4d(ctx->ggml_ctx, GGML_TYPE_F32, 5, 5, 1, channels); + ggml_set_name(kernel, kernel_name.empty() ? "blur_down.kernel" : kernel_name.c_str()); + ctx->bind_backend_tensor_data(kernel, kernel_data.data()); if (ctx->conv2d_direct_enabled) { return ggml_conv_2d_dw_direct(ctx->ggml_ctx, kernel, x, stride, stride, 2, 2, 1, 1); } @@ -311,11 +316,6 @@ namespace LTXVUpsampler { blocks["blur_down"] = std::shared_ptr(new BlurDownsample(mid_channels, den)); } - void load_fixed_tensors() { - auto blur_down = std::dynamic_pointer_cast(blocks["blur_down"]); - blur_down->load_fixed_tensors(); - } - ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { auto conv = std::dynamic_pointer_cast(blocks["conv"]); auto pixel_shuffle = std::dynamic_pointer_cast(blocks["pixel_shuffle"]); @@ -426,45 +426,17 @@ namespace LTXVUpsampler { sd::ggml_graph_cut::mark_graph_cut(x, "ltx_latent_upsampler.final", "x"); return x; } - - void load_fixed_tensors() { - if (!config.rational_resampler) { - return; - } - auto upsampler = std::dynamic_pointer_cast(blocks["upsampler"]); - upsampler->load_fixed_tensors(); - } }; struct LatentUpsamplerRunner : public GGMLRunner { + LatentUpsamplerConfig config; std::unique_ptr model; LatentUpsamplerRunner(ggml_backend_t backend, - ggml_backend_t params_backend) - : GGMLRunner(backend, params_backend) {} - - std::string get_desc() override { - return "ltx_latent_upsampler"; - } - - bool load_from_file(const std::string& file_path, int n_threads) { - LOG_INFO("loading LTX latent upsampler from '%s'", file_path.c_str()); - ModelLoader model_loader; - if (!model_loader.init_from_file(file_path)) { - LOG_ERROR("init LTX latent upsampler model loader from file failed: '%s'", file_path.c_str()); - return false; - } - - const auto& tensor_storage_map = model_loader.get_tensor_storage_map(); - bool has_regular_upsampler = has_tensor(tensor_storage_map, "upsampler.0.weight"); - bool has_rational_spatial = has_tensor(tensor_storage_map, "upsampler.conv.weight"); - if (!has_tensor(tensor_storage_map, "post_upsample_res_blocks.0.conv2.bias") || - (!has_regular_upsampler && !has_rational_spatial)) { - LOG_ERROR("unsupported LTX latent upsampler weights: expected upsampler tensors"); - return false; - } - - LatentUpsamplerConfig config = detect_config_from_weights(tensor_storage_map); + const String2TensorStorage& tensor_storage_map, + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), + config(LatentUpsamplerConfig::detect_from_weights(tensor_storage_map)) { if (config.dims != 3 || (!config.spatial_upsample && !config.temporal_upsample) || config.spatial_up_num < 1 || config.spatial_down_den < 1 || config.temporal_up_factor < 1) { LOG_ERROR("unsupported LTX latent upsampler config: dims=%d spatial=%d temporal=%d rational=%d scale=%.3f temporal_factor=%d", @@ -474,36 +446,21 @@ namespace LTXVUpsampler { config.rational_resampler, config.spatial_scale, config.temporal_up_factor); - return false; + return; } model = std::make_unique(config); model->init(params_ctx, tensor_storage_map, ""); - if (!alloc_params_buffer()) { - LOG_ERROR("LTX latent upsampler params buffer allocation failed"); - return false; - } + } - std::map tensors; - model->get_param_tensors(tensors); - std::set ignore_tensors; - if (config.rational_resampler) { - ignore_tensors.insert("upsampler.blur_down.kernel"); - } - if (!model_loader.load_tensors(tensors, ignore_tensors, n_threads)) { - LOG_ERROR("load LTX latent upsampler tensors failed"); - return false; - } - model->load_fixed_tensors(); + std::string get_desc() override { + return "ltx_latent_upsampler"; + } - LOG_INFO("LTX latent upsampler loaded: in_channels=%" PRId64 ", mid_channels=%" PRId64 ", blocks=%d, scale=%.3f, temporal_factor=%d, rational=%d", - config.in_channels, - config.mid_channels, - config.num_blocks_per_stage, - config.spatial_scale, - config.temporal_up_factor, - config.rational_resampler); - return true; + void get_param_tensors(std::map& tensors) { + if (model) { + model->get_param_tensors(tensors); + } } ggml_cgraph* build_graph(const sd::Tensor& x_tensor) { @@ -534,15 +491,15 @@ namespace LTXVUpsampler { (long long)x.shape()[4]); return {}; } - if (x.shape()[3] != model->config.in_channels) { + if (x.shape()[3] != config.in_channels) { LOG_ERROR("LTX latent upsampler expected %" PRId64 " channels, got %lld", - model->config.in_channels, + config.in_channels, (long long)x.shape()[3]); return {}; } size_t expected_dim = static_cast(x.dim()); auto get_graph = [&]() -> ggml_cgraph* { return build_graph(x); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), expected_dim); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), expected_dim); } }; diff --git a/otherarch/sdcpp/src/model/vae/auto_encoder_kl.hpp b/otherarch/sdcpp/src/model/vae/auto_encoder_kl.hpp index 49472ff47..478b18edb 100644 --- a/otherarch/sdcpp/src/model/vae/auto_encoder_kl.hpp +++ b/otherarch/sdcpp/src/model/vae/auto_encoder_kl.hpp @@ -213,9 +213,9 @@ protected: params["mix_factor"] = ggml_new_tensor_1d(ctx, wtype, 1); } - float get_alpha() { - float alpha = ggml_ext_backend_tensor_get_f32(params["mix_factor"]); - return sigmoid(alpha); + ggml_tensor* get_alpha(GGMLRunnerContext* ctx) { + auto mix_factor = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, params["mix_factor"]); + return ggml_sigmoid(ctx->ggml_ctx, mix_factor); } public: @@ -250,10 +250,12 @@ public: x = time_stack->forward(ctx, x); // b t c (h w) - float alpha = get_alpha(); - x = ggml_add(ctx->ggml_ctx, - ggml_ext_scale(ctx->ggml_ctx, x, alpha), - ggml_ext_scale(ctx->ggml_ctx, x_mix, 1.0f - alpha)); + auto alpha = get_alpha(ctx); + x = ggml_add(ctx->ggml_ctx, + x_mix, + ggml_mul(ctx->ggml_ctx, + ggml_sub(ctx->ggml_ctx, x, x_mix), + alpha)); x = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, x, 0, 2, 1, 3)); // b c t (h w) -> b t c (h w) x = ggml_reshape_4d(ctx->ggml_ctx, x, W, H, C, T * B); // b t c (h w) -> (b t) c h w @@ -664,13 +666,13 @@ struct AutoEncoderKL : public VAE { AutoEncoderKLModel ae; AutoEncoderKL(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - bool decode_only = false, - bool use_video_decoder = false, - SDVersion version = VERSION_SD1) - : decode_only(decode_only), VAE(version, backend, params_backend) { + bool decode_only = false, + bool use_video_decoder = false, + SDVersion version = VERSION_SD1, + std::shared_ptr weight_manager = nullptr) + : VAE(version, backend, prefix, weight_manager), decode_only(decode_only) { if (sd_version_is_sd1(version) || sd_version_is_sd2(version)) { scale_factor = 0.18215f; shift_factor = 0.f; @@ -718,8 +720,8 @@ struct AutoEncoderKL : public VAE { return "vae"; } - void get_param_tensors(std::map& tensors, const std::string prefix) override { - ae.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) override { + ae.get_param_tensors(tensors, weight_prefix); } ggml_cgraph* build_graph(const sd::Tensor& z_tensor, bool decode_graph) { @@ -742,7 +744,7 @@ struct AutoEncoderKL : public VAE { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(z, decode_graph); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), z.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), z.dim()); } sd::Tensor gaussian_latent_sample(const sd::Tensor& moments, std::shared_ptr rng) { diff --git a/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp b/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp index d41a79a43..997c57a5b 100644 --- a/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp +++ b/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp @@ -1,4 +1,4 @@ -#ifndef __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__ +#ifndef __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__ #define __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__ #include @@ -9,6 +9,7 @@ #include "core/ggml_extend.hpp" #include "model_loader.h" +#include "model_manager.h" namespace LTXV { @@ -997,13 +998,15 @@ namespace LTXV { struct LTXAudioVAERunner : public GGMLRunner { LTXAudioVAEConfig config; LTXAudioVAE model; + std::string weight_prefix; sd::Tensor bwe_skip_filter_tensor; LTXAudioVAERunner(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, - const std::string& prefix = "") - : GGMLRunner(backend, params_backend), + const std::string& prefix = "", + std::shared_ptr weight_manager = nullptr) + : GGMLRunner(backend, weight_manager), + weight_prefix(prefix), config(LTXAudioVAEConfig::detect_from_weights(tensor_storage_map)), model(config) { model.init(params_ctx, tensor_storage_map, prefix); @@ -1013,11 +1016,11 @@ namespace LTXV { } } - void get_param_tensors(std::map& tensors, const std::string prefix) { - model.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) { + model.get_param_tensors(tensors, weight_prefix); } - size_t get_params_buffer_size() { + size_t get_params_mem_size() { return model.get_params_mem_size(); } @@ -1037,7 +1040,7 @@ namespace LTXV { ggml_build_forward_expand(gf, waveform); return gf; }; - auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), 4); + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), 4); int64_t t1 = ggml_time_ms(); LOG_INFO("ltx audio vae decode completed, taking %.2fs", (t1 - t0) * 1.0f / 1000); return result; @@ -1064,7 +1067,8 @@ namespace LTXV { // ggml_backend_t backend = ggml_backend_cuda_init(0); LOG_INFO("loading ltx audio vae from '%s'", model_path.c_str()); - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file(model_path)) { LOG_ERROR("init model loader from file failed: '%s'", model_path.c_str()); return; @@ -1072,20 +1076,17 @@ namespace LTXV { auto& tensor_storage_map = model_loader.get_tensor_storage_map(); auto ltx_audio_vae = std::make_shared(backend, - backend, tensor_storage_map, - prefix); + prefix, + model_manager); - if (!ltx_audio_vae->alloc_params_buffer()) { - LOG_ERROR("ltx audio vae buffer allocation failed"); - return; - } - - std::map tensors; - ltx_audio_vae->get_param_tensors(tensors, ""); - - if (!model_loader.load_tensors(tensors)) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("LTX audio VAE test", + *ltx_audio_vae, + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register ltx audio vae tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/vae/ltx_vae.hpp b/otherarch/sdcpp/src/model/vae/ltx_vae.hpp index 86fcdcb09..a19ce820d 100644 --- a/otherarch/sdcpp/src/model/vae/ltx_vae.hpp +++ b/otherarch/sdcpp/src/model/vae/ltx_vae.hpp @@ -957,8 +957,8 @@ namespace LTXVAE { ggml_tensor* scaled_timestep = timestep; if (timestep_conditioning) { - auto multiplier = ggml_ext_backend_tensor_get_f32(params["timestep_scale_multiplier"]); - scaled_timestep = ggml_ext_scale(ctx->ggml_ctx, timestep, multiplier); + auto multiplier = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, params["timestep_scale_multiplier"]); + scaled_timestep = ggml_mul(ctx->ggml_ctx, timestep, multiplier); } x = conv_in->forward(ctx, x, causal_decoder); @@ -1008,8 +1008,8 @@ namespace LTXVAE { ggml_tensor* scaled_timestep = timestep; if (timestep_conditioning && timestep != nullptr) { - auto multiplier = ggml_ext_backend_tensor_get_f32(params["timestep_scale_multiplier"]); - scaled_timestep = ggml_ext_scale(ctx->ggml_ctx, timestep, multiplier); + auto multiplier = ggml_ext_cast_f32(ctx->ggml_ctx, ctx->backend, params["timestep_scale_multiplier"]); + scaled_timestep = ggml_mul(ctx->ggml_ctx, timestep, multiplier); } // conv_in with feat_map for left temporal context @@ -1223,11 +1223,11 @@ struct LTXVideoVAE : public VAE { LTXVAE::VideoVAE vae; LTXVideoVAE(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string& prefix, - bool decode_only = true, - SDVersion version = VERSION_LTXAV) + bool decode_only = true, + SDVersion version = VERSION_LTXAV, + std::shared_ptr weight_manager = nullptr) : decode_only(decode_only), ltx_vae_version(LTXVAE::detect_ltx_vae_version(tensor_storage_map, prefix)), timestep_conditioning(LTXVAE::detect_ltx_vae_timestep_conditioning(tensor_storage_map, prefix)), @@ -1239,7 +1239,7 @@ struct LTXVideoVAE : public VAE { patch_size, tensor_storage_map, prefix), - VAE(version, backend, params_backend) { + VAE(version, backend, prefix, weight_manager) { vae.init(params_ctx, tensor_storage_map, prefix); decode_timestep_tensor.values()[0] = vae.decode_timestep; } @@ -1271,8 +1271,8 @@ struct LTXVideoVAE : public VAE { } } - void get_param_tensors(std::map& tensors, const std::string prefix) override { - vae.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) override { + vae.get_param_tensors(tensors, weight_prefix); } struct TemporalTilePlan { @@ -1396,7 +1396,7 @@ struct LTXVideoVAE : public VAE { static_cast(start), chunk_overlap); }; - auto chunk = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, true), + auto chunk = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, true, true, true), expected_dim); if (chunk.empty()) { free_cache_ctx_and_buffer(); @@ -1426,7 +1426,7 @@ struct LTXVideoVAE : public VAE { const sd::Tensor& z, bool decode_graph) override { if (!decode_graph && decode_only) { - LOG_ERROR("LTX video VAE encode requires encoder weights; create the context with vae_decode_only=false"); + LOG_ERROR("LTX video VAE encode requires encoder weights"); return {}; } sd::Tensor input = z; @@ -1452,7 +1452,7 @@ struct LTXVideoVAE : public VAE { auto get_graph = [&]() -> ggml_cgraph* { return build_graph(input, decode_graph); }; - auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), expected_dim); + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), expected_dim); if (result.empty()) { return {}; } @@ -1465,7 +1465,7 @@ struct LTXVideoVAE : public VAE { auto get_graph = [&]() -> ggml_cgraph* { return build_latent_statistics_graph(z, normalize); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), static_cast(z.dim())); } @@ -1521,7 +1521,8 @@ struct LTXVideoVAE : public VAE { ggml_backend_t backend = sd_backend_cpu_init(); LOG_INFO("loading ltx vae from '%s'", model_path.c_str()); - ModelLoader model_loader; + auto model_manager = std::make_shared(); + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(model_path, "vae.")) { LOG_ERROR("init model loader from file failed: '%s'", model_path.c_str()); return; @@ -1529,22 +1530,19 @@ struct LTXVideoVAE : public VAE { auto& tensor_storage_map = model_loader.get_tensor_storage_map(); std::shared_ptr vae = std::make_shared(backend, - backend, tensor_storage_map, "first_stage_model", true, - VERSION_LTXAV); + VERSION_LTXAV, + model_manager); - if (!vae->alloc_params_buffer()) { - LOG_ERROR("vae buffer allocation failed"); - return; - } - - std::map tensors; - vae->get_param_tensors(tensors, "first_stage_model"); - - if (!model_loader.load_tensors(tensors)) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("LTX VAE test", + *vae, + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register ltx vae tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model/vae/tae.hpp b/otherarch/sdcpp/src/model/vae/tae.hpp index fcb62c2ea..7c6e1d35c 100644 --- a/otherarch/sdcpp/src/model/vae/tae.hpp +++ b/otherarch/sdcpp/src/model/vae/tae.hpp @@ -623,14 +623,14 @@ struct TinyImageAutoEncoder : public VAE { bool decode_only = false; TinyImageAutoEncoder(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - bool decoder_only = true, - SDVersion version = VERSION_SD1) - : decode_only(decoder_only), - taesd(decoder_only, version), - VAE(version, backend, params_backend) { + bool decoder_only = true, + SDVersion version = VERSION_SD1, + std::shared_ptr weight_manager = nullptr) + : VAE(version, backend, "tae", weight_manager), + decode_only(decoder_only), + taesd(decoder_only, version) { scale_input = false; taesd.init(params_ctx, tensor_storage_map, prefix); } @@ -639,8 +639,8 @@ struct TinyImageAutoEncoder : public VAE { return "taesd"; } - void get_param_tensors(std::map& tensors, const std::string prefix) { - taesd.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) override { + taesd.get_param_tensors(tensors, weight_prefix); } sd::Tensor vae_output_to_latents(const sd::Tensor& vae_output, std::shared_ptr rng) override { @@ -676,7 +676,7 @@ struct TinyImageAutoEncoder : public VAE { return build_graph(z_tensor, decode_graph); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), z_tensor.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), z_tensor.dim()); } }; @@ -686,13 +686,13 @@ struct TinyVideoAutoEncoder : public VAE { bool is_wide = false; TinyVideoAutoEncoder(ggml_backend_t backend, - ggml_backend_t params_backend, const String2TensorStorage& tensor_storage_map, const std::string prefix, - bool decoder_only = true, - SDVersion version = VERSION_WAN2) - : decode_only(decoder_only), - VAE(version, backend, params_backend) { + bool decoder_only = true, + SDVersion version = VERSION_WAN2, + std::shared_ptr weight_manager = nullptr) + : VAE(version, backend, "tae", weight_manager), + decode_only(decoder_only) { for (auto tensor_storage : tensor_storage_map) { if (tensor_storage.first.find(prefix + ".3.conv.6.weight") != std::string::npos) { is_wide = true; @@ -708,8 +708,8 @@ struct TinyVideoAutoEncoder : public VAE { return "taehv"; } - void get_param_tensors(std::map& tensors, const std::string prefix) { - taehv.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) override { + taehv.get_param_tensors(tensors, weight_prefix); } sd::Tensor vae_output_to_latents(const sd::Tensor& vae_output, std::shared_ptr rng) override { @@ -746,7 +746,7 @@ struct TinyVideoAutoEncoder : public VAE { return build_graph(z_tensor, decode_graph); }; - return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false), z_tensor.dim()); + return restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), z_tensor.dim()); } }; diff --git a/otherarch/sdcpp/src/model/vae/vae.hpp b/otherarch/sdcpp/src/model/vae/vae.hpp index bd0ce6c4e..af091bb57 100644 --- a/otherarch/sdcpp/src/model/vae/vae.hpp +++ b/otherarch/sdcpp/src/model/vae/vae.hpp @@ -1,12 +1,14 @@ -#ifndef __SD_MODEL_VAE_VAE_HPP__ +#ifndef __SD_MODEL_VAE_VAE_HPP__ #define __SD_MODEL_VAE_VAE_HPP__ #include "core/tensor_ggml.hpp" #include "model/common/block.hpp" +#include "model_manager.h" struct VAE : public GGMLRunner { protected: SDVersion version; + std::string weight_prefix; bool scale_input = true; virtual sd::Tensor _compute(const int n_threads, const sd::Tensor& z, @@ -62,8 +64,11 @@ protected: } public: - VAE(SDVersion version, ggml_backend_t backend, ggml_backend_t params_backend) - : version(version), GGMLRunner(backend, params_backend) {} + VAE(SDVersion version, + ggml_backend_t backend, + const std::string& weight_prefix = "", + std::shared_ptr weight_manager = nullptr) + : version(version), weight_prefix(weight_prefix), GGMLRunner(backend, weight_manager) {} int get_scale_factor() { int scale_factor = 8; @@ -214,7 +219,7 @@ public: virtual sd::Tensor vae_output_to_latents(const sd::Tensor& vae_output, std::shared_ptr rng) = 0; virtual sd::Tensor diffusion_to_vae_latents(const sd::Tensor& latents) = 0; virtual sd::Tensor vae_to_diffusion_latents(const sd::Tensor& latents) = 0; - virtual void get_param_tensors(std::map& tensors, const std::string prefix) = 0; + virtual void get_param_tensors(std::map& tensors) = 0; virtual void set_conv2d_scale(float scale) { SD_UNUSED(scale); }; virtual void set_temporal_tiling_enabled(bool enabled) { SD_UNUSED(enabled); }; virtual void set_tiling_params(const sd_tiling_params_t& params) { @@ -223,8 +228,10 @@ public: }; struct FakeVAE : public VAE { - FakeVAE(SDVersion version, ggml_backend_t backend, ggml_backend_t params_backend) - : VAE(version, backend, params_backend) {} + FakeVAE(SDVersion version, + ggml_backend_t backend, + std::shared_ptr weight_manager = nullptr) + : VAE(version, backend, "", weight_manager) {} int get_encoder_output_channels(int input_channels) { return input_channels; @@ -251,7 +258,7 @@ struct FakeVAE : public VAE { return latents; } - void get_param_tensors(std::map& tensors, const std::string prefix) override {} + void get_param_tensors(std::map& tensors) override {} std::string get_desc() override { return "fake_vae"; diff --git a/otherarch/sdcpp/src/model/vae/wan_vae.hpp b/otherarch/sdcpp/src/model/vae/wan_vae.hpp index c20764cde..8a845c7ca 100644 --- a/otherarch/sdcpp/src/model/vae/wan_vae.hpp +++ b/otherarch/sdcpp/src/model/vae/wan_vae.hpp @@ -1124,12 +1124,12 @@ namespace WAN { WanVAE ae; WanVAERunner(ggml_backend_t backend, - ggml_backend_t params_backend, - const String2TensorStorage& tensor_storage_map = {}, - const std::string prefix = "", - bool decode_only = false, - SDVersion version = VERSION_WAN2) - : decode_only(decode_only), ae(decode_only, version == VERSION_WAN2_2_TI2V), VAE(version, backend, params_backend) { + const String2TensorStorage& tensor_storage_map = {}, + const std::string prefix = "", + bool decode_only = false, + SDVersion version = VERSION_WAN2, + std::shared_ptr weight_manager = nullptr) + : VAE(version, backend, prefix, weight_manager), decode_only(decode_only), ae(decode_only, version == VERSION_WAN2_2_TI2V) { ae.init(params_ctx, tensor_storage_map, prefix); } @@ -1137,8 +1137,8 @@ namespace WAN { return "wan_vae"; } - void get_param_tensors(std::map& tensors, const std::string prefix) override { - ae.get_param_tensors(tensors, prefix); + void get_param_tensors(std::map& tensors) override { + ae.get_param_tensors(tensors, weight_prefix); } sd::Tensor vae_output_to_latents(const sd::Tensor& vae_output, std::shared_ptr rng) override { @@ -1255,7 +1255,7 @@ namespace WAN { return build_graph(input, decode_graph); } }; - auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, true), + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, true, true, true), input.empty() ? z.dim() : input.dim()); if (!result.empty() && z.dim() == 4) { result.squeeze_(2); @@ -1268,7 +1268,7 @@ namespace WAN { auto get_graph = [&]() -> ggml_cgraph* { return build_graph_partial(z, decode_graph, i); }; - auto out_opt = GGMLRunner::compute(get_graph, n_threads, true); + auto out_opt = GGMLRunner::compute(get_graph, n_threads, true, true, true); if (!out_opt.has_value()) { return {}; } @@ -1281,7 +1281,7 @@ namespace WAN { sd::Tensor output = std::move(out); for (i = 1; i < t; i++) { - auto chunk_opt = GGMLRunner::compute(get_graph, n_threads, true); + auto chunk_opt = GGMLRunner::compute(get_graph, n_threads, true, true, true); if (!chunk_opt.has_value()) { return {}; } @@ -1327,27 +1327,24 @@ namespace WAN { // ggml_backend_t backend = ggml_backend_cuda_init(0); ggml_backend_t backend = sd_backend_cpu_init(); ggml_type model_data_type = GGML_TYPE_F16; - std::shared_ptr vae = std::make_shared(backend, backend, String2TensorStorage{}, "", false, VERSION_WAN2_2_TI2V); + auto model_manager = std::make_shared(); + std::shared_ptr vae = std::make_shared(backend, String2TensorStorage{}, "first_stage_model", false, VERSION_WAN2_2_TI2V, model_manager); { LOG_INFO("loading from '%s'", file_path.c_str()); - if (!vae->alloc_params_buffer()) { - LOG_ERROR("vae buffer allocation failed"); - return; - } - std::map tensors; - vae->get_param_tensors(tensors, "first_stage_model"); - - ModelLoader model_loader; + ModelLoader& model_loader = model_manager->loader(); if (!model_loader.init_from_file_and_convert_name(file_path, "vae.")) { LOG_ERROR("init model loader from file failed: '%s'", file_path.c_str()); return; } - bool success = model_loader.load_tensors(tensors); - - if (!success) { - LOG_ERROR("load tensors from model loader failed"); + if (!model_manager->register_runner_params("Wan VAE test", + *vae, + ModelManager::ResidencyMode::ParamBackend, + backend, + backend) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register wan vae tensors with model manager failed"); return; } diff --git a/otherarch/sdcpp/src/model_loader.cpp b/otherarch/sdcpp/src/model_loader.cpp index da53772f8..a1788bfbd 100644 --- a/otherarch/sdcpp/src/model_loader.cpp +++ b/otherarch/sdcpp/src/model_loader.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -218,10 +219,28 @@ void convert_tensor(void* src, /*================================================= ModelLoader ==================================================*/ +ModelLoader::ModelLoader() + : n_threads_(sd_get_num_physical_cores()) { +} + +size_t ModelLoader::add_file_path(const std::string& file_path) { + if (model_files_processed) { + file_data.clear(); + model_files_processed = false; + } + file_paths_.push_back(file_path); + return file_paths_.size() - 1; +} + void ModelLoader::add_tensor_storage(const TensorStorage& tensor_storage) { tensor_storage_map[tensor_storage.name] = tensor_storage; } +void ModelLoader::set_n_threads(int n_threads) { + n_threads_ = n_threads > 0 ? n_threads : sd_get_num_physical_cores(); + LOG_DEBUG("using %d threads for model loading", n_threads_); +} + bool ModelLoader::init_from_file(const std::string& file_path, const std::string& prefix) { return [&](const std::string& file_path) { // kcpp u8 file path if (is_directory(file_path)) { @@ -287,8 +306,7 @@ bool ModelLoader::init_from_gguf_file(const std::string& file_path, const std::s return false; } - file_paths_.push_back(file_path); - size_t file_index = file_paths_.size() - 1; + size_t file_index = add_file_path(file_path); for (auto& tensor_storage : tensor_storages) { // LOG_DEBUG("%s", tensor_storage.name.c_str()); @@ -316,8 +334,7 @@ bool ModelLoader::init_from_safetensors_file(const std::string& file_path, const return false; } - file_paths_.push_back(file_path); - size_t file_index = file_paths_.size() - 1; + size_t file_index = add_file_path(file_path); for (auto& tensor_storage : tensor_storages) { if (is_unused_tensor(tensor_storage.name)) { @@ -353,8 +370,7 @@ bool ModelLoader::init_from_torch_legacy_file(const std::string& file_path, cons return false; } - file_paths_.push_back(file_path); - size_t file_index = file_paths_.size() - 1; + size_t file_index = add_file_path(file_path); for (auto& tensor_storage : tensor_storages) { if (is_unused_tensor(tensor_storage.name)) { @@ -384,8 +400,7 @@ bool ModelLoader::init_from_torch_zip_file(const std::string& file_path, const s return false; } - file_paths_.push_back(file_path); - size_t file_index = file_paths_.size() - 1; + size_t file_index = add_file_path(file_path); for (auto& tensor_storage : tensor_storages) { if (!starts_with(tensor_storage.name, prefix)) { @@ -788,8 +803,6 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) { return; } - int64_t start_time = ggml_time_ms(); - std::vector processed_tensor_storages; for (const auto& [name, tensor_storage] : tensor_storage_map) { if (is_unused_tensor(tensor_storage.name)) { @@ -840,20 +853,12 @@ void ModelLoader::process_model_files(bool enable_mmap, bool writable_mmap) { } else { LOG_WARN("failed to memory-map '%s' (falling back to read())", file_path.c_str()); } - } else if (!is_zip) { - LOG_INFO("NOT using mmap for '%s' (mmap disabled by caller)", - file_path.c_str()); } file_data.push_back(std::move(fdata)); } model_files_processed = true; - - int64_t end_time = ggml_time_ms(); - int64_t process_time_ms = end_time - start_time; - - LOG_INFO("model files processing completed in %.2fs", process_time_ms / 1000.f); } std::vector ModelLoader::mmap_tensors(std::map& tensors, @@ -947,7 +952,9 @@ std::vector ModelLoader::mmap_tensors(std::map* target_tensor_names) { process_model_files(enable_mmap, false); std::atomic read_time_ms(0); @@ -956,14 +963,26 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread std::atomic convert_time_ms(0); std::atomic bytes_processed(0); - int num_threads_to_use = n_threads_p > 0 ? n_threads_p : sd_get_num_physical_cores(); - LOG_DEBUG("using %d threads for model loading", num_threads_to_use); + int num_threads_to_use = n_threads_; int64_t start_time = ggml_time_ms(); size_t total_tensors_to_process = 0; + std::vector file_tensors_to_process_counts; + file_tensors_to_process_counts.reserve(file_data.size()); for (const auto& fdata : file_data) { - total_tensors_to_process += fdata.tensors.size(); + size_t file_tensors_to_process = 0; + if (target_tensor_names == nullptr) { + file_tensors_to_process = fdata.tensors.size(); + } else { + for (const TensorStorage& tensor_storage : fdata.tensors) { + if (target_tensor_names->find(tensor_storage.name) != target_tensor_names->end()) { + file_tensors_to_process++; + } + } + } + file_tensors_to_process_counts.push_back(file_tensors_to_process); + total_tensors_to_process += file_tensors_to_process; } bool success = true; @@ -971,17 +990,38 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread const int64_t t_start = start_time; int last_n_threads = 1; - for (auto& fdata : file_data) { + for (size_t file_index = 0; file_index < file_data.size(); ++file_index) { + auto& fdata = file_data[file_index]; const std::string& file_path = fdata.path; - LOG_DEBUG("loading tensors from %s", file_path.c_str()); const std::vector& file_tensors = fdata.tensors; + std::vector tensors_to_process; + size_t file_tensors_to_process = file_tensors_to_process_counts[file_index]; + tensors_to_process.reserve(file_tensors_to_process); + if (target_tensor_names == nullptr) { + for (const TensorStorage& tensor_storage : file_tensors) { + tensors_to_process.push_back(&tensor_storage); + } + } else { + for (const TensorStorage& tensor_storage : file_tensors) { + if (target_tensor_names->find(tensor_storage.name) != target_tensor_names->end()) { + tensors_to_process.push_back(&tensor_storage); + } + } + } + if (tensors_to_process.empty()) { + continue; + } + LOG_DEBUG("loading %zu/%zu tensors from %s", + tensors_to_process.size(), + file_tensors.size(), + file_path.c_str()); bool is_zip = fdata.is_zip; std::shared_ptr mmapped = fdata.mmapped; - int n_threads = is_zip ? 1 : std::min(num_threads_to_use, (int)file_tensors.size()); + int n_threads = is_zip ? 1 : std::min(num_threads_to_use, (int)tensors_to_process.size()); if (n_threads < 1) { n_threads = 1; } @@ -990,6 +1030,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread std::atomic tensor_idx(0); std::atomic failed(false); std::vector workers; + std::mutex rpc_backend_mutex; for (int i = 0; i < n_threads; ++i) { workers.emplace_back([&, file_path, is_zip]() { @@ -1017,11 +1058,11 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread while (true) { int64_t t0, t1; size_t idx = tensor_idx.fetch_add(1); - if (idx >= file_tensors.size() || failed) { + if (idx >= tensors_to_process.size() || failed) { break; } - const TensorStorage& tensor_storage = file_tensors[idx]; + const TensorStorage& tensor_storage = *tensors_to_process[idx]; ggml_tensor* dst_tensor = nullptr; t0 = ggml_time_ms(); @@ -1146,7 +1187,19 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread if (dst_tensor->buffer != nullptr && !ggml_backend_buffer_is_host(dst_tensor->buffer)) { t0 = ggml_time_ms(); - ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); + + // RPC backends require serialized access to prevent concurrency issues + const char* buffer_type_name = ggml_backend_buft_name(ggml_backend_buffer_get_type(dst_tensor->buffer)); + bool is_rpc_buffer = buffer_type_name != nullptr && + std::string(buffer_type_name).find("RPC") != std::string::npos; + + if (is_rpc_buffer) { + std::lock_guard lock(rpc_backend_mutex); + ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); + } else { + ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); + } + t1 = ggml_time_ms(); copy_to_backend_time_ms.fetch_add(t1 - t0); } @@ -1161,16 +1214,18 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread while (true) { size_t current_idx = tensor_idx.load(); - if (current_idx >= file_tensors.size() || failed) { + if (current_idx >= tensors_to_process.size() || failed) { break; } size_t curr_num = total_tensors_processed + current_idx; float elapsed_seconds = (ggml_time_ms() - t_start) / 1000.0f; - pretty_bytes_progress(static_cast(curr_num), - static_cast(total_tensors_to_process), - bytes_processed.load(), - elapsed_seconds); - std::this_thread::sleep_for(std::chrono::milliseconds(200)); + if (total_tensors_to_process > 0) { + pretty_bytes_progress(static_cast(curr_num), + static_cast(total_tensors_to_process), + bytes_processed.load(), + elapsed_seconds); + } + std::this_thread::sleep_for(std::chrono::milliseconds(total_tensors_to_process <= 4 ? 10 : 200)); } for (auto& w : workers) { @@ -1181,12 +1236,14 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread success = false; break; } - total_tensors_processed += file_tensors.size(); - pretty_bytes_progress(static_cast(total_tensors_processed), - static_cast(total_tensors_to_process), - bytes_processed.load(), - (ggml_time_ms() - t_start) / 1000.0f); - if (total_tensors_processed < total_tensors_to_process) { + total_tensors_processed += tensors_to_process.size(); + if (total_tensors_to_process > 0) { + pretty_bytes_progress(static_cast(total_tensors_processed), + static_cast(total_tensors_to_process), + bytes_processed.load(), + (ggml_time_ms() - t_start) / 1000.0f); + } + if (total_tensors_processed < total_tensors_to_process && total_tensors_to_process > 0) { printf("\n"); } } @@ -1201,9 +1258,77 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_thread return success; } +bool ModelLoader::load_float_tensor(const std::string& name, + std::vector& data, + int n_threads, + bool use_mmap) { + data.clear(); + + auto tensor_storage_it = tensor_storage_map.find(name); + if (tensor_storage_it == tensor_storage_map.end()) { + return false; + } + + const TensorStorage& tensor_storage = tensor_storage_it->second; + int64_t n_elements = tensor_storage.nelements(); + if (n_elements <= 0) { + LOG_ERROR("tensor '%s' has invalid element count: %" PRId64, name.c_str(), n_elements); + return false; + } + if (tensor_storage.n_dims <= 0 || tensor_storage.n_dims > GGML_MAX_DIMS) { + LOG_ERROR("tensor '%s' has unsupported dims: %d", name.c_str(), tensor_storage.n_dims); + return false; + } + + std::vector loaded_data(static_cast(n_elements)); + ggml_init_params params; + params.mem_size = ggml_tensor_overhead(); + params.mem_buffer = nullptr; + params.no_alloc = true; + + ggml_context* ctx = ggml_init(params); + if (ctx == nullptr) { + LOG_ERROR("failed to create context for tensor '%s'", name.c_str()); + return false; + } + + ggml_tensor* tensor = ggml_new_tensor(ctx, GGML_TYPE_F32, tensor_storage.n_dims, tensor_storage.ne); + ggml_set_name(tensor, name.c_str()); + tensor->data = loaded_data.data(); + + bool loaded = false; + auto on_new_tensor_cb = [&](const TensorStorage& current_tensor_storage, ggml_tensor** dst_tensor) -> bool { + *dst_tensor = nullptr; + if (current_tensor_storage.name != name) { + return true; + } + if (current_tensor_storage.nelements() != n_elements) { + LOG_ERROR("tensor '%s' element count changed during load", name.c_str()); + return false; + } + *dst_tensor = tensor; + loaded = true; + return true; + }; + + std::set target_tensor_names{name}; + if (n_threads > 0) { + set_n_threads(n_threads); + } + bool success = load_tensors(on_new_tensor_cb, use_mmap, &target_tensor_names); + ggml_free(ctx); + + if (!success || !loaded) { + data.clear(); + return false; + } + + data = std::move(loaded_data); + return true; +} + bool ModelLoader::load_tensors(std::map& tensors, std::set ignore_tensors, - int n_threads, bool enable_mmap) { std::set tensor_names_in_file; std::mutex tensor_names_mutex; @@ -1247,7 +1372,7 @@ bool ModelLoader::load_tensors(std::map& tensors, return true; }; - bool success = load_tensors(on_new_tensor_cb, n_threads, enable_mmap); + bool success = load_tensors(on_new_tensor_cb, enable_mmap); if (!success) { LOG_ERROR("load tensors from file failed"); return false; diff --git a/otherarch/sdcpp/src/model_loader.h b/otherarch/sdcpp/src/model_loader.h index 3863116f3..0d894a1a2 100644 --- a/otherarch/sdcpp/src/model_loader.h +++ b/otherarch/sdcpp/src/model_loader.h @@ -34,7 +34,9 @@ protected: std::vector file_data; bool model_files_processed = false; String2TensorStorage tensor_storage_map; + int n_threads_; + size_t add_file_path(const std::string& file_path); void add_tensor_storage(const TensorStorage& tensor_storage); bool init_from_gguf_file(const std::string& file_path, const std::string& prefix = ""); @@ -44,6 +46,8 @@ protected: bool init_from_diffusers_file(const std::string& file_path, const std::string& prefix = ""); public: + ModelLoader(); + bool init_from_file(const std::string& file_path, const std::string& prefix = ""); void convert_tensors_name(); bool init_from_file_and_convert_name(const std::string& file_path, @@ -55,16 +59,23 @@ public: std::map get_diffusion_model_wtype_stat(); std::map get_vae_wtype_stat(); String2TensorStorage& get_tensor_storage_map() { return tensor_storage_map; } + const String2TensorStorage& get_tensor_storage_map() const { return tensor_storage_map; } + void set_n_threads(int n_threads); void set_wtype_override(ggml_type wtype, std::string tensor_type_rules = ""); void process_model_files(bool enable_mmap = false, bool writable_mmap = true); std::vector mmap_tensors(std::map& tensors, std::set ignore_tensors = {}, bool writable = true); - bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, int n_threads = 0, bool use_mmap = false); + bool load_tensors(on_new_tensor_cb_t on_new_tensor_cb, + bool use_mmap = false, + const std::set* target_tensor_names = nullptr); bool load_tensors(std::map& tensors, std::set ignore_tensors = {}, - int n_threads = 0, bool use_mmap = false); + bool load_float_tensor(const std::string& name, + std::vector& data, + int n_threads = 0, + bool use_mmap = false); std::vector get_tensor_names() const { std::vector names; diff --git a/otherarch/sdcpp/src/model_manager.cpp b/otherarch/sdcpp/src/model_manager.cpp new file mode 100644 index 000000000..5287e1069 --- /dev/null +++ b/otherarch/sdcpp/src/model_manager.cpp @@ -0,0 +1,939 @@ +#include "model_manager.h" + +#include +#include +#include +#include +#include + +#include "core/ggml_extend_backend.h" +#include "core/util.h" +#include "model/adapter/lora.hpp" + +static size_t aligned_offset(const void* buffer, size_t offset, size_t alignment) { + GGML_ASSERT(alignment != 0 && (alignment & (alignment - 1)) == 0); + size_t align = (alignment - ((reinterpret_cast(buffer) + offset) % alignment)) % alignment; + return offset + align; +} + +static bool lora_specs_equal(const std::vector& lhs, + const std::vector& rhs) { + if (lhs.size() != rhs.size()) { + return false; + } + for (size_t i = 0; i < lhs.size(); ++i) { + if (lhs[i].path != rhs[i].path || + lhs[i].multiplier != rhs[i].multiplier || + lhs[i].is_high_noise != rhs[i].is_high_noise || + lhs[i].tensor_name_prefix_filter != rhs[i].tensor_name_prefix_filter || + lhs[i].required != rhs[i].required) { + return false; + } + } + return true; +} + +static std::string lora_id(const ModelManager::LoraSpec& lora) { + return lora.is_high_noise ? "|high_noise|" + lora.path : lora.path; +} + +static bool backend_supports_host_buffer(ggml_backend_t backend) { + if (backend == nullptr) { + return false; + } + if (sd_backend_is_cpu(backend)) { + return true; + } + ggml_backend_dev_t dev = ggml_backend_get_device(backend); + if (dev == nullptr) { + return false; + } + ggml_backend_dev_props props; + ggml_backend_dev_get_props(dev, &props); + return props.caps.buffer_from_host_ptr; +} + +ModelManager::~ModelManager() { + release_all(); +} + +void ModelManager::set_common_ignore_tensors(std::set ignore_tensors) { + common_ignore_tensors_ = std::move(ignore_tensors); +} + +void ModelManager::set_loras(std::vector loras, SDVersion version) { + if (loras.empty() && loras_.empty()) { + lora_version_ = version; + return; + } + if (lora_version_ == version && lora_specs_equal(loras_, loras)) { + return; + } + + loras_ = std::move(loras); + lora_version_ = version; + current_lora_epoch_++; + reset_lora_applied_params(); +} + +std::set ModelManager::tensor_names() const { + std::set names; + for (const auto& state : tensor_states_) { + if (state != nullptr) { + names.insert(state->name); + } + } + return names; +} + +size_t estimate_tensors_size(const std::map& tensors) { + size_t size = 0; + std::unordered_set seen; + for (const auto& pair : tensors) { + ggml_tensor* tensor = pair.second; + if (tensor == nullptr || seen.find(tensor) != seen.end()) { + continue; + } + seen.insert(tensor); + size += ggml_nbytes(tensor); + } + return size; +} + +bool ModelManager::register_param_tensors(const std::string& desc, + std::map tensors, + ResidencyMode residency_mode, + ggml_backend_t compute_backend, + ggml_backend_t params_backend, + size_t* registered_tensor_size) { + if (desc.empty()) { + LOG_ERROR("model manager tensor desc is empty"); + return false; + } + if (registered_tensor_size != nullptr) { + *registered_tensor_size += estimate_tensors_size(tensors); + } + + std::vector> new_states; + new_states.reserve(tensors.size()); + + for (const auto& pair : tensors) { + const std::string& name = pair.first; + ggml_tensor* tensor = pair.second; + if (tensor == nullptr) { + continue; + } + if (tensor_states_by_name_.find(name) != tensor_states_by_name_.end()) { + LOG_ERROR("model manager tensor name '%s' is already registered", name.c_str()); + return false; + } + ggml_set_name(tensor, name.c_str()); + + auto state = std::make_unique(); + state->name = name; + state->tensor = tensor; + state->desc = desc; + state->residency_mode = residency_mode; + state->compute_backend = compute_backend; + state->params_backend = params_backend; + new_states.push_back(std::move(state)); + } + + for (auto& state : new_states) { + TensorState* registered_state = state.get(); + tensor_states_by_name_[registered_state->name] = registered_state; + tensor_states_.push_back(std::move(state)); + } + return true; +} + +bool ModelManager::validate_registered_tensors() { + bool ok = true; + for (const auto& state : tensor_states_) { + if (state == nullptr) { + ok = false; + continue; + } + bool state_ok = validate_tensor(*state); + if (state_ok) { + state->metadata_validated = true; + } + ok = state_ok && ok; + } + return ok; +} + +bool ModelManager::load_tensors_to_params_backend(const std::vector& states) { + std::vector need_load; + need_load.reserve(states.size()); + for (TensorState* state : states) { + if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) { + continue; + } + if (!state->metadata_validated) { + if (!validate_tensor(*state)) { + return false; + } + state->metadata_validated = true; + } + if (!state->loaded_to_params_backend) { + need_load.push_back(state); + } + } + if (need_load.empty()) { + return true; + } + + std::vector created_storage_blocks; + if (!mmap_params(need_load, created_storage_blocks)) { + for (ParamsStorageBlock* block : created_storage_blocks) { + if (block != nullptr) { + free_params_storage_block(*block); + erase_params_storage_block(block); + } + } + return false; + } + + std::vector need_alloc; + need_alloc.reserve(need_load.size()); + for (TensorState* state : need_load) { + if (state->tensor != nullptr && state->tensor->data == nullptr && state->tensor->view_src == nullptr) { + need_alloc.push_back(state); + } + } + + if (!alloc_params_buffers(need_alloc, created_storage_blocks) || + !load_tensors(need_load)) { + for (ParamsStorageBlock* block : created_storage_blocks) { + if (block != nullptr) { + free_params_storage_block(*block); + erase_params_storage_block(block); + } + } + return false; + } + for (ParamsStorageBlock* block : created_storage_blocks) { + if (block != nullptr && block->buffer != nullptr) { + LOG_DEBUG("model manager prepared params backend buffer (%6.2f MB, %zu tensors, %s)", + ggml_backend_buffer_get_size(block->buffer) / (1024.f * 1024.f), + block->states.size(), + ggml_backend_buffer_is_host(block->buffer) ? "RAM" : "VRAM"); + } + } + + return true; +} + +bool ModelManager::stage_tensors_to_compute_backend(const std::vector& states) { + std::map> states_by_compute_backend; + for (TensorState* state : states) { + if (state == nullptr || should_ignore(*state) || is_optional_missing_tensor(state->name)) { + continue; + } + if (state->compute_backend == nullptr) { + LOG_ERROR("model manager compute backend is null for tensor '%s'", state->name.c_str()); + return false; + } + if (state->params_backend == nullptr) { + LOG_ERROR("model manager params backend is null for tensor '%s'", state->name.c_str()); + return false; + } + if (state->compute_backend == state->params_backend || state->staged_to_compute_backend) { + continue; + } + if (!state->loaded_to_params_backend || state->tensor == nullptr || state->tensor->data == nullptr) { + LOG_ERROR("model manager tensor '%s' is not loaded to params backend", state->name.c_str()); + return false; + } + states_by_compute_backend[state->compute_backend].push_back(state); + } + + for (const auto& pair : states_by_compute_backend) { + ggml_backend_t compute_backend = pair.first; + const std::vector& states = pair.second; + if (states.empty()) { + continue; + } + + int64_t t0 = ggml_time_ms(); + + ggml_init_params init_params; + init_params.mem_size = std::max(1, states.size()) * ggml_tensor_overhead(); + init_params.mem_buffer = nullptr; + init_params.no_alloc = true; + + ggml_context* staging_ctx = ggml_init(init_params); + GGML_ASSERT(staging_ctx != nullptr); + + std::vector> staged_tensors; + staged_tensors.reserve(states.size()); + for (TensorState* state : states) { + ggml_tensor* staging_tensor = ggml_dup_tensor(staging_ctx, state->tensor); + ggml_set_name(staging_tensor, state->tensor->name); + staged_tensors.push_back({state, staging_tensor}); + } + + ggml_backend_buffer_t compute_buffer = ggml_backend_alloc_ctx_tensors(staging_ctx, compute_backend); + if (compute_buffer == nullptr) { + LOG_ERROR("model manager alloc compute params backend buffer failed, num_tensors = %zu", + staged_tensors.size()); + ggml_free(staging_ctx); + return false; + } + ggml_backend_buffer_set_usage(compute_buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + for (auto& staged_tensor : staged_tensors) { + TensorState* state = staged_tensor.first; + ggml_tensor* managed_tensor = state->tensor; + ggml_tensor* staging_tensor = staged_tensor.second; + ggml_backend_tensor_copy(managed_tensor, staging_tensor); + std::swap(managed_tensor->buffer, staging_tensor->buffer); + std::swap(managed_tensor->data, staging_tensor->data); + std::swap(managed_tensor->extra, staging_tensor->extra); + } + ggml_backend_synchronize(compute_backend); + + auto block = std::make_unique(); + block->compute_backend = compute_backend; + block->buffer = compute_buffer; + block->staging_ctx = staging_ctx; + block->staged_tensors = std::move(staged_tensors); + for (auto& staged_tensor : block->staged_tensors) { + TensorState* state = staged_tensor.first; + state->staged_to_compute_backend = true; + } + compute_staging_blocks_.push_back(std::move(block)); + + int64_t t1 = ggml_time_ms(); + LOG_DEBUG("model manager staged compute params (%6.2f MB, %zu tensors) to %s, taking %.2fs", + ggml_backend_buffer_get_size(compute_buffer) / (1024.f * 1024.f), + states.size(), + ggml_backend_name(compute_backend), + (t1 - t0) * 1.0f / 1000); + } + + return true; +} + +bool ModelManager::apply_loras_to_params(const std::vector& states) { + if (loras_.empty()) { + return true; + } + + struct LoraApplyGroup { + std::map model_tensors; + std::vector states; + }; + + std::map groups; + for (TensorState* state : states) { + if (state == nullptr || state->tensor == nullptr || + should_ignore(*state) || is_optional_missing_tensor(state->name)) { + continue; + } + if (state->applied_lora_epoch == current_lora_epoch_) { + continue; + } + if (state->compute_backend == nullptr) { + LOG_ERROR("model manager compute backend is null for lora target tensor '%s'", state->name.c_str()); + return false; + } + if (state->tensor->data == nullptr) { + LOG_ERROR("model manager lora target tensor '%s' is not prepared", state->name.c_str()); + return false; + } + LoraApplyGroup& group = groups[state->compute_backend]; + group.model_tensors[state->name] = state->tensor; + group.states.push_back(state); + } + + if (groups.empty()) { + return true; + } + + std::set all_tensor_names = tensor_names(); + for (auto& group_pair : groups) { + ggml_backend_t compute_backend = group_pair.first; + LoraApplyGroup& group = group_pair.second; + for (const LoraSpec& lora_spec : loras_) { + if (group.model_tensors.empty()) { + continue; + } + + std::string id = lora_id(lora_spec); + auto lora = std::make_shared(id, + compute_backend, + compute_backend, + lora_spec.path, + lora_spec.is_high_noise ? "model.high_noise_" : "", + lora_version_); + + LoraModel::filter_t lora_tensor_filter = nullptr; + if (!lora_spec.tensor_name_prefix_filter.empty()) { + lora_tensor_filter = [&](const std::string& tensor_name) { + return starts_with(tensor_name, lora_spec.tensor_name_prefix_filter); + }; + } + if (!lora->load_from_file(n_threads_, lora_tensor_filter)) { + LOG_WARN("load lora tensors from %s failed", lora_spec.path.c_str()); + if (lora_spec.required) { + return false; + } + continue; + } + if (lora->lora_tensors.empty()) { + if (lora_spec.required) { + LOG_ERROR("required lora has no tensors: %s", lora_spec.path.c_str()); + return false; + } + continue; + } + lora->multiplier = lora_spec.multiplier; + lora->apply(group.model_tensors, all_tensor_names, lora_version_, n_threads_, false); + lora->release_loaded_tensors(); + } + + for (TensorState* state : group.states) { + if (state != nullptr) { + state->applied_lora_epoch = current_lora_epoch_; + } + } + } + return true; +} + +void ModelManager::reset_lora_applied_params() { + release_compute_staging_blocks(true); + release_params_storage_blocks(true); + for (auto& state : tensor_states_) { + state->applied_lora_epoch = UINT64_MAX; + } +} + +bool ModelManager::should_ignore(const TensorState& state) const { + for (const auto& ignore_prefix : common_ignore_tensors_) { + if (starts_with(state.name, ignore_prefix)) { + return true; + } + } + return false; +} + +bool ModelManager::is_optional_missing_tensor(const std::string& name) const { + return name.find("cond_stage_model.transformer.text_model.encoder.layers.23") != std::string::npos || + name.find("alphas_cumprod") != std::string::npos; +} + +bool ModelManager::validate_tensor(const TensorState& state) const { + if (state.tensor == nullptr || should_ignore(state) || is_optional_missing_tensor(state.name)) { + return true; + } + + const auto& tensor_storage_map = model_loader_.get_tensor_storage_map(); + auto ts_it = tensor_storage_map.find(state.name); + if (ts_it == tensor_storage_map.end()) { + LOG_ERROR("%s tensor '%s' not in model metadata", state.desc.c_str(), state.name.c_str()); + return false; + } + + const TensorStorage& tensor_storage = ts_it->second; + if (state.tensor->ne[0] != tensor_storage.ne[0] || + state.tensor->ne[1] != tensor_storage.ne[1] || + state.tensor->ne[2] != tensor_storage.ne[2] || + state.tensor->ne[3] != tensor_storage.ne[3]) { + LOG_ERROR( + "%s tensor '%s' has wrong shape in model metadata: got [%d, %d, %d, %d], expected [%d, %d, %d, %d]", + state.desc.c_str(), + state.name.c_str(), + (int)tensor_storage.ne[0], (int)tensor_storage.ne[1], (int)tensor_storage.ne[2], (int)tensor_storage.ne[3], + (int)state.tensor->ne[0], (int)state.tensor->ne[1], (int)state.tensor->ne[2], (int)state.tensor->ne[3]); + return false; + } + return true; +} + +bool ModelManager::mmap_params(const std::vector& states, + std::vector& created_storage_blocks) { + std::map mmap_candidates; + std::map mmap_states; + for (TensorState* state : states) { + if (state == nullptr || !can_mmap_storage(*state) || state->tensor == nullptr || + state->tensor->data != nullptr || state->tensor->view_src != nullptr) { + continue; + } + mmap_candidates[state->name] = state->tensor; + mmap_states[state->name] = state; + } + if (mmap_candidates.empty()) { + return true; + } + + auto mmap_store = model_loader_.mmap_tensors(mmap_candidates, {}, true); + if (mmap_store.empty()) { + return true; + } + + auto block = std::make_unique(); + block->mmap_tensor_stores = std::move(mmap_store); + ParamsStorageBlock* raw = block.get(); + for (const auto& pair : mmap_states) { + TensorState* state = pair.second; + if (state != nullptr && state->tensor != nullptr && state->tensor->data != nullptr) { + block->states.push_back(state); + } + } + + if (!block->states.empty()) { + params_storage_blocks_.push_back(std::move(block)); + created_storage_blocks.push_back(raw); + } + return true; +} + +bool ModelManager::can_mmap_storage(const TensorState& state) const { + if (!enable_mmap_ || state.residency_mode != ResidencyMode::ParamBackend) { + return false; + } + if (state.compute_backend == nullptr || state.params_backend == nullptr) { + return false; + } + return sd_backend_is_cpu(state.compute_backend) || + sd_backend_is_cpu(state.params_backend) || + backend_supports_host_buffer(state.compute_backend); +} + +bool ModelManager::alloc_params_buffers(const std::vector& states, + std::vector& created_storage_blocks) { + std::map, std::vector> states_by_buffer_type; + for (TensorState* state : states) { + if (state == nullptr || state->tensor == nullptr) { + continue; + } + ggml_backend_buffer_type_t params_buft = params_buffer_type_for(*state); + if (params_buft == nullptr) { + return false; + } + states_by_buffer_type[{params_buft, static_cast(state->residency_mode)}].push_back(state); + } + + for (const auto& pair : states_by_buffer_type) { + ggml_backend_buffer_type_t params_buft = pair.first.first; + const std::vector& states = pair.second; + size_t alignment = ggml_backend_buft_get_alignment(params_buft); + size_t max_size = ggml_backend_buft_get_max_size(params_buft); + + auto alloc_chunk = [&](const std::vector& chunk, size_t chunk_size) -> bool { + if (chunk.empty() || chunk_size == 0) { + return true; + } + + ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(params_buft, chunk_size); + if (buffer == nullptr) { + LOG_ERROR("model manager alloc params backend buffer failed, size = %.2fMB", + chunk_size / (1024.0 * 1024.0)); + return false; + } + ggml_backend_buffer_set_usage(buffer, GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + std::vector initialized_tensors; + void* base = ggml_backend_buffer_get_base(buffer); + size_t offset = aligned_offset(base, 0, ggml_backend_buffer_get_alignment(buffer)); + for (TensorState* state : chunk) { + ggml_tensor* tensor = state->tensor; + size_t tensor_size = GGML_PAD(ggml_backend_buffer_get_alloc_size(buffer, tensor), + ggml_backend_buffer_get_alignment(buffer)); + enum ggml_status status = ggml_backend_tensor_alloc(buffer, tensor, static_cast(base) + offset); + if (status != GGML_STATUS_SUCCESS) { + LOG_ERROR("model manager failed to initialize params tensor '%s'", ggml_get_name(tensor)); + for (ggml_tensor* initialized : initialized_tensors) { + initialized->buffer = nullptr; + initialized->data = nullptr; + initialized->extra = nullptr; + } + LOG_DEBUG("model manager releasing params backend buffer (%6.2f MB, %zu tensors, %s)", + ggml_backend_buffer_get_size(buffer) / (1024.f * 1024.f), + initialized_tensors.size(), + ggml_backend_buffer_is_host(buffer) ? "RAM" : "VRAM"); + ggml_backend_buffer_free(buffer); + return false; + } + initialized_tensors.push_back(tensor); + offset += tensor_size; + } + + auto block = std::make_unique(); + block->buffer = buffer; + block->states = chunk; + ParamsStorageBlock* raw = block.get(); + params_storage_blocks_.push_back(std::move(block)); + created_storage_blocks.push_back(raw); + + return true; + }; + + std::vector chunk; + size_t chunk_size = 0; + for (TensorState* state : states) { + ggml_tensor* tensor = state->tensor; + size_t tensor_size = GGML_PAD(ggml_backend_buft_get_alloc_size(params_buft, tensor), alignment); + // Some backends, e.g. Vulkan, report a preferred chunk size here rather than a + // hard per-tensor allocation limit. Oversized tensors are allocated alone. + if (!chunk.empty() && max_size > 0 && chunk_size + tensor_size > max_size) { + if (!alloc_chunk(chunk, chunk_size)) { + return false; + } + chunk.clear(); + chunk_size = 0; + } + chunk.push_back(state); + chunk_size += tensor_size; + } + + if (!alloc_chunk(chunk, chunk_size)) { + return false; + } + } + + return true; +} + +bool ModelManager::load_tensors(const std::vector& states) { + std::map states_by_name; + std::set target_tensor_names; + for (TensorState* state : states) { + if (state == nullptr) { + continue; + } + states_by_name[state->name] = state; + target_tensor_names.insert(state->name); + } + if (states_by_name.empty()) { + return true; + } + + std::set loaded_names; + std::mutex loaded_names_mutex; + auto on_new_tensor_cb = [&](const TensorStorage& tensor_storage, ggml_tensor** dst_tensor) -> bool { + const std::string& name = tensor_storage.name; + *dst_tensor = nullptr; + + auto state_it = states_by_name.find(name); + if (state_it == states_by_name.end()) { + return true; + } + + TensorState* state = state_it->second; + if (state == nullptr || state->tensor == nullptr) { + LOG_ERROR("model manager tensor '%s' is null", name.c_str()); + return false; + } + + if (state->tensor->ne[0] != tensor_storage.ne[0] || + state->tensor->ne[1] != tensor_storage.ne[1] || + state->tensor->ne[2] != tensor_storage.ne[2] || + state->tensor->ne[3] != tensor_storage.ne[3]) { + LOG_ERROR( + "model manager tensor '%s' has wrong shape in model file: got [%d, %d, %d, %d], expected [%d, %d, %d, %d]", + name.c_str(), + (int)tensor_storage.ne[0], (int)tensor_storage.ne[1], (int)tensor_storage.ne[2], (int)tensor_storage.ne[3], + (int)state->tensor->ne[0], (int)state->tensor->ne[1], (int)state->tensor->ne[2], (int)state->tensor->ne[3]); + return false; + } + + { + std::lock_guard lock(loaded_names_mutex); + loaded_names.insert(name); + } + *dst_tensor = state->tensor; + return true; + }; + + if (!model_loader_.load_tensors(on_new_tensor_cb, enable_mmap_, &target_tensor_names)) { + LOG_ERROR("model manager load tensors failed"); + return false; + } + + bool missing = false; + for (const auto& pair : states_by_name) { + const std::string& name = pair.first; + if (loaded_names.find(name) == loaded_names.end()) { + LOG_ERROR("model manager tensor '%s' was not loaded", name.c_str()); + missing = true; + } + } + if (missing) { + return false; + } + + for (const auto& pair : states_by_name) { + pair.second->loaded_to_params_backend = true; + } + return true; +} + +ggml_backend_buffer_type_t ModelManager::params_buffer_type_for(const TensorState& state) const { + if (state.params_backend == nullptr) { + LOG_ERROR("model manager params backend is null for tensor '%s'", state.name.c_str()); + return nullptr; + } + ggml_backend_buffer_type_t params_buft = nullptr; + if (state.compute_backend != nullptr && state.params_backend != state.compute_backend) { + ggml_backend_dev_t compute_dev = ggml_backend_get_device(state.compute_backend); + if (compute_dev != nullptr) { + params_buft = ggml_backend_dev_host_buffer_type(compute_dev); + } + } + if (params_buft == nullptr) { + params_buft = ggml_backend_get_default_buffer_type(state.params_backend); + } + return params_buft; +} + +void ModelManager::free_compute_staging_block(ComputeStagingBlock& block) { + for (auto& staged_tensor : block.staged_tensors) { + TensorState* state = staged_tensor.first; + ggml_tensor* staging_tensor = staged_tensor.second; + if (state == nullptr || state->tensor == nullptr || staging_tensor == nullptr) { + continue; + } + ggml_tensor* managed_tensor = state->tensor; + managed_tensor->buffer = staging_tensor->buffer; + managed_tensor->data = staging_tensor->data; + managed_tensor->extra = staging_tensor->extra; + staging_tensor->buffer = nullptr; + staging_tensor->data = nullptr; + staging_tensor->extra = nullptr; + + state->staged_to_compute_backend = false; + state->applied_lora_epoch = UINT64_MAX; + } + + if (block.buffer != nullptr) { + LOG_DEBUG("model manager releasing compute params (%6.2f MB, %zu tensors) from %s", + ggml_backend_buffer_get_size(block.buffer) / (1024.f * 1024.f), + block.staged_tensors.size(), + block.compute_backend != nullptr ? ggml_backend_name(block.compute_backend) : "unknown"); + ggml_backend_buffer_free(block.buffer); + block.buffer = nullptr; + } + if (block.staging_ctx != nullptr) { + ggml_free(block.staging_ctx); + block.staging_ctx = nullptr; + } + block.staged_tensors.clear(); +} + +void ModelManager::release_compute_staging_blocks(bool force, + const std::unordered_set* target_states) { + for (auto it = compute_staging_blocks_.begin(); it != compute_staging_blocks_.end();) { + ComputeStagingBlock* block = it->get(); + bool can_release = force; + if (!can_release) { + can_release = std::all_of(block->staged_tensors.begin(), + block->staged_tensors.end(), + [target_states](const std::pair& pair) { + TensorState* state = pair.first; + if (state == nullptr) { + return true; + } + if (target_states != nullptr && + target_states->find(state) == target_states->end()) { + return false; + } + return state->active_prepare_count == 0; + }); + } + + if (can_release) { + free_compute_staging_block(*block); + it = compute_staging_blocks_.erase(it); + } else { + ++it; + } + } +} + +void ModelManager::free_params_storage_block(ParamsStorageBlock& block) { + if (block.buffer != nullptr) { + LOG_DEBUG("model manager releasing params backend buffer (%6.2f MB, %zu tensors, %s)", + ggml_backend_buffer_get_size(block.buffer) / (1024.f * 1024.f), + block.states.size(), + ggml_backend_buffer_is_host(block.buffer) ? "RAM" : "VRAM"); + ggml_backend_buffer_free(block.buffer); + block.buffer = nullptr; + } + block.mmap_tensor_stores.clear(); + + for (TensorState* state : block.states) { + if (state == nullptr || state->tensor == nullptr) { + continue; + } + state->tensor->buffer = nullptr; + state->tensor->data = nullptr; + state->tensor->extra = nullptr; + + state->loaded_to_params_backend = false; + state->applied_lora_epoch = UINT64_MAX; + } + block.states.clear(); +} + +void ModelManager::release_params_storage_blocks(bool force, + const std::unordered_set* target_states) { + for (auto it = params_storage_blocks_.begin(); it != params_storage_blocks_.end();) { + ParamsStorageBlock* block = it->get(); + bool can_release = force; + if (!can_release) { + can_release = std::all_of(block->states.begin(), + block->states.end(), + [target_states](TensorState* state) { + if (state == nullptr) { + return true; + } + if (target_states != nullptr && + target_states->find(state) == target_states->end()) { + return false; + } + return state->active_prepare_count == 0 && + !state->staged_to_compute_backend && + state->residency_mode == ResidencyMode::Disk; + }); + } + + if (can_release) { + free_params_storage_block(*block); + it = params_storage_blocks_.erase(it); + } else { + ++it; + } + } +} + +void ModelManager::erase_params_storage_block(ParamsStorageBlock* block) { + auto it = std::find_if(params_storage_blocks_.begin(), + params_storage_blocks_.end(), + [block](const std::unique_ptr& item) { + return item.get() == block; + }); + if (it != params_storage_blocks_.end()) { + params_storage_blocks_.erase(it); + } +} + +void ModelManager::release_all() { + for (auto& state : tensor_states_) { + state->active_prepare_count = 0; + state->applied_lora_epoch = UINT64_MAX; + } + release_compute_staging_blocks(true); + release_params_storage_blocks(true); +} + +bool ModelManager::resolve_required_tensor_states(const std::vector& tensors, + std::vector& required_states) const { + required_states.clear(); + std::unordered_set seen; + for (ggml_tensor* tensor : tensors) { + if (tensor == nullptr) { + continue; + } + const char* raw_name = ggml_get_name(tensor); + if (raw_name == nullptr || raw_name[0] == '\0') { + LOG_ERROR("model manager unnamed tensor is not registered"); + return false; + } + auto state_it = tensor_states_by_name_.find(raw_name); + if (state_it == tensor_states_by_name_.end()) { + LOG_ERROR("model manager tensor '%s' is not registered", raw_name); + return false; + } + TensorState* state = state_it->second; + if (state == nullptr) { + LOG_ERROR("model manager tensor '%s' has no tensor state", raw_name); + return false; + } + if (seen.insert(state).second) { + required_states.push_back(state); + } + } + return true; +} + +bool ModelManager::prepare_params(const std::vector& tensors) { + if (tensors.empty()) { + return true; + } + + std::vector required_states; + if (!resolve_required_tensor_states(tensors, required_states)) { + return false; + } + + if (!load_tensors_to_params_backend(required_states)) { + return false; + } + + if (!stage_tensors_to_compute_backend(required_states)) { + release_compute_staging_blocks(false); + release_params_storage_blocks(false); + return false; + } + + if (!apply_loras_to_params(required_states)) { + release_compute_staging_blocks(false); + release_params_storage_blocks(false); + return false; + } + + for (TensorState* state : required_states) { + if (state == nullptr) { + continue; + } + state->active_prepare_count++; + } + return true; +} + +void ModelManager::finish_compute_backend_usage(const std::vector& states) { + if (states.empty()) { + return; + } + + std::unordered_set target_states; + for (TensorState* state : states) { + if (state == nullptr || !target_states.insert(state).second) { + continue; + } + if (state->active_prepare_count > 0) { + state->active_prepare_count--; + } + } + release_compute_staging_blocks(false, &target_states); +} + +void ModelManager::release_compute_backend_params(const std::vector& tensors) { + if (tensors.empty()) { + return; + } + std::vector required_states; + if (!resolve_required_tensor_states(tensors, required_states)) { + return; + } + finish_compute_backend_usage(required_states); +} + +void ModelManager::release_params_backend_params(const std::vector& tensors) { + if (tensors.empty()) { + return; + } + std::vector required_states; + if (!resolve_required_tensor_states(tensors, required_states)) { + return; + } + if (required_states.empty()) { + return; + } + std::unordered_set target_states(required_states.begin(), required_states.end()); + release_params_storage_blocks(false, &target_states); +} diff --git a/otherarch/sdcpp/src/model_manager.h b/otherarch/sdcpp/src/model_manager.h new file mode 100644 index 000000000..1a414c15c --- /dev/null +++ b/otherarch/sdcpp/src/model_manager.h @@ -0,0 +1,167 @@ +#ifndef __MODEL_MANAGER_H__ +#define __MODEL_MANAGER_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "model_loader.h" +#include "weight_manager.h" + +class ModelManager : public RunnerWeightManager { +public: + enum class ResidencyMode { + Disk, + ParamBackend, + }; + + struct LoraSpec { + std::string path; + float multiplier = 1.0f; + bool is_high_noise = false; + std::string tensor_name_prefix_filter; + bool required = false; + }; + +private: + struct TensorState { + std::string name; + ggml_tensor* tensor = nullptr; + std::string desc; + + ResidencyMode residency_mode = ResidencyMode::ParamBackend; + ggml_backend_t compute_backend = nullptr; + ggml_backend_t params_backend = nullptr; + bool metadata_validated = false; + + int active_prepare_count = 0; + + bool loaded_to_params_backend = false; + bool staged_to_compute_backend = false; + uint64_t applied_lora_epoch = UINT64_MAX; + }; + + struct ParamsStorageBlock { + ggml_backend_buffer_t buffer = nullptr; + std::vector mmap_tensor_stores; + std::vector states; + }; + + struct ComputeStagingBlock { + ggml_backend_t compute_backend = nullptr; + ggml_backend_buffer_t buffer = nullptr; + ggml_context* staging_ctx = nullptr; + std::vector> staged_tensors; + }; + + ModelLoader model_loader_; + std::vector> tensor_states_; + std::map tensor_states_by_name_; + std::vector> params_storage_blocks_; + std::vector> compute_staging_blocks_; + std::set common_ignore_tensors_; + std::vector loras_; + SDVersion lora_version_ = VERSION_COUNT; + uint64_t current_lora_epoch_ = 0; + int n_threads_ = 0; + bool enable_mmap_ = false; + + void finish_compute_backend_usage(const std::vector& states); + void release_all(); + + bool resolve_required_tensor_states(const std::vector& tensors, + std::vector& required_states) const; + bool should_ignore(const TensorState& state) const; + bool is_optional_missing_tensor(const std::string& name) const; + bool validate_tensor(const TensorState& state) const; + + bool load_tensors_to_params_backend(const std::vector& states); + bool apply_loras_to_params(const std::vector& states); + bool mmap_params(const std::vector& states, + std::vector& created_storage_blocks); + bool can_mmap_storage(const TensorState& state) const; + bool alloc_params_buffers(const std::vector& states, + std::vector& created_storage_blocks); + bool load_tensors(const std::vector& states); + bool stage_tensors_to_compute_backend(const std::vector& states); + + ggml_backend_buffer_type_t params_buffer_type_for(const TensorState& state) const; + void release_compute_staging_blocks(bool force = false, + const std::unordered_set* target_states = nullptr); + void release_params_storage_blocks(bool force = false, + const std::unordered_set* target_states = nullptr); + void free_compute_staging_block(ComputeStagingBlock& block); + void free_params_storage_block(ParamsStorageBlock& block); + void erase_params_storage_block(ParamsStorageBlock* block); + void reset_lora_applied_params(); + +public: + ~ModelManager() override; + + ModelLoader& loader() { return model_loader_; } + const ModelLoader& loader() const { return model_loader_; } + + void set_n_threads(int n_threads) { + n_threads_ = n_threads; + model_loader_.set_n_threads(n_threads); + } + void set_enable_mmap(bool enable_mmap) { enable_mmap_ = enable_mmap; } + void set_common_ignore_tensors(std::set ignore_tensors); + void set_loras(std::vector loras, SDVersion version); + + std::set tensor_names() const; + + bool register_param_tensors(const std::string& desc, + std::map tensors, + ResidencyMode residency_mode, + ggml_backend_t compute_backend, + ggml_backend_t params_backend, + size_t* registered_tensor_size = nullptr); + + template + bool register_runner_params(const std::string& desc, + Runner& runner, + ResidencyMode residency_mode, + ggml_backend_t compute_backend, + ggml_backend_t params_backend, + size_t* registered_tensor_size = nullptr) { + std::map tensors; + runner.get_param_tensors(tensors); + return register_param_tensors(desc, + std::move(tensors), + residency_mode, + compute_backend, + params_backend, + registered_tensor_size); + } + + template + bool register_runner_params(const std::string& desc, + Runner& runner, + const std::string& prefix, + ResidencyMode residency_mode, + ggml_backend_t compute_backend, + ggml_backend_t params_backend, + size_t* registered_tensor_size = nullptr) { + std::map tensors; + runner.get_param_tensors(tensors, prefix); + return register_param_tensors(desc, + std::move(tensors), + residency_mode, + compute_backend, + params_backend, + registered_tensor_size); + } + + bool validate_registered_tensors(); + + bool prepare_params(const std::vector& tensors) override; + void release_compute_backend_params(const std::vector& tensors) override; + void release_params_backend_params(const std::vector& tensors) override; +}; + +#endif // __MODEL_MANAGER_H__ diff --git a/otherarch/sdcpp/src/name_conversion.cpp b/otherarch/sdcpp/src/name_conversion.cpp index e316f8c4b..4b7b4008d 100644 --- a/otherarch/sdcpp/src/name_conversion.cpp +++ b/otherarch/sdcpp/src/name_conversion.cpp @@ -990,7 +990,46 @@ bool is_first_stage_model_name(const std::string& name) { return false; } +static std::string convert_esrgan_tensor_name(std::string name) { + static std::unordered_map esrgan_name_map; + + if (esrgan_name_map.empty()) { + esrgan_name_map["model.0."] = "conv_first."; + + constexpr int max_num_blocks = 64; + for (int i = 0; i < max_num_blocks; i++) { + std::string block_prefix = "model.1.sub." + std::to_string(i) + "."; + for (int rdb = 1; rdb <= 3; rdb++) { + for (int conv = 1; conv <= 5; conv++) { + esrgan_name_map[block_prefix + "RDB" + std::to_string(rdb) + ".conv" + std::to_string(conv) + ".0."] = + "body." + std::to_string(i) + ".rdb" + std::to_string(rdb) + ".conv" + std::to_string(conv) + "."; + } + } + esrgan_name_map[block_prefix + "weight"] = "conv_body.weight"; + esrgan_name_map[block_prefix + "bias"] = "conv_body.bias"; + } + + // RealESRGAN stores only the learned layers in a Sequential. These indices + // cover the common x1, x2 and x4 layouts. + esrgan_name_map["model.2."] = "conv_hr."; + esrgan_name_map["model.3."] = "conv_up1."; + esrgan_name_map["model.4."] = "conv_last."; + esrgan_name_map["model.5."] = "conv_hr."; + esrgan_name_map["model.6."] = "conv_up2."; + esrgan_name_map["model.7."] = "conv_last."; + esrgan_name_map["model.8."] = "conv_hr."; + esrgan_name_map["model.10."] = "conv_last."; + } + + replace_with_prefix_map(name, esrgan_name_map); + return name; +} + std::string convert_tensor_name(std::string name, SDVersion version) { + if (version == VERSION_ESRGAN) { + return convert_esrgan_tensor_name(std::move(name)); + } + bool is_lora = false; bool is_lycoris_underline = false; bool is_underline = false; diff --git a/otherarch/sdcpp/src/runtime/guidance.cpp b/otherarch/sdcpp/src/runtime/guidance.cpp index a83680cd5..f925b4b8c 100644 --- a/otherarch/sdcpp/src/runtime/guidance.cpp +++ b/otherarch/sdcpp/src/runtime/guidance.cpp @@ -172,8 +172,8 @@ namespace sd::guidance { momentum_buffer_ = deltas; } - float diff_norm = 0.0f; - const int standard_res = 2 * 1024 / 8; // Use SDXL as the standard resolution (1024x1024, 8x8 patches, 4=2x2 channels) + float diff_norm = 0.0f; + const int standard_res = 2 * 1024 / 8; // Use SDXL as the standard resolution (1024x1024, 8x8 patches, 4=2x2 channels) if (params_.norm_threshold > 0.0f) { diff_norm = std::sqrt((deltas * deltas).sum()) * standard_res / std::sqrt(static_cast(deltas.numel())); } diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index 678787c08..68a4d6aae 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #include "core/ggml_extend.hpp" #include "core/ggml_graph_cut.h" @@ -12,6 +14,7 @@ #include "core/rng_philox.hpp" #include "core/util.h" #include "model_loader.h" +#include "model_manager.h" #include "stable-diffusion.h" #include "conditioning/conditioner.hpp" @@ -51,6 +54,8 @@ const char* sd_vae_format_name(enum sd_vae_format_t format); static SDVersion sd_vae_format_to_version(enum sd_vae_format_t format, SDVersion fallback); +#include + const char* model_version_to_str[] = { "SD 1.x", "SD 1.x Inpaint", @@ -89,6 +94,7 @@ const char* model_version_to_str[] = { "Longcat-Image", "PiD", "Ideogram 4", + "ESRGAN", }; const char* sampling_methods_str[] = { @@ -156,15 +162,15 @@ static float get_cache_reuse_threshold(const sd_cache_params_t& params) { /*=============================================== StableDiffusionGGML ================================================*/ +static_assert(std::atomic::is_always_lock_free, + "sd_cancel_mode_t must be lock-free"); + class StableDiffusionGGML { public: - std::vector mmap_tensor_store; SDBackendManager backend_manager; SDVersion version; - bool vae_decode_only = false; bool external_vae_is_invalid = false; - bool free_params_immediately = false; bool circular_x = false; bool circular_y = false; @@ -183,30 +189,26 @@ public: std::shared_ptr audio_vae_model; std::shared_ptr control_net; std::vector> generation_extensions; - std::vector> cond_stage_lora_models; - std::vector> diffusion_lora_models; - std::vector> first_stage_lora_models; + std::vector> runtime_lora_models; bool apply_lora_immediately = false; std::map> kcpp_lora_cache; bool kcpp_lora_cache_populate = false; std::string taesd_path; sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr}; - bool offload_params_to_cpu = false; - float max_vram = 0.f; - bool stream_layers = false; + bool enable_mmap = false; + sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment; + bool stream_layers = false; std::string backend_spec; std::string params_backend_spec; bool is_using_v_parameterization = false; bool is_using_edm_v_parameterization = false; - std::map tensors; - - // lora_name => multiplier - std::unordered_map curr_lora_state; + std::shared_ptr model_manager; std::shared_ptr denoiser = std::make_shared(); + std::vector file_alphas_cumprod; StableDiffusionGGML() = default; @@ -228,15 +230,22 @@ public: return module_backend; } - std::string toLowerCase(const std::string& str) { - std::string result; - std::locale loc; + std::atomic cancellation_flag = SD_CANCEL_RESET; - for (char ch : str) { - result += std::tolower(ch, loc); // Use locale-aware tolower - } + void set_cancel_flag(enum sd_cancel_mode_t flag) { + cancellation_flag.store(flag, std::memory_order_release); + } - return result; + void reset_cancel_flag() { + set_cancel_flag(SD_CANCEL_RESET); + } + + enum sd_cancel_mode_t get_cancel_flag() { + return cancellation_flag.load(std::memory_order_acquire); + } + + size_t max_graph_vram_bytes_for_module(SDBackendModule module) { + return max_vram_assignment.bytes_for_backend(backend_for(module)); } bool ensure_backend_pair(SDBackendModule module) { @@ -246,14 +255,31 @@ public: return params_backend_for(module) != nullptr; } - bool init_backend(const sd_ctx_params_t* sd_ctx_params) { + template + bool register_runner_params(const std::string& desc, + const std::shared_ptr& model, + SDBackendModule module, + size_t* params_mem_size = nullptr) { + if (model == nullptr) { + return true; + } + std::map group_tensors; + model->get_param_tensors(group_tensors); + if (model_manager == nullptr) { + return true; + } + return model_manager->register_param_tensors(desc, + std::move(group_tensors), + backend_manager.params_backend_is_disk(module) ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend, + backend_for(module), + params_backend_for(module), + params_mem_size); + } + + bool init_backend() { std::string error; - if (!backend_manager.init(sd_ctx_params->backend, - sd_ctx_params->params_backend, - offload_params_to_cpu, - sd_ctx_params->keep_clip_on_cpu, - sd_ctx_params->keep_vae_on_cpu, - sd_ctx_params->keep_control_net_on_cpu, + if (!backend_manager.init(backend_spec.c_str(), + params_backend_spec.c_str(), &error)) { LOG_ERROR("backend config failed: %s", error.c_str()); return false; @@ -271,30 +297,71 @@ public: } } + void refresh_compvis_denoiser_sigmas() { + auto comp_vis_denoiser = std::dynamic_pointer_cast(denoiser); + if (!comp_vis_denoiser) { + return; + } + std::vector alphas_cumprod(TIMESTEPS); + if (file_alphas_cumprod.size() == TIMESTEPS) { + alphas_cumprod = file_alphas_cumprod; + } else { + calculate_alphas_cumprod(alphas_cumprod.data()); + } + for (int i = 0; i < TIMESTEPS; i++) { + comp_vis_denoiser->sigmas[i] = std::sqrt((1 - alphas_cumprod[i]) / alphas_cumprod[i]); + comp_vis_denoiser->log_sigmas[i] = std::log(comp_vis_denoiser->sigmas[i]); + } + } + + void load_alphas_cumprod(ModelLoader& model_loader) { + file_alphas_cumprod.clear(); + + std::vector loaded_alphas; + if (!model_loader.load_float_tensor("alphas_cumprod", loaded_alphas, n_threads, enable_mmap)) { + return; + } + if (loaded_alphas.size() != TIMESTEPS) { + LOG_WARN("ignore alphas_cumprod from model file: expected %d values, got %zu", + TIMESTEPS, + loaded_alphas.size()); + return; + } + for (float alpha : loaded_alphas) { + if (!std::isfinite(alpha) || alpha <= 0.0f || alpha > 1.0f) { + LOG_WARN("ignore invalid alphas_cumprod from model file"); + return; + } + } + + file_alphas_cumprod = std::move(loaded_alphas); + LOG_DEBUG("loaded alphas_cumprod from model file"); + } + bool init(const sd_ctx_params_t* sd_ctx_params_kcpp) { // kcpp make sd_ctx_params mutable sd_ctx_params_t sd_ctx_params_local = *sd_ctx_params_kcpp; sd_ctx_params_t *sd_ctx_params = &sd_ctx_params_local; - n_threads = sd_ctx_params->n_threads; - vae_decode_only = sd_ctx_params->vae_decode_only; - free_params_immediately = sd_ctx_params->free_params_immediately; - offload_params_to_cpu = sd_ctx_params->offload_params_to_cpu; - max_vram = sd_ctx_params->max_vram; - stream_layers = sd_ctx_params->stream_layers; - backend_spec = SAFE_STR(sd_ctx_params->backend); - params_backend_spec = SAFE_STR(sd_ctx_params->params_backend); - if (stream_layers && max_vram == 0.f) { - LOG_WARN("--stream-layers has no effect without --max-vram set; ignoring"); - stream_layers = false; - } - if (stream_layers && !offload_params_to_cpu && params_backend_spec.empty()) { - // Streaming needs CPU-resident params. - LOG_WARN("--stream-layers has no effect without --offload-to-cpu (or --params-backend); ignoring"); - stream_layers = false; + n_threads = sd_ctx_params->n_threads; + enable_mmap = sd_ctx_params->enable_mmap; + stream_layers = sd_ctx_params->stream_layers; + backend_spec = SAFE_STR(sd_ctx_params->backend); + params_backend_spec = SAFE_STR(sd_ctx_params->params_backend); + max_vram_assignment.reset(0.f); + { + std::string error; + if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) { + LOG_ERROR("%s", error.c_str()); + return false; + } } - bool use_tae = false; - bool use_audio_vae = false; + std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers); + add_rpc_devices(rpc_servers_spec); + + bool use_tae = false; + bool use_audio_vae = false; + bool use_control_net = false; rng = get_rng(sd_ctx_params->rng_type); if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) { @@ -305,10 +372,20 @@ public: ggml_log_set(ggml_log_callback_default, nullptr); - if (!init_backend(sd_ctx_params)) { + if (!init_backend()) { return false; } - max_vram = sd::ggml_graph_cut::resolve_max_vram_gib(max_vram, backend_for(SDBackendModule::DIFFUSION)); + { + std::string error; + if (!max_vram_assignment.canonicalize_backend_keys(&error)) { + LOG_ERROR("%s", error.c_str()); + return false; + } + } + if (stream_layers && !backend_manager.params_backend_is_cpu(SDBackendModule::DIFFUSION)) { + LOG_WARN("--stream-layers has no effect unless diffusion params backend is cpu; ignoring"); + stream_layers = false; + } std::string clip_vision_fixed = SAFE_STR(sd_ctx_params->clip_vision_path); std::string clipg_path_fixed = SAFE_STR(sd_ctx_params->clip_g_path); @@ -320,7 +397,10 @@ public: std::string embed_connector_fixed = SAFE_STR(sd_ctx_params->embeddings_connectors_path); std::string vae_path_fixed = SAFE_STR(sd_ctx_params->vae_path); - ModelLoader model_loader; + model_manager = std::make_shared(); + model_manager->set_n_threads(n_threads); + model_manager->set_enable_mmap(enable_mmap); + ModelLoader& model_loader = model_manager->loader(); if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) { LOG_INFO("loading model from '%s'", sd_ctx_params->model_path); @@ -385,6 +465,15 @@ public: tempver = model_loader.get_sd_version(); } + auto toLowerCase = [](const std::string& str) -> std::string { + std::string result; + std::locale loc; + for (char ch : str) { + result += std::tolower(ch, loc); // Use locale-aware tolower + } + return result; + }; + bool iswan = sd_version_is_wan(tempver); bool is_wan21 = sd_version_is_wan(tempver) && tempver != VERSION_WAN2_2_TI2V; bool is_qwenimg = sd_version_is_qwen_image(tempver); @@ -589,6 +678,14 @@ public: } } + if (strlen(SAFE_STR(sd_ctx_params->pulid_weights_path)) > 0) { + LOG_INFO("loading PuLID weights from '%s'", sd_ctx_params->pulid_weights_path); + if (!model_loader.init_from_file(sd_ctx_params->pulid_weights_path, + "model.diffusion_model.")) { + LOG_WARN("loading PuLID weights from '%s' failed", sd_ctx_params->pulid_weights_path); + } + } + if (strlen(SAFE_STR(sd_ctx_params->llm_path)) > 0) { LOG_INFO("loading llm from '%s'", sd_ctx_params->llm_path); if (!model_loader.init_from_file(sd_ctx_params->llm_path, "text_encoders.llm.")) { @@ -636,6 +733,15 @@ public: } } + if (strlen(SAFE_STR(sd_ctx_params->control_net_path)) > 0) { + if (!model_loader.init_from_file(sd_ctx_params->control_net_path)) { + LOG_ERROR("init control net model loader from file failed: '%s'", sd_ctx_params->control_net_path); + return false; + } else { + use_control_net = true; + } + } + model_loader.convert_tensors_name(); version = model_loader.get_sd_version(); @@ -659,6 +765,7 @@ public: if (wtype != GGML_TYPE_COUNT || tensor_type_rules.size() > 0) { model_loader.set_wtype_override(wtype, tensor_type_rules); } + model_loader.process_model_files(enable_mmap, true); std::map wtype_stat = model_loader.get_wtype_stat(); std::map conditioner_wtype_stat = model_loader.get_conditioner_wtype_stat(); @@ -699,8 +806,8 @@ public: } } // Avoid full-model LoRA merge buffers on constrained setups. - const bool streaming_constrained = stream_layers || - sd_ctx_params->offload_params_to_cpu; + const bool params_offloaded = params_backend_for(SDBackendModule::DIFFUSION) != backend_for(SDBackendModule::DIFFUSION); + const bool streaming_constrained = stream_layers || params_offloaded; if (have_quantized_weight || streaming_constrained) { apply_lora_immediately = false; } else { @@ -712,51 +819,17 @@ public: apply_lora_immediately = false; } - std::map mmap_able_tensors; - bool enable_mmap_tensors = false; - bool needs_writable_mmap = false; - if (sd_ctx_params->enable_mmap) { - if (apply_lora_immediately) { - needs_writable_mmap = true; - LOG_WARN("in mode 'immediately', LoRAs will cause extra memory usage with mmap"); - } - enable_mmap_tensors = true; + if (enable_mmap && apply_lora_immediately) { + LOG_WARN("in mode 'immediately', LoRAs will cause extra memory usage with mmap"); } + load_alphas_cumprod(model_loader); - // split definition to avoid msvc choking on the extra parameter handling - auto module_can_mmap = [&](SDBackendModule module) { - return enable_mmap_tensors && - (backend_manager.runtime_backend_is_cpu(module) || - backend_manager.params_backend_is_cpu(module) || - backend_manager.runtime_backend_supports_host_buffer(module)); - }; + size_t text_encoder_params_mem_size = 0; + size_t unet_params_mem_size = 0; + size_t vae_params_mem_size = 0; + size_t control_net_params_mem_size = 0; + size_t extension_params_mem_size = 0; - auto get_param_tensors_p = [&](auto&& model, bool do_mmap, const char* prefix) { - std::map temp; - model->get_param_tensors(temp, prefix); - for (const auto& [key, tensor] : temp) { - tensors[key] = tensor; - if (do_mmap) { - mmap_able_tensors[key] = tensor; - } - } - }; - - auto get_param_tensors = [&](auto&& model, bool do_mmap) { - std::map temp; - model->get_param_tensors(temp); - for (const auto& [key, tensor] : temp) { - tensors[key] = tensor; - if (do_mmap) { - mmap_able_tensors[key] = tensor; - } - } - }; - - if (sd_version_is_control(version)) { - // Might need vae encode for control cond - vae_decode_only = false; - } bool tae_preview_only = sd_ctx_params->tae_preview_only; if (version == VERSION_SDXS_512_DS || version == VERSION_SDXS_09) { tae_preview_only = false; @@ -767,8 +840,6 @@ public: LOG_INFO("Using circular padding for convolutions"); } - const size_t max_graph_vram_bytes = sd::ggml_graph_cut::max_vram_gib_to_bytes(max_vram); - { if (!ensure_backend_pair(SDBackendModule::TE) || !ensure_backend_pair(SDBackendModule::DIFFUSION)) { @@ -777,33 +848,34 @@ public: if (sd_version_is_sd3(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), - tensor_storage_map); + tensor_storage_map, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); } else if (sd_version_is_pid(version)) { - vae_decode_only = false; cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), - tensor_storage_map, - version); - diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), - tensor_storage_map, - "model.diffusion_model.net"); - } else if (sd_version_is_ideogram4(version)) { - cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, version, "", - false); + false, + model_manager); + diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), + tensor_storage_map, + "model.diffusion_model.net", + model_manager); + } else if (sd_version_is_ideogram4(version)) { + cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), + tensor_storage_map, + version, + "", + false, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); } else if (sd_version_is_flux(version)) { bool is_chroma = false; for (auto pair : tensor_storage_map) { @@ -814,66 +886,71 @@ public: } if (is_chroma) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, sd_ctx_params->chroma_use_t5_mask, - sd_ctx_params->chroma_t5_mask_pad); + sd_ctx_params->chroma_t5_mask_pad, + false, + model_manager); } else if (version == VERSION_OVIS_IMAGE) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, version, "", - false); + false, + model_manager); } else { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), - tensor_storage_map); + tensor_storage_map, + model_manager); } diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", version, - sd_ctx_params->chroma_use_dit_mask); + sd_ctx_params->chroma_use_dit_mask, + model_manager); } else if (sd_version_is_flux2(version)) { bool is_chroma = false; cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, - version); + version, + "", + false, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", version, - sd_ctx_params->chroma_use_dit_mask); + sd_ctx_params->chroma_use_dit_mask, + model_manager); } else if (sd_version_is_ltxav(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), - tensor_storage_map); + tensor_storage_map, + "text_encoders.llm", + "text_embedding_projection", + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); } else if (sd_version_is_wan(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, true, 0, - true); + true, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", - version); + version, + model_manager); if (strlen(SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path)) > 0) { high_noise_diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.high_noise_diffusion_model", - version); + version, + model_manager); } if (diffusion_model->get_desc() == "Wan2.1-I2V-14B" || diffusion_model->get_desc() == "Wan2.1-FLF2V-14B" || @@ -882,150 +959,163 @@ public: return false; } clip_vision = std::make_shared(backend_for(SDBackendModule::CLIP_VISION), - params_backend_for(SDBackendModule::CLIP_VISION), - tensor_storage_map); - clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes); - get_param_tensors(clip_vision, module_can_mmap(SDBackendModule::CLIP_VISION)); + tensor_storage_map, + model_manager); + clip_vision->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::CLIP_VISION)); + if (!register_runner_params("CLIP vision", + clip_vision, + SDBackendModule::CLIP_VISION)) { + return false; + } } } else if (sd_version_is_qwen_image(version)) { - bool enable_vision = false; - if (!vae_decode_only) { - enable_vision = true; - } cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, version, "", - enable_vision); + true, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", version, - sd_ctx_params->qwen_image_zero_cond_t); + sd_ctx_params->qwen_image_zero_cond_t, + model_manager); } else if (sd_version_is_longcat(version)) { - bool enable_vision = false; - if (!vae_decode_only) { - enable_vision = true; - } cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, version, "", - enable_vision); + true, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", version, - sd_ctx_params->chroma_use_dit_mask); + sd_ctx_params->chroma_use_dit_mask, + model_manager); } else if (version == VERSION_HIDREAM_O1) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), - tensor_storage_map); + tensor_storage_map, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model"); + "model", + model_manager); } else if (sd_version_is_anima(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), - tensor_storage_map); + tensor_storage_map, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); } else if (sd_version_is_z_image(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, - version); + version, + "", + false, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", - version); + version, + model_manager); } else if (sd_version_is_ernie_image(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, - version); + version, + "", + false, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); } else if (sd_version_is_lens(version)) { cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, - version); + version, + "", + false, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, - "model.diffusion_model"); + "model.diffusion_model", + model_manager); } else { // SD1.x SD2.x SDXL std::map embbeding_map; for (uint32_t i = 0; i < sd_ctx_params->embedding_count; i++) { embbeding_map.emplace(SAFE_STR(sd_ctx_params->embeddings[i].name), SAFE_STR(sd_ctx_params->embeddings[i].path)); } cond_stage_model = std::make_shared(backend_for(SDBackendModule::TE), - params_backend_for(SDBackendModule::TE), tensor_storage_map, embbeding_map, - version); + version, + model_manager); diffusion_model = std::make_shared(backend_for(SDBackendModule::DIFFUSION), - params_backend_for(SDBackendModule::DIFFUSION), tensor_storage_map, "model.diffusion_model", - version); + version, + model_manager); if (sd_ctx_params->diffusion_conv_direct) { LOG_INFO("Using Conv2d direct in the diffusion model"); diffusion_model->set_conv2d_direct_enabled(true); } } - cond_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes); - get_param_tensors(cond_stage_model, module_can_mmap(SDBackendModule::TE)); + cond_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::TE)); + if (!register_runner_params("Conditioner model", + cond_stage_model, + SDBackendModule::TE, + &text_encoder_params_mem_size)) { + return false; + } - diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes); + diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::DIFFUSION)); diffusion_model->set_stream_layers_enabled(stream_layers); - get_param_tensors(diffusion_model, module_can_mmap(SDBackendModule::DIFFUSION)); - - if (sd_version_is_unet_edit(version)) { - vae_decode_only = false; + if (!register_runner_params("Diffusion model", + diffusion_model, + SDBackendModule::DIFFUSION, + &unet_params_mem_size)) { + return false; } if (high_noise_diffusion_model) { - high_noise_diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes); + high_noise_diffusion_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::DIFFUSION)); high_noise_diffusion_model->set_stream_layers_enabled(stream_layers); - get_param_tensors(high_noise_diffusion_model, module_can_mmap(SDBackendModule::DIFFUSION)); + if (!register_runner_params("High noise diffusion model", + high_noise_diffusion_model, + SDBackendModule::DIFFUSION, + &unet_params_mem_size)) { + return false; + } } if (!ensure_backend_pair(SDBackendModule::VAE)) { return false; } - auto create_tae = [&]() -> std::shared_ptr { + auto create_tae = [&](bool decode_only) -> std::shared_ptr { if (sd_version_is_wan(version) || sd_version_is_qwen_image(version) || sd_version_is_anima(version) || sd_version_is_ltxav(version)) { return std::make_shared(backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE), tensor_storage_map, "decoder", - vae_decode_only, - version); + decode_only, + version, + model_manager); } else { auto model = std::make_shared(backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE), tensor_storage_map, "decoder.layers", - vae_decode_only, - version); + decode_only, + version, + model_manager); return model; } }; @@ -1043,28 +1133,28 @@ public: auto create_vae = [&]() -> std::shared_ptr { if (sd_version_is_ltxav(version)) { return std::make_shared(backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE), tensor_storage_map, "first_stage_model", - vae_decode_only, - version); + false, + version, + model_manager); } else if (sd_version_is_wan(version) || sd_version_is_qwen_image(version) || sd_version_is_anima(version)) { return std::make_shared(backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE), tensor_storage_map, "first_stage_model", - vae_decode_only, - version); + false, + version, + model_manager); } else { auto model = std::make_shared(backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE), tensor_storage_map, "first_stage_model", - vae_decode_only, false, - vae_version); + false, + vae_version, + model_manager); if (sd_version_is_sdxl(version) && (strlen(SAFE_STR(sd_ctx_params->vae_path)) == 0 || sd_ctx_params->force_sdxl_vae_conv_scale || external_vae_is_invalid)) { float vae_conv_2d_scale = 1.f / 32.f; @@ -1078,36 +1168,61 @@ public: } }; - bool vae_mmap = module_can_mmap(SDBackendModule::VAE); - if (version == VERSION_CHROMA_RADIANCE || version == VERSION_HIDREAM_O1) { LOG_INFO("using FakeVAE"); first_stage_model = std::make_shared(version, backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE)); + model_manager); + if (!register_runner_params("VAE", + first_stage_model, + SDBackendModule::VAE, + &vae_params_mem_size)) { + return false; + } } else if (use_tae && !tae_preview_only) { LOG_INFO("using TAE for encoding / decoding"); - first_stage_model = create_tae(); - first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes); - get_param_tensors_p(first_stage_model, vae_mmap, "tae"); + first_stage_model = create_tae(false); + first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::VAE)); + if (!register_runner_params("VAE", + first_stage_model, + SDBackendModule::VAE, + &vae_params_mem_size)) { + return false; + } } else { LOG_INFO("using VAE for encoding / decoding"); first_stage_model = create_vae(); - first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes); - get_param_tensors_p(first_stage_model, vae_mmap, "first_stage_model"); + first_stage_model->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::VAE)); + if (!register_runner_params("VAE", + first_stage_model, + SDBackendModule::VAE, + &vae_params_mem_size)) { + return false; + } if (use_tae && tae_preview_only) { LOG_INFO("using TAE for preview"); - preview_vae = create_tae(); - preview_vae->set_max_graph_vram_bytes(max_graph_vram_bytes); - get_param_tensors_p(preview_vae, vae_mmap, "tae"); + preview_vae = create_tae(true); + preview_vae->set_max_graph_vram_bytes(max_graph_vram_bytes_for_module(SDBackendModule::VAE)); + if (!register_runner_params("preview VAE", + preview_vae, + SDBackendModule::VAE, + &vae_params_mem_size)) { + return false; + } } } if (use_audio_vae) { audio_vae_model = std::make_shared(backend_for(SDBackendModule::VAE), - params_backend_for(SDBackendModule::VAE), - tensor_storage_map); - get_param_tensors_p(audio_vae_model, vae_mmap, ""); + tensor_storage_map, + "", + model_manager); + if (!register_runner_params("LTX audio VAE", + audio_vae_model, + SDBackendModule::VAE, + &vae_params_mem_size)) { + return false; + } } if (sd_ctx_params->vae_conv_direct) { @@ -1118,18 +1233,26 @@ public: } } - if (strlen(SAFE_STR(sd_ctx_params->control_net_path)) > 0) { + if (use_control_net) { if (!ensure_backend_pair(SDBackendModule::CONTROL_NET)) { return false; } control_net = std::make_shared(backend_for(SDBackendModule::CONTROL_NET), params_backend_for(SDBackendModule::CONTROL_NET), - tensor_storage_map, - version); + model_loader.get_tensor_storage_map(), + version, + "", + model_manager); if (sd_ctx_params->diffusion_conv_direct) { LOG_INFO("Using Conv2d direct in the control net"); control_net->set_conv2d_direct_enabled(true); } + if (!register_runner_params("ControlNet", + control_net, + SDBackendModule::CONTROL_NET, + &control_net_params_mem_size)) { + return false; + } } { @@ -1140,6 +1263,7 @@ public: version, tensor_storage_map, model_loader, + model_manager, n_threads, [this](SDBackendModule module) { return ensure_backend_pair(module); }, [this](SDBackendModule module) { return backend_for(module); }, @@ -1151,15 +1275,21 @@ public: if (photomaker_extension->is_enabled()) { generation_extensions.push_back(photomaker_extension); } + + auto pulid_extension = create_pulid_extension(); + if (!pulid_extension->init(extension_ctx)) { + return false; + } + if (pulid_extension->is_enabled()) { + generation_extensions.push_back(pulid_extension); + } } - { - GenerationExtensionTensorContext extension_tensor_ctx{ - tensors, - mmap_able_tensors, - module_can_mmap, - }; - for (auto& extension : generation_extensions) { - extension->collect_param_tensors(extension_tensor_ctx); + for (auto& extension : generation_extensions) { + if (!register_runner_params(extension->name(), + extension, + SDBackendModule::PHOTOMAKER, + &extension_params_mem_size)) { + return false; } } @@ -1199,21 +1329,9 @@ public: circular_y = sd_ctx_params->circular_y; } - ggml_init_params params; - params.mem_size = static_cast(10 * 1024) * 1024; // 10M - params.mem_buffer = nullptr; - params.no_alloc = false; - // LOG_DEBUG("mem_size %u ", params.mem_size); - ggml_context* ctx = ggml_init(params); // for alphas_cumprod and is_using_v_parameterization check - GGML_ASSERT(ctx != nullptr); - ggml_tensor* alphas_cumprod_tensor = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, TIMESTEPS); - calculate_alphas_cumprod((float*)alphas_cumprod_tensor->data); - - // load weights - LOG_DEBUG("loading weights"); + LOG_DEBUG("validating model metadata"); std::set ignore_tensors; - tensors["alphas_cumprod"] = alphas_cumprod_tensor; if (use_tae && !tae_preview_only) { ignore_tensors.insert("first_stage_model."); } @@ -1224,13 +1342,6 @@ public: ignore_tensors.insert("model.diffusion_model.__32x32__"); ignore_tensors.insert("model.diffusion_model.__index_timestep_zero__"); - if (vae_decode_only) { - ignore_tensors.insert("first_stage_model.encoder"); - ignore_tensors.insert("first_stage_model.conv1"); - ignore_tensors.insert("first_stage_model.quant"); - ignore_tensors.insert("tae.encoder"); - ignore_tensors.insert("text_encoders.llm.visual."); - } if (audio_vae_model) { ignore_tensors.insert("audio_vae.encoder"); } @@ -1262,93 +1373,15 @@ public: ignore_tensors.insert("model.visual.deepstack_merger_list."); } - if (enable_mmap_tensors) { - if (mmap_able_tensors.empty()) { - LOG_DEBUG("no tensors could be memory-mapped"); - } else { - mmap_tensor_store = model_loader.mmap_tensors(mmap_able_tensors, ignore_tensors, needs_writable_mmap); - } - } - - if (clip_vision && !clip_vision->alloc_params_buffer()) { - LOG_ERROR("CLIP vision params buffer allocation failed"); - ggml_free(ctx); - return false; - } - if (cond_stage_model && !cond_stage_model->alloc_params_buffer()) { - LOG_ERROR("Conditioner model params buffer allocation failed"); - ggml_free(ctx); - return false; - } - if (diffusion_model && !diffusion_model->alloc_params_buffer()) { - LOG_ERROR("Diffusion model params buffer allocation failed"); - ggml_free(ctx); - return false; - } - if (high_noise_diffusion_model && !high_noise_diffusion_model->alloc_params_buffer()) { - LOG_ERROR("High noise diffusion model params buffer allocation failed"); - ggml_free(ctx); - return false; - } - if (first_stage_model && !first_stage_model->alloc_params_buffer()) { - LOG_ERROR("VAE params buffer allocation failed"); - ggml_free(ctx); - return false; - } - if (preview_vae && !preview_vae->alloc_params_buffer()) { - LOG_ERROR("preview VAE params buffer allocation failed"); - ggml_free(ctx); - return false; - } - if (audio_vae_model && !audio_vae_model->alloc_params_buffer()) { - LOG_ERROR("LTX audio VAE params buffer allocation failed"); - ggml_free(ctx); - return false; - } - for (auto& extension : generation_extensions) { - if (!extension->alloc_params_buffer()) { - LOG_ERROR("%s params buffer allocation failed", extension->name()); - ggml_free(ctx); - return false; - } - } - - bool success = model_loader.load_tensors(tensors, ignore_tensors, n_threads, sd_ctx_params->enable_mmap); - if (!success) { - LOG_ERROR("load tensors from model loader failed"); - ggml_free(ctx); + model_manager->set_common_ignore_tensors(ignore_tensors); + if (!model_manager->validate_registered_tensors()) { + LOG_ERROR("model metadata validation failed"); return false; } - LOG_DEBUG("finished loaded file"); + LOG_DEBUG("model metadata validated; weights will be prepared lazily"); { - size_t clip_params_mem_size = cond_stage_model->get_params_buffer_size(); - size_t unet_params_mem_size = diffusion_model->get_params_buffer_size(); - if (high_noise_diffusion_model) { - unet_params_mem_size += high_noise_diffusion_model->get_params_buffer_size(); - } - size_t vae_params_mem_size = 0; - vae_params_mem_size = first_stage_model->get_params_buffer_size(); - if (preview_vae) { - vae_params_mem_size += preview_vae->get_params_buffer_size(); - } - if (audio_vae_model) { - vae_params_mem_size += audio_vae_model->get_params_buffer_size(); - } - size_t control_net_params_mem_size = 0; - if (control_net) { - if (!control_net->load_from_file(SAFE_STR(sd_ctx_params->control_net_path), n_threads)) { - ggml_free(ctx); - return false; - } - control_net_params_mem_size = control_net->get_params_buffer_size(); - } - size_t extension_params_mem_size = 0; - for (auto& extension : generation_extensions) { - extension_params_mem_size += extension->get_params_buffer_size(); - } - size_t total_params_ram_size = 0; size_t total_params_vram_size = 0; auto add_params_memory = [&](size_t size, SDBackendModule module) { @@ -1377,12 +1410,11 @@ public: return sd_backend_is_cpu(module_backend) ? "RAM" : "VRAM"; }; - if (!add_params_memory(clip_params_mem_size, SDBackendModule::TE) || + if (!add_params_memory(text_encoder_params_mem_size, SDBackendModule::TE) || !add_params_memory(extension_params_mem_size, SDBackendModule::PHOTOMAKER) || !add_params_memory(unet_params_mem_size, SDBackendModule::DIFFUSION) || !add_params_memory(vae_params_mem_size, SDBackendModule::VAE) || !add_params_memory(control_net_params_mem_size, SDBackendModule::CONTROL_NET)) { - ggml_free(ctx); return false; } @@ -1393,8 +1425,8 @@ public: total_params_size / 1024.0 / 1024.0, total_params_vram_size / 1024.0 / 1024.0, total_params_ram_size / 1024.0 / 1024.0, - clip_params_mem_size / 1024.0 / 1024.0, - params_memory_location(clip_params_mem_size, SDBackendModule::TE), + text_encoder_params_mem_size / 1024.0 / 1024.0, + params_memory_location(text_encoder_params_mem_size, SDBackendModule::TE), unet_params_mem_size / 1024.0 / 1024.0, params_memory_location(unet_params_mem_size, SDBackendModule::DIFFUSION), vae_params_mem_size / 1024.0 / 1024.0, @@ -1411,12 +1443,7 @@ public: if (pred_type == PREDICTION_COUNT) { if (sd_version_is_sd2(version)) { - // check is_using_v_parameterization_for_sd2 - if (is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version))) { - pred_type = V_PRED; - } else { - pred_type = EPS_PRED; - } + pred_type = is_using_v_parameterization_for_sd2(sd_version_is_inpaint(version)) ? V_PRED : EPS_PRED; } else if (sd_version_is_sdxl(version)) { if (tensor_storage_map.find("edm_vpred.sigma_max") != tensor_storage_map.end()) { // CosXL models @@ -1509,25 +1536,27 @@ public: } default: { LOG_ERROR("Unknown predition type %i", pred_type); - ggml_free(ctx); return false; } } - auto comp_vis_denoiser = std::dynamic_pointer_cast(denoiser); - if (comp_vis_denoiser) { - for (int i = 0; i < TIMESTEPS; i++) { - comp_vis_denoiser->sigmas[i] = std::sqrt((1 - ((float*)alphas_cumprod_tensor->data)[i]) / ((float*)alphas_cumprod_tensor->data)[i]); - comp_vis_denoiser->log_sigmas[i] = std::log(comp_vis_denoiser->sigmas[i]); - } - } + refresh_compvis_denoiser_sigmas(); } - ggml_free(ctx); return true; } bool is_using_v_parameterization_for_sd2(bool is_inpaint = false) { + struct RunnerDoneOnExit { + GGMLRunner* runner = nullptr; + ~RunnerDoneOnExit() { + if (runner != nullptr) { + runner->runner_done(); + } + } + }; + RunnerDoneOnExit diffusion_runner_done{diffusion_model.get()}; + sd::Tensor x_t = sd::full({8, 8, 4, 1}, 0.5f); sd::Tensor c = sd::full({1024, 2, 1, 1}, 0.5f); sd::Tensor steps = sd::full({1}, 999.0f); @@ -1549,7 +1578,6 @@ public: auto out_opt = diffusion_model->compute(n_threads, diffusion_params); GGML_ASSERT(!out_opt.empty()); out = std::move(out_opt); - diffusion_model->free_compute_buffer(); double result = static_cast((out - x_t).mean()); int64_t t1 = ggml_time_ms(); @@ -1557,42 +1585,46 @@ public: return result < -1; } - std::shared_ptr load_lora_model_from_file(const std::string& lora_id, - float multiplier, - SDBackendModule module, - LoraModel::filter_t lora_tensor_filter = nullptr) { + static std::string lora_log_id(const ModelManager::LoraSpec& lora) { + return lora.is_high_noise ? "|high_noise|" + lora.path : lora.path; + } + + std::shared_ptr load_lora_model(const ModelManager::LoraSpec& lora_spec, + SDBackendModule module, + LoraModel::filter_t module_filter = nullptr) { + if (!ensure_backend_pair(module)) { + return nullptr; + } + if (lora_spec.is_high_noise) { + LOG_DEBUG("high noise lora: %s", lora_spec.path.c_str()); + } // kcpp // first check the cache - std::string lora_key = "|" + std::to_string(static_cast(module)) + "|" + lora_id; + std::string lora_key = "|" + std::to_string(static_cast(module)) + "|" + lora_log_id(lora_spec); if (!apply_lora_immediately) { auto it = kcpp_lora_cache.find(lora_key); if (it != kcpp_lora_cache.end()) { if (it->second) { - it->second->multiplier = multiplier; + it->second->multiplier = lora_spec.multiplier; } return it->second; } } - std::string lora_path = lora_id; - static std::string high_noise_tag = "|high_noise|"; - bool is_high_noise = false; - if (starts_with(lora_path, high_noise_tag)) { - lora_path = lora_path.substr(high_noise_tag.size()); - is_high_noise = true; - LOG_DEBUG("high noise lora: %s", lora_path.c_str()); - } - if (!ensure_backend_pair(module)) { - return nullptr; - } - auto lora = std::make_shared(lora_id, + auto lora = std::make_shared(lora_log_id(lora_spec), backend_for(module), backend_for(module), - lora_path, - is_high_noise ? "model.high_noise_" : "", + lora_spec.path, + lora_spec.is_high_noise ? "model.high_noise_" : "", version); + LoraModel::filter_t lora_tensor_filter = module_filter; + if (!lora_spec.tensor_name_prefix_filter.empty()) { + lora_tensor_filter = [module_filter, prefix = lora_spec.tensor_name_prefix_filter](const std::string& tensor_name) { + return starts_with(tensor_name, prefix) && (!module_filter || module_filter(tensor_name)); + }; + } if (!lora->load_from_file(n_threads, lora_tensor_filter)) { - LOG_WARN("load lora tensors from %s failed", lora_path.c_str()); + LOG_WARN("load lora tensors from %s failed", lora_spec.path.c_str()); // also cache negatives to avoid I/O at runtime lora = nullptr; if (!apply_lora_immediately && kcpp_lora_cache_populate) @@ -1600,60 +1632,13 @@ public: return lora; } - lora->multiplier = multiplier; + lora->multiplier = lora_spec.multiplier; if (!apply_lora_immediately && kcpp_lora_cache_populate) kcpp_lora_cache[lora_key] = lora; return lora; } - void apply_loras_immediately(const std::unordered_map& lora_state) { - std::unordered_map lora_state_diff; - for (auto& kv : lora_state) { - const std::string& lora_name = kv.first; - float multiplier = kv.second; - lora_state_diff[lora_name] += multiplier; - } - for (auto& kv : curr_lora_state) { - const std::string& lora_name = kv.first; - float curr_multiplier = kv.second; - lora_state_diff[lora_name] -= curr_multiplier; - } - - if (lora_state_diff.empty()) { - return; - } - - LOG_INFO("apply lora immediately"); - - size_t rm = lora_state_diff.size() - lora_state.size(); - if (rm != 0) { - LOG_INFO("attempting to apply %lu LoRAs (removing %lu applied LoRAs)", lora_state.size(), rm); - } else { - LOG_INFO("attempting to apply %lu LoRAs", lora_state.size()); - } - - for (auto& kv : lora_state_diff) { - int64_t t0 = ggml_time_ms(); - - auto lora = load_lora_model_from_file(kv.first, kv.second, SDBackendModule::DIFFUSION); - if (!lora || lora->lora_tensors.empty()) { - continue; - } - lora->apply(tensors, version, n_threads); - lora->free_params_buffer(); - - int64_t t1 = ggml_time_ms(); - - LOG_INFO("lora '%s' applied, taking %.2fs", kv.first.c_str(), (t1 - t0) * 1.0f / 1000); - } - - curr_lora_state = lora_state; - } - - void apply_loras_at_runtime(const std::unordered_map& lora_state) { - cond_stage_lora_models.clear(); - diffusion_lora_models.clear(); - first_stage_lora_models.clear(); + void clear_lora_adapters() { if (cond_stage_model) { cond_stage_model->set_weight_adapter(nullptr); } @@ -1666,39 +1651,74 @@ public: if (first_stage_model) { first_stage_model->set_weight_adapter(nullptr); } - if (lora_state.empty()) { + } + + std::vector> load_runtime_loras_for_module(const std::vector& loras, + const std::set& model_tensor_names, + SDBackendModule module, + LoraModel::filter_t module_filter = nullptr) { + std::vector> module_lora_models; + for (const auto& lora_spec : loras) { + auto lora = load_lora_model(lora_spec, module, module_filter); + if (lora == nullptr) { + if (lora_spec.required) { + LOG_ERROR("required lora load failed: %s", lora_spec.path.c_str()); + } + continue; + } + if (lora->lora_tensors.empty()) { + continue; + } + + lora->preprocess_lora_tensors(model_tensor_names); + runtime_lora_models.push_back(lora); + module_lora_models.push_back(std::move(lora)); + } + return module_lora_models; + } + + void apply_loras_immediately(const std::vector& loras) { + if (model_manager == nullptr) { + if (!loras.empty()) { + LOG_WARN("model manager is not available for immediate lora"); + } return; } + + clear_lora_adapters(); + runtime_lora_models.clear(); + + model_manager->set_loras(loras, version); + } + + void apply_loras_at_runtime(const std::vector& loras) { + if (model_manager != nullptr) { + model_manager->set_loras({}, version); + } + runtime_lora_models.clear(); + clear_lora_adapters(); + if (loras.empty()) { + return; + } + + std::set model_tensor_names; + if (model_manager != nullptr) { + model_tensor_names = model_manager->tensor_names(); + } + LOG_INFO("apply lora at runtime"); if (cond_stage_model) { - std::vector> lora_models; - auto lora_state_diff = lora_state; - for (auto& lora_model : cond_stage_lora_models) { - auto iter = lora_state_diff.find(lora_model->lora_id); - - if (iter != lora_state_diff.end()) { - lora_model->multiplier = iter->second; - lora_models.push_back(lora_model); - lora_state_diff.erase(iter); - } - } - cond_stage_lora_models = lora_models; auto lora_tensor_filter = [&](const std::string& tensor_name) { if (is_cond_stage_model_name(tensor_name)) { return true; } return false; }; - for (auto& kv : lora_state_diff) { - const std::string& lora_id = kv.first; - float multiplier = kv.second; - - auto lora = load_lora_model_from_file(lora_id, multiplier, SDBackendModule::TE, lora_tensor_filter); - if (lora && !lora->lora_tensors.empty()) { - lora->preprocess_lora_tensors(tensors); - cond_stage_lora_models.push_back(lora); - } - } + auto cond_stage_lora_models = + load_runtime_loras_for_module(loras, + model_tensor_names, + SDBackendModule::TE, + lora_tensor_filter); // Only attach the adapter when there are LoRAs targeting the cond_stage model. // An empty MultiLoraAdapter still routes every linear/conv through // forward_with_lora() instead of the direct kernel path — slower for no benefit. @@ -1708,34 +1728,17 @@ public: } } if (diffusion_model) { - std::vector> lora_models; - auto lora_state_diff = lora_state; - for (auto& lora_model : diffusion_lora_models) { - auto iter = lora_state_diff.find(lora_model->lora_id); - - if (iter != lora_state_diff.end()) { - lora_model->multiplier = iter->second; - lora_models.push_back(lora_model); - lora_state_diff.erase(iter); - } - } - diffusion_lora_models = lora_models; auto lora_tensor_filter = [&](const std::string& tensor_name) { if (is_diffusion_model_name(tensor_name)) { return true; } return false; }; - for (auto& kv : lora_state_diff) { - const std::string& lora_name = kv.first; - float multiplier = kv.second; - - auto lora = load_lora_model_from_file(lora_name, multiplier, SDBackendModule::DIFFUSION, lora_tensor_filter); - if (lora && !lora->lora_tensors.empty()) { - lora->preprocess_lora_tensors(tensors); - diffusion_lora_models.push_back(lora); - } - } + auto diffusion_lora_models = + load_runtime_loras_for_module(loras, + model_tensor_names, + SDBackendModule::DIFFUSION, + lora_tensor_filter); if (!diffusion_lora_models.empty()) { auto multi_lora_adapter = std::make_shared(diffusion_lora_models); diffusion_model->set_weight_adapter(multi_lora_adapter); @@ -1746,34 +1749,17 @@ public: } if (first_stage_model) { - std::vector> lora_models; - auto lora_state_diff = lora_state; - for (auto& lora_model : first_stage_lora_models) { - auto iter = lora_state_diff.find(lora_model->lora_id); - - if (iter != lora_state_diff.end()) { - lora_model->multiplier = iter->second; - lora_models.push_back(lora_model); - lora_state_diff.erase(iter); - } - } - first_stage_lora_models = lora_models; auto lora_tensor_filter = [&](const std::string& tensor_name) { if (is_first_stage_model_name(tensor_name)) { return true; } return false; }; - for (auto& kv : lora_state_diff) { - const std::string& lora_name = kv.first; - float multiplier = kv.second; - - auto lora = load_lora_model_from_file(lora_name, multiplier, SDBackendModule::VAE, lora_tensor_filter); - if (lora && !lora->lora_tensors.empty()) { - lora->preprocess_lora_tensors(tensors); - first_stage_lora_models.push_back(lora); - } - } + auto first_stage_lora_models = + load_runtime_loras_for_module(loras, + model_tensor_names, + SDBackendModule::VAE, + lora_tensor_filter); if (!first_stage_lora_models.empty()) { auto multi_lora_adapter = std::make_shared(first_stage_lora_models); first_stage_model->set_weight_adapter(multi_lora_adapter); @@ -1782,46 +1768,42 @@ public: } void lora_stat() { - if (!cond_stage_lora_models.empty()) { - LOG_INFO("cond_stage_lora_models:"); - for (auto& lora_model : cond_stage_lora_models) { - lora_model->stat(); - } - } - - if (!diffusion_lora_models.empty()) { - LOG_INFO("diffusion_lora_models:"); - for (auto& lora_model : diffusion_lora_models) { - lora_model->stat(); - } - } - - if (!first_stage_lora_models.empty()) { - LOG_INFO("first_stage_lora_models:"); - for (auto& lora_model : first_stage_lora_models) { + if (!runtime_lora_models.empty()) { + LOG_INFO("runtime_lora_models:"); + for (auto& lora_model : runtime_lora_models) { lora_model->stat(); } } } void apply_loras(const sd_lora_t* loras, uint32_t lora_count) { - std::unordered_map lora_f2m; + std::vector all_loras; + all_loras.reserve(lora_count); for (uint32_t i = 0; i < lora_count; i++) { std::string lora_id = SAFE_STR(loras[i].path); + ModelManager::LoraSpec lora_spec; + lora_spec.path = lora_id; + lora_spec.multiplier = loras[i].multiplier; + lora_spec.is_high_noise = loras[i].is_high_noise; + all_loras.push_back(std::move(lora_spec)); if (loras[i].is_high_noise) { lora_id = "|high_noise|" + lora_id; } - lora_f2m[lora_id] = loras[i].multiplier; LOG_DEBUG("lora %s:%.2f", lora_id.c_str(), loras[i].multiplier); } + + for (auto& extension : generation_extensions) { + extension->collect_loras(all_loras); + } + int64_t t0 = ggml_time_ms(); if (apply_lora_immediately) { - apply_loras_immediately(lora_f2m); + apply_loras_immediately(all_loras); } else { - apply_loras_at_runtime(lora_f2m); + apply_loras_at_runtime(all_loras); } int64_t t1 = ggml_time_ms(); - if (!lora_f2m.empty()) { + if (!all_loras.empty()) { LOG_INFO("apply_loras completed, taking %.2fs", (t1 - t0) * 1.0f / 1000); } } @@ -1833,6 +1815,7 @@ public: } void prepare_generation_extensions(const sd_pm_params_t& pm_params, + const sd_pulid_params_t& pulid_params, ConditionerParams& condition_params, int total_steps) { reset_generation_extensions(); @@ -1840,11 +1823,9 @@ public: cond_stage_model.get(), condition_params, pm_params, - tensors, - version, + pulid_params, n_threads, total_steps, - free_params_immediately, }; for (auto& extension : generation_extensions) { @@ -2108,11 +2089,17 @@ public: sd_get_preview_mode()}; } - void report_sample_progress(int step, size_t total_steps, int64_t t0) { - int64_t t1 = ggml_time_us(); + void report_sample_progress(int step, size_t total_steps, int64_t* last_progress_us) { if (step > 0 || step == -(int)total_steps) { - int showstep = std::abs(step); - pretty_progress(showstep, (int)total_steps, (t1 - t0) / 1000000.f / showstep); + int64_t now = ggml_time_us(); + int showstep = std::abs(step); + float step_seconds = last_progress_us != nullptr && *last_progress_us > 0 + ? (now - *last_progress_us) / 1000000.f + : 0.f; + pretty_progress(showstep, (int)total_steps, step_seconds); + if (last_progress_us != nullptr) { + *last_progress_us = now; + } } } @@ -2166,6 +2153,18 @@ public: float frame_rate, const sd_cache_params_t* cache_params, const sd::Tensor& video_positions = {}) { + struct RunnerDoneOnExit { + GGMLRunner* runner = nullptr; + ~RunnerDoneOnExit() { + if (runner != nullptr) { + runner->runner_done(); + } + } + }; + RunnerDoneOnExit sample_diffusion_runner_done{work_diffusion_model.get()}; + + RunnerDoneOnExit sample_control_runner_done{!control_image.empty() && control_net != nullptr ? control_net.get() : nullptr}; + std::vector skip_layers(guidance.slg.layers, guidance.slg.layers + guidance.slg.layer_count); float cfg_scale = guidance.txt_cfg; float img_cfg_scale = guidance.img_cfg; @@ -2211,7 +2210,7 @@ public: noise *= eta; } - int64_t t0 = ggml_time_us(); + int64_t last_progress_us = ggml_time_us(); sd::Tensor x_t = !noise.empty() ? denoiser->noise_scaling(sigmas[0], noise, init_latent) : init_latent; @@ -2219,8 +2218,14 @@ public: SamplePreviewContext preview = prepare_sample_preview_context(); auto denoise = [&](const sd::Tensor& x, float sigma, int step) -> sd::guidance::GuiderOutput { + if (get_cancel_flag() == SD_CANCEL_ALL) { + LOG_DEBUG("cancelling generation"); + return {}; + } + if (step == 1 || step == -1) { pretty_progress(0, (int)steps, 0); + last_progress_us = ggml_time_us(); } std::vector scaling = denoiser->get_scalings(sigma); @@ -2258,7 +2263,7 @@ public: if (sd_should_preview_denoised() && preview.callback != nullptr) { preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false); } - report_sample_progress(step, steps, t0); + report_sample_progress(step, steps, &last_progress_us); sd::guidance::GuiderOutput output; output.pred = denoised; return output; @@ -2338,6 +2343,10 @@ public: return std::move(cached_output); } + for (const auto& extension : generation_extensions) { + extension->before_diffusion(diffusion_params, step); + } + auto output_opt = work_diffusion_model->compute(n_threads, diffusion_params); if (output_opt.empty()) { LOG_ERROR("diffusion model compute failed"); @@ -2441,7 +2450,7 @@ public: if (sd_should_preview_denoised() && preview.callback != nullptr) { preview_image(step, denoised, version, preview.mode, preview.callback, preview.data, false); } - report_sample_progress(step, steps, t0); + report_sample_progress(step, steps, &last_progress_us); output.pred = denoised; return output; }; @@ -2587,15 +2596,6 @@ public: if (sd_version_is_pid(version)) { return sd::ops::clamp((x + 1.f) * 0.5f, 0.0f, 1.0f); } - // Free resident diffusion params before VAE allocates its compute buffer. - if (stream_layers) { - if (diffusion_model) { - diffusion_model->release_streaming_residency(); - } - if (high_noise_diffusion_model) { - high_noise_diffusion_model->release_streaming_residency(); - } - } auto latents = first_stage_model->diffusion_to_vae_latents(x); first_stage_model->set_temporal_tiling_enabled(vae_tiling_params.temporal_tiling); return first_stage_model->decode(n_threads, latents, vae_tiling_params, decode_video, circular_x, circular_y); @@ -2624,9 +2624,6 @@ public: return {}; } auto waveform = audio_vae_model->decode(n_threads, audio_latent); - if (free_params_immediately) { - audio_vae_model->free_params_buffer(); - } return waveform; } @@ -2946,31 +2943,27 @@ void sd_hires_params_init(sd_hires_params_t* hires_params) { } void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) { - *sd_ctx_params = {}; - sd_ctx_params->vae_decode_only = true; - sd_ctx_params->free_params_immediately = true; - sd_ctx_params->n_threads = sd_get_num_physical_cores(); - sd_ctx_params->wtype = SD_TYPE_COUNT; - sd_ctx_params->rng_type = CUDA_RNG; - sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT; - sd_ctx_params->prediction = PREDICTION_COUNT; - sd_ctx_params->lora_apply_mode = LORA_APPLY_AUTO; - sd_ctx_params->offload_params_to_cpu = false; - sd_ctx_params->max_vram = 0.f; - sd_ctx_params->stream_layers = false; - sd_ctx_params->enable_mmap = false; - sd_ctx_params->keep_clip_on_cpu = false; - sd_ctx_params->keep_control_net_on_cpu = false; - sd_ctx_params->keep_vae_on_cpu = false; - sd_ctx_params->diffusion_flash_attn = false; - sd_ctx_params->circular_x = false; - sd_ctx_params->circular_y = false; - sd_ctx_params->chroma_use_dit_mask = true; - sd_ctx_params->chroma_use_t5_mask = false; - sd_ctx_params->chroma_t5_mask_pad = 1; - sd_ctx_params->vae_format = SD_VAE_FORMAT_AUTO; - sd_ctx_params->backend = nullptr; - sd_ctx_params->params_backend = nullptr; + *sd_ctx_params = {}; + sd_ctx_params->n_threads = sd_get_num_physical_cores(); + sd_ctx_params->wtype = SD_TYPE_COUNT; + sd_ctx_params->rng_type = CUDA_RNG; + sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT; + sd_ctx_params->prediction = PREDICTION_COUNT; + sd_ctx_params->lora_apply_mode = LORA_APPLY_AUTO; + sd_ctx_params->max_vram = nullptr; + sd_ctx_params->stream_layers = false; + sd_ctx_params->enable_mmap = false; + sd_ctx_params->diffusion_flash_attn = false; + sd_ctx_params->circular_x = false; + sd_ctx_params->circular_y = false; + sd_ctx_params->chroma_use_dit_mask = true; + sd_ctx_params->chroma_use_t5_mask = false; + sd_ctx_params->chroma_t5_mask_pad = 1; + sd_ctx_params->vae_format = SD_VAE_FORMAT_AUTO; + sd_ctx_params->backend = nullptr; + sd_ctx_params->params_backend = nullptr; + sd_ctx_params->rpc_servers = nullptr; + sd_ctx_params->pulid_weights_path = nullptr; } char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { @@ -2996,22 +2989,17 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { "taesd_path: %s\n" "control_net_path: %s\n" "photo_maker_path: %s\n" + "pulid_weights_path: %s\n" "tensor_type_rules: %s\n" - "vae_decode_only: %s\n" - "free_params_immediately: %s\n" "n_threads: %d\n" "wtype: %s\n" "rng_type: %s\n" "sampler_rng_type: %s\n" "prediction: %s\n" - "offload_params_to_cpu: %s\n" - "max_vram: %.3f\n" + "max_vram: %s\n" "stream_layers: %s\n" "backend: %s\n" "params_backend: %s\n" - "keep_clip_on_cpu: %s\n" - "keep_control_net_on_cpu: %s\n" - "keep_vae_on_cpu: %s\n" "flash_attn: %s\n" "diffusion_flash_attn: %s\n" "circular_x: %s\n" @@ -3036,22 +3024,17 @@ char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) { SAFE_STR(sd_ctx_params->taesd_path), SAFE_STR(sd_ctx_params->control_net_path), SAFE_STR(sd_ctx_params->photo_maker_path), + SAFE_STR(sd_ctx_params->pulid_weights_path), SAFE_STR(sd_ctx_params->tensor_type_rules), - BOOL_STR(sd_ctx_params->vae_decode_only), - BOOL_STR(sd_ctx_params->free_params_immediately), sd_ctx_params->n_threads, sd_type_name(sd_ctx_params->wtype), sd_rng_type_name(sd_ctx_params->rng_type), sd_rng_type_name(sd_ctx_params->sampler_rng_type), sd_prediction_name(sd_ctx_params->prediction), - BOOL_STR(sd_ctx_params->offload_params_to_cpu), - sd_ctx_params->max_vram, + SAFE_STR(sd_ctx_params->max_vram), BOOL_STR(sd_ctx_params->stream_layers), SAFE_STR(sd_ctx_params->backend), SAFE_STR(sd_ctx_params->params_backend), - BOOL_STR(sd_ctx_params->keep_clip_on_cpu), - BOOL_STR(sd_ctx_params->keep_control_net_on_cpu), - BOOL_STR(sd_ctx_params->keep_vae_on_cpu), BOOL_STR(sd_ctx_params->flash_attn), BOOL_STR(sd_ctx_params->diffusion_flash_attn), BOOL_STR(sd_ctx_params->circular_x), @@ -3136,6 +3119,7 @@ void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params) { sd_img_gen_params->batch_count = 1; sd_img_gen_params->control_strength = 0.9f; sd_img_gen_params->pm_params = {nullptr, 0, nullptr, 20.f}; + sd_img_gen_params->pulid_params = {nullptr, 1.0f}; sd_img_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; sd_cache_params_init(&sd_img_gen_params->cache); sd_hires_params_init(&sd_img_gen_params->hires); @@ -3278,6 +3262,15 @@ void free_sd_ctx(sd_ctx_t* sd_ctx) { free(sd_ctx); } +SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) { + if (sd_ctx && sd_ctx->sd) { + if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) { + mode = SD_CANCEL_ALL; + } + sd_ctx->sd->set_cancel_flag(mode); + } +} + static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd, const sd::Tensor& waveform) { if (sd == nullptr || waveform.empty()) { @@ -3437,6 +3430,7 @@ struct GenerationRequest { sd_guidance_params_t guidance = {}; sd_guidance_params_t high_noise_guidance = {}; sd_pm_params_t pm_params = {}; + sd_pulid_params_t pulid_params = {}; sd_hires_params_t hires = {}; int frames = -1; int requested_frames = -1; @@ -3462,6 +3456,7 @@ struct GenerationRequest { has_ref_images = sd_img_gen_params->ref_images_count > 0; guidance = sd_img_gen_params->sample_params.guidance; pm_params = sd_img_gen_params->pm_params; + pulid_params = sd_img_gen_params->pulid_params; hires = sd_img_gen_params->hires; cache_params = &sd_img_gen_params->cache; resolve(sd_ctx); @@ -4074,6 +4069,15 @@ struct ImageGenerationEmbeds { SDCondition img_uncond; }; +struct ConditionerRunnerDoneOnExit { + Conditioner* conditioner = nullptr; + ~ConditionerRunnerDoneOnExit() { + if (conditioner != nullptr) { + conditioner->runner_done(); + } + } +}; + struct CircularAxesState { bool circular_x = false; bool circular_y = false; @@ -4212,7 +4216,7 @@ static std::optional prepare_image_generation_latents(sd } } - if (!control_image_tensor.empty() && !sd_ctx->sd->vae_decode_only) { + if (!control_image_tensor.empty()) { control_latent = sd_ctx->sd->encode_first_stage(control_image_tensor); if (control_latent.empty()) { LOG_ERROR("failed to encode control image"); @@ -4369,6 +4373,8 @@ static std::optional prepare_image_generation_embeds(sd_c GenerationRequest* request, SamplePlan* plan, ImageGenerationLatents* latents) { + ConditionerRunnerDoneOnExit conditioner_runner_done{sd_ctx->sd->cond_stage_model.get()}; + ConditionerParams condition_params; condition_params.text = request->prompt; condition_params.clip_skip = request->clip_skip; @@ -4377,6 +4383,7 @@ static std::optional prepare_image_generation_embeds(sd_c condition_params.ref_images = &latents->ref_images; sd_ctx->sd->prepare_generation_extensions(request->pm_params, + request->pulid_params, condition_params, plan->total_steps); int64_t prepare_start_ms = ggml_time_ms(); @@ -4440,10 +4447,6 @@ static std::optional prepare_image_generation_embeds(sd_c int64_t t1 = ggml_time_ms(); LOG_INFO("get_learned_condition completed, taking %.2fs", (t1 - prepare_start_ms) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->cond_stage_model->free_params_buffer(); - } - ImageGenerationEmbeds embeds; embeds.img_uncond = std::move(img_uncond); embeds.cond = std::move(cond); @@ -4455,22 +4458,33 @@ static std::optional prepare_image_generation_embeds(sd_c static sd_image_t* decode_image_outputs(sd_ctx_t* sd_ctx, const GenerationRequest& request, const std::vector>& final_latents) { - if (final_latents.size() != static_cast(request.batch_count)) { - LOG_ERROR("expected %d latents, got %zu", request.batch_count, final_latents.size()); + if (final_latents.empty()) { + LOG_ERROR("no latent images to decode"); return nullptr; } - LOG_INFO("decoding %zu latents", final_latents.size()); + if (final_latents.size() > static_cast(request.batch_count)) { + LOG_ERROR("expected at most %d latents, got %zu", request.batch_count, final_latents.size()); + return nullptr; + } + if (final_latents.size() < static_cast(request.batch_count)) { + LOG_INFO("decoding %zu/%d latents", final_latents.size(), request.batch_count); + } else { + LOG_INFO("decoding %zu latents", final_latents.size()); + } std::vector> decoded_images; - int64_t t0 = ggml_time_ms(); + int64_t t0 = ggml_time_ms(); + bool cancelled = false; for (size_t i = 0; i < final_latents.size(); i++) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling latent decodings"); + cancelled = true; + break; + } int64_t t1 = ggml_time_ms(); sd::Tensor image = sd_ctx->sd->decode_first_stage(final_latents[i]); if (image.empty()) { LOG_ERROR("decode_first_stage failed for latent %" PRId64, i + 1); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->first_stage_model->free_params_buffer(); - } return nullptr; } decoded_images.push_back(std::move(image)); @@ -4480,8 +4494,9 @@ static sd_image_t* decode_image_outputs(sd_ctx_t* sd_ctx, int64_t t4 = ggml_time_ms(); LOG_INFO("decode_first_stage completed, taking %.2fs", (t4 - t0) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->first_stage_model->free_params_buffer(); + if (decoded_images.empty()) { + LOG_ERROR(cancelled ? "cancelled before any latent images were decoded" : "no decoded images"); + return nullptr; } sd_image_t* result_images = (sd_image_t*)calloc(request.batch_count, sizeof(sd_image_t)); @@ -4501,6 +4516,11 @@ static sd::Tensor upscale_hires_latent(sd_ctx_t* sd_ctx, const sd::Tensor& latent, const GenerationRequest& request, UpscalerGGML* upscaler) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling hires latent upscale"); + return {}; + } + auto get_hires_latent_target_shape = [&]() { std::vector target_shape = latent.shape(); if (target_shape.size() < 2) { @@ -4562,11 +4582,6 @@ static sd::Tensor upscale_hires_latent(sd_ctx_t* sd_ctx, } else if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL || request.hires.upscaler == SD_HIRES_UPSCALER_LANCZOS || request.hires.upscaler == SD_HIRES_UPSCALER_NEAREST) { - if (sd_ctx->sd->vae_decode_only) { - LOG_ERROR("hires %s upscaler requires VAE encoder weights; create the context with vae_decode_only=false", - sd_hires_upscaler_name(request.hires.upscaler)); - return {}; - } if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL && upscaler == nullptr) { LOG_ERROR("hires model upscaler context is null"); return {}; @@ -4578,6 +4593,10 @@ static sd::Tensor upscale_hires_latent(sd_ctx_t* sd_ctx, sd_hires_upscaler_name(request.hires.upscaler)); return {}; } + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling hires image upscale"); + return {}; + } sd::Tensor upscaled_tensor; if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) { @@ -4614,6 +4633,10 @@ static sd::Tensor upscale_hires_latent(sd_ctx_t* sd_ctx, upscaled_tensor = sd::ops::clamp(upscaled_tensor, 0.0f, 1.0f); } + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling hires latent encode"); + return {}; + } sd::Tensor upscaled_latent = sd_ctx->sd->encode_first_stage(upscaled_tensor); if (upscaled_latent.empty()) { LOG_ERROR("encode_first_stage failed after hires %s upscale", @@ -4678,6 +4701,8 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s return nullptr; } + sd_ctx->sd->reset_cancel_flag(); + int64_t t0 = ggml_time_ms(); sd_ctx->sd->vae_tiling_params = sd_img_gen_params->vae_tiling_params; GenerationRequest request(sd_ctx, sd_img_gen_params); @@ -4713,6 +4738,18 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s std::vector> final_latents; int64_t denoise_start = ggml_time_ms(); for (int b = 0; b < request.batch_count; b++) { + sd_cancel_mode_t cancel = sd_ctx->sd->get_cancel_flag(); + if (cancel == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation"); + return nullptr; + } + if (cancel == SD_CANCEL_NEW_LATENTS) { + LOG_INFO("cancelling new latent generation, returning %zu/%d completed latents", + final_latents.size(), + request.batch_count); + break; + } + int64_t sampling_start = ggml_time_ms(); int64_t cur_seed = request.seed + b; LOG_INFO("generating image: %i/%i - seed %" PRId64, b + 1, request.batch_count, cur_seed); @@ -4756,39 +4793,41 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s b + 1, request.batch_count, (sampling_end - sampling_start) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return nullptr; } - if (sd_ctx->sd->free_params_immediately && !request.hires.enabled) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } int64_t denoise_end = ggml_time_ms(); LOG_INFO("generating %zu latent images completed, taking %.2fs", final_latents.size(), (denoise_end - denoise_start) * 1.0f / 1000); + if (final_latents.empty()) { + LOG_ERROR("no latent images generated"); + return nullptr; + } if (request.hires.enabled && request.hires.target_width > 0) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before hires fix"); + return nullptr; + } LOG_INFO("hires fix: upscaling to %dx%d", request.hires.target_width, request.hires.target_height); std::unique_ptr hires_upscaler; if (request.hires.upscaler == SD_HIRES_UPSCALER_MODEL) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before hires model load"); + return nullptr; + } LOG_INFO("hires fix: loading model upscaler from '%s'", request.hires.model_path); hires_upscaler = std::make_unique(sd_ctx->sd->n_threads, false, request.hires.upscale_tile_size, sd_ctx->sd->backend_spec, sd_ctx->sd->params_backend_spec); - const size_t max_graph_vram_bytes = sd::ggml_graph_cut::max_vram_gib_to_bytes(sd_ctx->sd->max_vram); + const size_t max_graph_vram_bytes = sd_ctx->sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER); hires_upscaler->set_max_graph_vram_bytes(max_graph_vram_bytes); if (!hires_upscaler->load_from_file(request.hires.model_path, - sd_ctx->sd->offload_params_to_cpu, sd_ctx->sd->n_threads)) { LOG_ERROR("load hires model upscaler failed"); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return nullptr; } } @@ -4811,6 +4850,10 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s std::vector> hires_final_latents; int64_t hires_denoise_start = ggml_time_ms(); for (int b = 0; b < (int)final_latents.size(); b++) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation during hires fix"); + return nullptr; + } int64_t cur_seed = request.seed + b; sd_ctx->sd->rng->manual_seed(cur_seed); sd_ctx->sd->sampler_rng->manual_seed(cur_seed); @@ -4820,9 +4863,6 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s request, hires_upscaler.get()); if (upscaled.empty()) { - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return nullptr; } @@ -4877,14 +4917,8 @@ SD_API sd_image_t* generate_image(sd_ctx_t* sd_ctx, const sd_img_gen_params_t* s b + 1, (int)final_latents.size(), (hires_sample_end - hires_sample_start) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return nullptr; } - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } int64_t hires_denoise_end = ggml_time_ms(); LOG_INFO("hires fix completed, taking %.2fs", (hires_denoise_end - hires_denoise_start) * 1.0f / 1000); @@ -4932,11 +4966,6 @@ static std::optional prepare_video_generation_latents(sd } if (!start_image.empty() || !end_image.empty()) { - if (sd_ctx->sd->vae_decode_only) { - LOG_ERROR("LTXAV image conditioning requires VAE encoder weights; create the context with vae_decode_only=false"); - return std::nullopt; - } - if (!start_image.empty() && !end_image.empty()) { LOG_INFO("FLF2V"); } else if (!start_image.empty()) { @@ -5220,6 +5249,8 @@ static ImageGenerationEmbeds prepare_video_generation_embeds(sd_ctx_t* sd_ctx, const sd_vid_gen_params_t* sd_vid_gen_params, const GenerationRequest& request, const ImageGenerationLatents& latents) { + ConditionerRunnerDoneOnExit conditioner_runner_done{sd_ctx->sd->cond_stage_model.get()}; + ImageGenerationEmbeds embeds; ConditionerParams condition_params; condition_params.clip_skip = request.clip_skip; @@ -5242,9 +5273,6 @@ static ImageGenerationEmbeds prepare_video_generation_embeds(sd_ctx_t* sd_ctx, int64_t t1 = ggml_time_ms(); LOG_INFO("get_learned_condition completed, taking %.2fs", (t1 - prepare_start_ms) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->cond_stage_model->free_params_buffer(); - } return embeds; } @@ -5256,6 +5284,10 @@ static sd_image_t* decode_video_outputs(sd_ctx_t* sd_ctx, LOG_ERROR("no latent video to decode"); return nullptr; } + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling video decode"); + return nullptr; + } sd::Tensor video_latent = final_latent; if (sd_version_is_ltxav(sd_ctx->sd->version) && video_latent.shape()[3] > sd_ctx->sd->get_latent_channel()) { @@ -5271,9 +5303,6 @@ static sd_image_t* decode_video_outputs(sd_ctx_t* sd_ctx, sd::Tensor vid = sd_ctx->sd->decode_first_stage(video_latent, true); int64_t t5 = ggml_time_ms(); LOG_INFO("decode_first_stage completed, taking %.2fs", (t5 - t4) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->first_stage_model->free_params_buffer(); - } if (vid.empty()) { LOG_ERROR("decode_first_stage failed for video"); return nullptr; @@ -5338,17 +5367,40 @@ static sd::Tensor upscale_ltx_spatial_video_latent(sd_ctx_t* sd_ctx, return {}; } + auto upsampler_manager = std::make_shared(); + upsampler_manager->set_n_threads(sd_ctx->sd->n_threads); + upsampler_manager->set_enable_mmap(sd_ctx->sd->enable_mmap); + ModelLoader& model_loader = upsampler_manager->loader(); + if (!model_loader.init_from_file(model_path)) { + LOG_ERROR("init LTX latent upsampler model loader from file failed: '%s'", model_path); + return {}; + } + std::unique_ptr upsampler = std::make_unique(sd_ctx->sd->backend_for(SDBackendModule::UPSCALER), - sd_ctx->sd->params_backend_for(SDBackendModule::UPSCALER)); - const size_t max_graph_vram_bytes = sd::ggml_graph_cut::max_vram_gib_to_bytes(sd_ctx->sd->max_vram); + model_loader.get_tensor_storage_map(), + upsampler_manager); + const size_t max_graph_vram_bytes = sd_ctx->sd->max_graph_vram_bytes_for_module(SDBackendModule::UPSCALER); upsampler->set_max_graph_vram_bytes(max_graph_vram_bytes); - if (!upsampler->load_from_file(model_path, sd_ctx->sd->n_threads)) { - LOG_ERROR("load LTX latent upsampler failed"); + if (upsampler->model == nullptr) { + LOG_ERROR("init LTX latent upsampler from metadata failed"); + return {}; + } + + std::map tensors; + upsampler->get_param_tensors(tensors); + if (!upsampler_manager->register_param_tensors("LTX latent upsampler", + std::move(tensors), + ModelManager::ResidencyMode::ParamBackend, + sd_ctx->sd->backend_for(SDBackendModule::UPSCALER), + sd_ctx->sd->params_backend_for(SDBackendModule::UPSCALER)) || + !upsampler_manager->validate_registered_tensors()) { + LOG_ERROR("register LTX latent upsampler tensors with model manager failed"); return {}; } sd::Tensor upscaled = upsampler->compute(sd_ctx->sd->n_threads, unnormalized); + upsampler_manager.reset(); upsampler.reset(); if (upscaled.empty()) { LOG_ERROR("LTX latent spatial upscale failed"); @@ -5382,11 +5434,6 @@ static bool apply_ltxv_refine_image_conditioning(sd_ctx_t* sd_ctx, sd_vid_gen_params->end_image.data == nullptr) { return true; } - if (sd_ctx->sd->vae_decode_only) { - LOG_ERROR("LTXV refine image conditioning requires VAE encoder weights; create the context with vae_decode_only=false"); - return false; - } - constexpr float conditioning_strength = 1.f; int latent_channels = sd_ctx->sd->get_latent_channel(); sd::Tensor video_latent = *latent; @@ -5486,6 +5533,9 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, if (audio_out != nullptr) { *audio_out = nullptr; } + + sd_ctx->sd->reset_cancel_flag(); + if (num_frames_out != nullptr) { *num_frames_out = 0; } @@ -5547,6 +5597,10 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, sd::Tensor noise = sd::Tensor::randn_like(x_t, sd_ctx->sd->rng); if (plan.high_noise_sample_steps > 0) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before high-noise sampling"); + return false; + } LOG_DEBUG("sample(high noise) %dx%dx%d", W, H, T); int64_t sampling_start = ggml_time_ms(); @@ -5581,20 +5635,18 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, int64_t sampling_end = ggml_time_ms(); if (x_t_sampled.empty()) { LOG_ERROR("sampling(high noise) failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->high_noise_diffusion_model->free_params_buffer(); - } return false; } x_t = std::move(x_t_sampled); noise = {}; LOG_INFO("sampling(high noise) completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->high_noise_diffusion_model->free_params_buffer(); - } } + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before sampling"); + return false; + } LOG_DEBUG("sample %dx%dx%d", W, H, T); int64_t sampling_start = ggml_time_ms(); sd::Tensor final_latent = sd_ctx->sd->sample(sd_ctx->sd->diffusion_model, @@ -5625,15 +5677,16 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, int64_t sampling_end = ggml_time_ms(); if (final_latent.empty()) { - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } LOG_ERROR("sampling failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); return false; } LOG_INFO("sampling completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); if (latent_upscale_enabled) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before latent upscale"); + return false; + } int64_t upscale_start = ggml_time_ms(); sd::Tensor upscaled_latent = upscale_ltx_spatial_video_latent(sd_ctx, request.hires.model_path, @@ -5641,9 +5694,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, latents.audio_length); int64_t upscale_end = ggml_time_ms(); if (upscaled_latent.empty()) { - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return false; } LOG_INFO("LTX latent spatial upscale completed, taking %.2fs", @@ -5676,9 +5726,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, LOG_ERROR("failed to resize LTX audio latent for latent upscale: %d -> %d", latents.audio_length, target_audio_length); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return false; } x_t = pack_ltxav_audio_and_video_latents(video_latent, audio_latent); @@ -5699,6 +5746,10 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, } sd::Tensor hires_denoise_mask; sd::Tensor hires_video_positions; + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before latent upscale refine"); + return false; + } if (!apply_ltxv_refine_image_conditioning(sd_ctx, sd_vid_gen_params, hires_request, @@ -5706,9 +5757,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, &x_t, &hires_denoise_mask, &hires_video_positions)) { - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } return false; } noise = sd::Tensor::randn_like(x_t, sd_ctx->sd->rng); @@ -5765,9 +5813,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, hires_request.cache_params, hires_video_positions); sampling_end = ggml_time_ms(); - if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); - } if (final_latent.empty()) { LOG_ERROR("sampling(latent upscale) failed after %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); @@ -5775,8 +5820,6 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, } LOG_INFO("sampling(latent upscale) completed, taking %.2fs", (sampling_end - sampling_start) * 1.0f / 1000); - } else if (sd_ctx->sd->free_params_immediately) { - sd_ctx->sd->diffusion_model->free_params_buffer(); } int64_t latent_end = ggml_time_ms(); @@ -5786,6 +5829,10 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, if (sd_version_is_ltxav(sd_ctx->sd->version) && latents.audio_length > 0 && sd_ctx->sd->audio_vae_model != nullptr) { + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before audio decode"); + return false; + } int64_t audio_latent_decode_start = ggml_time_ms(); auto audio_latent = unpack_ltxav_audio_latent(final_latent, @@ -5818,6 +5865,11 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, final_latent = sd::ops::slice(final_latent, 2, latents.ref_image_num, final_latent.shape()[2]); } + if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { + LOG_ERROR("cancelling generation before video decode"); + free_sd_audio(generated_audio); + return false; + } auto result = decode_video_outputs(sd_ctx, latent_upscale_enabled ? hires_request : request, final_latent, num_frames_out); if (result == nullptr) { free_sd_audio(generated_audio); diff --git a/otherarch/sdcpp/src/upscaler.cpp b/otherarch/sdcpp/src/upscaler.cpp index 8635f6778..d02366ecb 100644 --- a/otherarch/sdcpp/src/upscaler.cpp +++ b/otherarch/sdcpp/src/upscaler.cpp @@ -18,6 +18,12 @@ UpscalerGGML::UpscalerGGML(int n_threads, params_backend_spec(std::move(params_backend_spec)) { } +UpscalerGGML::~UpscalerGGML() { + // ModelManager holds raw ggml tensor pointers owned by the runner context. + model_manager.reset(); + esrgan_upscaler.reset(); +} + void UpscalerGGML::set_max_graph_vram_bytes(size_t max_vram_bytes) { max_graph_vram_bytes = max_vram_bytes; if (esrgan_upscaler) { @@ -33,17 +39,12 @@ void UpscalerGGML::set_stream_layers_enabled(bool enabled) { } bool UpscalerGGML::load_from_file(const std::string& esrgan_path, - bool offload_params_to_cpu, int n_threads) { ggml_log_set(ggml_log_callback_default, nullptr); std::string error; if (!backend_manager.init(backend_spec.c_str(), params_backend_spec.c_str(), - offload_params_to_cpu, - false, - false, - false, &error)) { LOG_ERROR("upscaler backend config failed: %s", error.c_str()); return false; @@ -72,22 +73,39 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path, return false; } - ModelLoader model_loader; - if (!model_loader.init_from_file_and_convert_name(esrgan_path)) { + model_manager = std::make_shared(); + model_manager->set_n_threads(n_threads); + model_manager->set_enable_mmap(false); + + ModelLoader& model_loader = model_manager->loader(); + if (!model_loader.init_from_file_and_convert_name(esrgan_path, "", VERSION_ESRGAN)) { LOG_ERROR("init model loader from file failed: '%s'", esrgan_path.c_str()); + return false; } model_loader.set_wtype_override(model_data_type); LOG_INFO("Upscaler weight type: %s", ggml_type_name(model_data_type)); esrgan_upscaler = std::make_shared(backend_for(SDBackendModule::UPSCALER), - params_backend_for(SDBackendModule::UPSCALER), - tile_size, - model_loader.get_tensor_storage_map()); + model_loader.get_tensor_storage_map(), + model_manager); + if (esrgan_upscaler == nullptr || esrgan_upscaler->rrdb_net == nullptr) { + LOG_ERROR("init esrgan model from metadata failed: '%s'", esrgan_path.c_str()); + return false; + } esrgan_upscaler->set_max_graph_vram_bytes(max_graph_vram_bytes); esrgan_upscaler->set_stream_layers_enabled(stream_layers_enabled); if (direct) { esrgan_upscaler->set_conv2d_direct_enabled(true); } - if (!esrgan_upscaler->load_from_file(esrgan_path, n_threads)) { + + std::map tensors; + esrgan_upscaler->get_param_tensors(tensors); + if (!model_manager->register_param_tensors("ESRGAN", + std::move(tensors), + backend_manager.params_backend_is_disk(SDBackendModule::UPSCALER) ? ModelManager::ResidencyMode::Disk : ModelManager::ResidencyMode::ParamBackend, + backend_for(SDBackendModule::UPSCALER), + params_backend_for(SDBackendModule::UPSCALER)) || + !model_manager->validate_registered_tensors()) { + LOG_ERROR("register esrgan tensors with model manager failed"); return false; } return true; @@ -95,6 +113,7 @@ bool UpscalerGGML::load_from_file(const std::string& esrgan_path, sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_tensor) { sd::Tensor upscaled; + const int scale = esrgan_upscaler->config.scale; if (tile_size <= 0 || (input_tensor.shape()[0] <= tile_size && input_tensor.shape()[1] <= tile_size)) { upscaled = esrgan_upscaler->compute(n_threads, input_tensor); } else { @@ -108,9 +127,9 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te }; upscaled = process_tiles_2d(input_tensor, - static_cast(input_tensor.shape()[0] * esrgan_upscaler->scale), - static_cast(input_tensor.shape()[1] * esrgan_upscaler->scale), - esrgan_upscaler->scale, + static_cast(input_tensor.shape()[0] * scale), + static_cast(input_tensor.shape()[1] * scale), + scale, tile_size, tile_size, 0.25f, @@ -129,8 +148,9 @@ sd::Tensor UpscalerGGML::upscale_tensor(const sd::Tensor& input_te sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor) { // upscale_factor, unused for RealESRGAN_x4plus_anime_6B.pth sd_image_t upscaled_image = {0, 0, 0, nullptr}; - int output_width = (int)input_image.width * esrgan_upscaler->scale; - int output_height = (int)input_image.height * esrgan_upscaler->scale; + const int scale = esrgan_upscaler->config.scale; + int output_width = (int)input_image.width * scale; + int output_height = (int)input_image.height * scale; LOG_INFO("upscaling from (%i x %i) to (%i x %i)", input_image.width, input_image.height, output_width, output_height); @@ -153,7 +173,6 @@ struct upscaler_ctx_t { }; upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str, - bool offload_params_to_cpu, bool direct, int n_threads, int tile_size, @@ -170,7 +189,7 @@ upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str, return nullptr; } - if (!upscaler_ctx->upscaler->load_from_file(esrgan_path, offload_params_to_cpu, n_threads)) { + if (!upscaler_ctx->upscaler->load_from_file(esrgan_path, n_threads)) { delete upscaler_ctx->upscaler; upscaler_ctx->upscaler = nullptr; free(upscaler_ctx); @@ -187,7 +206,7 @@ int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) { if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr || upscaler_ctx->upscaler->esrgan_upscaler == nullptr) { return 1; } - return upscaler_ctx->upscaler->esrgan_upscaler->scale; + return upscaler_ctx->upscaler->esrgan_upscaler->config.scale; } void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) { diff --git a/otherarch/sdcpp/src/upscaler.h b/otherarch/sdcpp/src/upscaler.h index c9fce1844..38150f59f 100644 --- a/otherarch/sdcpp/src/upscaler.h +++ b/otherarch/sdcpp/src/upscaler.h @@ -4,6 +4,7 @@ #include "core/ggml_extend_backend.h" #include "core/tensor.hpp" #include "model/upscaler/esrgan.hpp" +#include "model_manager.h" #include "stable-diffusion.h" #include @@ -11,6 +12,7 @@ struct UpscalerGGML { SDBackendManager backend_manager; + std::shared_ptr model_manager; ggml_type model_data_type = GGML_TYPE_F16; std::shared_ptr esrgan_upscaler; std::string esrgan_path; @@ -27,9 +29,9 @@ struct UpscalerGGML { int tile_size = 128, std::string backend_spec = "", std::string params_backend_spec = ""); + ~UpscalerGGML(); bool load_from_file(const std::string& esrgan_path, - bool offload_params_to_cpu, int n_threads); void set_max_graph_vram_bytes(size_t max_vram_bytes); void set_stream_layers_enabled(bool enabled); diff --git a/otherarch/sdcpp/src/weight_manager.h b/otherarch/sdcpp/src/weight_manager.h new file mode 100644 index 000000000..28d6cf5c4 --- /dev/null +++ b/otherarch/sdcpp/src/weight_manager.h @@ -0,0 +1,15 @@ +#ifndef __WEIGHT_MANAGER_H__ +#define __WEIGHT_MANAGER_H__ + +#include + +struct ggml_tensor; + +struct RunnerWeightManager { + virtual ~RunnerWeightManager() = default; + virtual bool prepare_params(const std::vector& tensors) = 0; + virtual void release_compute_backend_params(const std::vector& tensors) = 0; + virtual void release_params_backend_params(const std::vector& tensors) = 0; +}; + +#endif // __WEIGHT_MANAGER_H__ From 1e81db242685390d4c0b2612f8ab9efa01a616b0 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:35:58 +0800 Subject: [PATCH 014/292] updated cmake --- CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c44618e7e..40816ed33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -495,8 +495,9 @@ add_library(sdtype_adapter otherarch/sdcpp/src/core/ggml_graph_cut.cpp otherarch/sdcpp/src/core/ggml_graph_cut.h otherarch/sdcpp/examples/cli/image_metadata.cpp - otherarch/sdcpp/src/core/layer_registry.cpp - otherarch/sdcpp/src/core/layer_registry.h + otherarch/sdcpp/src/model_manager.cpp + otherarch/sdcpp/src/model_manager.h + otherarch/sdcpp/src/extensions/pulid_extension.cpp otherarch/sdcpp/src/model_loader.cpp otherarch/sdcpp/src/extensions/photomaker_extension.cpp otherarch/sdcpp/src/runtime/sample-cache.cpp From ea21e03955e116423890af168b03360bd6bbfb2a Mon Sep 17 00:00:00 2001 From: Ruben Ortlam Date: Wed, 17 Jun 2026 10:59:35 +0200 Subject: [PATCH 015/292] Revert "cuda: reset cuda context after reading memory size (#23935)" (#24715) This reverts commit 0f7fada56bea32c9e0db3874b801d887ff6c4529. --- ggml/src/ggml-cuda/ggml-cuda.cu | 75 ++++----------------------------- 1 file changed, 9 insertions(+), 66 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 01c29a8c6..34cdbc81c 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -622,18 +622,6 @@ ggml_backend_cuda_context::~ggml_backend_cuda_context() { // cuda buffer -struct ggml_backend_cuda_device_context { - int device; - std::string name; - std::string description; - std::string pci_bus_id; - int op_offload_min_batch_size; -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - std::mutex device_mutex; - int active_count = 0; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) -}; - struct ggml_backend_cuda_buffer_context { int device; void * dev_ptr = nullptr; @@ -651,13 +639,6 @@ struct ggml_backend_cuda_buffer_context { static void ggml_backend_cuda_buffer_free_buffer(ggml_backend_buffer_t buffer) { ggml_backend_cuda_buffer_context * ctx = (ggml_backend_cuda_buffer_context *)buffer->context; - -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) buffer->buft->device->context; - std::lock_guard lock(dev_ctx->device_mutex); - dev_ctx->active_count--; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - delete ctx; } @@ -810,12 +791,6 @@ static ggml_backend_buffer_t ggml_backend_cuda_buffer_type_alloc_buffer(ggml_bac ggml_backend_cuda_buffer_context * ctx = new ggml_backend_cuda_buffer_context(buft_ctx->device, dev_ptr); -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) buft->device->context; - std::lock_guard lock(dev_ctx->device_mutex); - dev_ctx->active_count++; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - return ggml_backend_buffer_init(buft, ggml_backend_cuda_buffer_interface, ctx, size); } @@ -1515,12 +1490,6 @@ static bool ggml_backend_buft_is_cuda_host(ggml_backend_buffer_type_t buft) { } static void ggml_backend_cuda_host_buffer_free_buffer(ggml_backend_buffer_t buffer) { -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) buffer->buft->device->context; - std::lock_guard lock(dev_ctx->device_mutex); - dev_ctx->active_count--; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - CUDA_CHECK(cudaFreeHost(buffer->context)); } @@ -1529,8 +1498,6 @@ static void * ggml_cuda_host_malloc(size_t size) { return nullptr; } - ggml_cuda_set_device(0); // cudaMallocHost can create the implicit CUDA device context, make sure that this is consistently done on device 0. - void * ptr = nullptr; cudaError_t err = cudaMallocHost((void **) &ptr, size); if (err != cudaSuccess) { @@ -1556,12 +1523,6 @@ static ggml_backend_buffer_t ggml_backend_cuda_host_buffer_type_alloc_buffer(ggm buffer->buft = buft; buffer->iface.free_buffer = ggml_backend_cuda_host_buffer_free_buffer; -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) buft->device->context; - std::lock_guard lock(dev_ctx->device_mutex); - dev_ctx->active_count++; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - return buffer; } @@ -3179,12 +3140,6 @@ static const char * ggml_backend_cuda_get_name(ggml_backend_t backend) { static void ggml_backend_cuda_free(ggml_backend_t backend) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *)backend->context; -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) backend->device->context; - std::lock_guard lock(dev_ctx->device_mutex); - dev_ctx->active_count--; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - delete cuda_ctx; delete backend; } @@ -4916,6 +4871,14 @@ void ggml_backend_cuda_unregister_host_buffer(void * buffer) { // backend device +struct ggml_backend_cuda_device_context { + int device; + std::string name; + std::string description; + std::string pci_bus_id; + int op_offload_min_batch_size; +}; + static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; return ctx->name.c_str(); @@ -5004,11 +4967,6 @@ static bool ggml_backend_cuda_get_available_uma_memory(long * available_memory_k static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; - -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - std::lock_guard lock(ctx->device_mutex); -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_cuda_set_device(ctx->device); CUDA_CHECK(cudaMemGetInfo(free, total)); @@ -5035,13 +4993,6 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * } #endif // defined(__linux__) -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - // If no backends or buffers are active, the cudaMemGetInfo call above lazily created a CUDA - // context that permanently consumes VRAM. Reset the device to free it. - if (ctx->active_count == 0) { - CUDA_CHECK(cudaDeviceReset()); - } -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) } static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend_dev_t dev) { @@ -5745,21 +5696,13 @@ ggml_backend_t ggml_backend_cuda_init(int device) { return nullptr; } - ggml_backend_dev_t dev = ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), device); - ggml_backend_t cuda_backend = new ggml_backend { /* .guid = */ ggml_backend_cuda_guid(), /* .iface = */ ggml_backend_cuda_interface, - /* .device = */ dev, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_cuda_reg(), device), /* .context = */ ctx, }; -#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - ggml_backend_cuda_device_context * dev_ctx = (ggml_backend_cuda_device_context *) dev->context; - std::lock_guard lock(dev_ctx->device_mutex); - dev_ctx->active_count++; -#endif // !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) - return cuda_backend; } From 558e221b70c63d4bb1d54b73f9a0bdd3d9475a8f Mon Sep 17 00:00:00 2001 From: Winston Ma Date: Wed, 17 Jun 2026 17:14:48 +0800 Subject: [PATCH 016/292] vulkan: record actual memory properties during buffer creation (#24326) --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 243ab76ee..9a36b45de 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -3008,13 +3008,13 @@ static vk_buffer ggml_vk_create_buffer(vk_device& device, size_t size, const std if (memory_type_indices.empty()) { continue; } - buf->memory_property_flags = req_flags; bool done = false; for (auto mtype_it = memory_type_indices.begin(); mtype_it != memory_type_indices.end(); mtype_it++) { try { buf->device_memory = device->device.allocateMemory({ mem_req.size, *mtype_it, &mem_flags_info }); + buf->memory_property_flags = mem_props.memoryTypes[*mtype_it].propertyFlags; done = true; break; } catch (const vk::SystemError& e) { From 8086439a4cea94c71a5dfb8fe4ad1546aebd640f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 17 Jun 2026 13:25:47 +0200 Subject: [PATCH 017/292] webui: export conversations as jsonl (#24688) * webui: export conversations as jsonl each session is one jsonl file, a session header line followed by one line per message exporting multiple conversations bundles them into a zip, one jsonl file each * webui: import jsonl and zip conversation exports parse the new jsonl session format and zip archives on import keep supporting the legacy json format --- tools/ui/package-lock.json | 8 + tools/ui/package.json | 1 + .../SettingsChatImportExportTab.svelte | 38 ++-- tools/ui/src/lib/enums/files.enums.ts | 6 +- .../ui/src/lib/stores/conversations.svelte.ts | 180 ++++++++++++++++-- 5 files changed, 189 insertions(+), 44 deletions(-) diff --git a/tools/ui/package-lock.json b/tools/ui/package-lock.json index 61fa529d2..9d0cdfea6 100644 --- a/tools/ui/package-lock.json +++ b/tools/ui/package-lock.json @@ -40,6 +40,7 @@ "eslint-config-prettier": "10.1.8", "eslint-plugin-storybook": "10.4.2", "eslint-plugin-svelte": "3.19.0", + "fflate": "0.8.3", "globals": "16.5.0", "highlight.js": "11.11.1", "http-server": "14.1.1", @@ -9454,6 +9455,13 @@ } } }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "dev": true, + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", diff --git a/tools/ui/package.json b/tools/ui/package.json index 2f47992e1..480392288 100644 --- a/tools/ui/package.json +++ b/tools/ui/package.json @@ -59,6 +59,7 @@ "eslint-config-prettier": "10.1.8", "eslint-plugin-storybook": "10.4.2", "eslint-plugin-svelte": "3.19.0", + "fflate": "0.8.3", "globals": "16.5.0", "highlight.js": "11.11.1", "http-server": "14.1.1", diff --git a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte index b7b91d65b..a86d68584 100644 --- a/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte +++ b/tools/ui/src/lib/components/app/settings/SettingsChat/SettingsChatImportExportTab.svelte @@ -132,14 +132,18 @@ async function handleExportConfirm(selectedConversations: DatabaseConversation[]) { try { - const allData: ExportedConversations = await Promise.all( + const allData: ExportedConversation[] = await Promise.all( selectedConversations.map(async (conv) => { const messages = await conversationsStore.getConversationMessages(conv.id); return { conv: $state.snapshot(conv), messages: $state.snapshot(messages) }; }) ); - conversationsStore.downloadConversationFile(allData); + if (allData.length === 1) { + conversationsStore.downloadConversationFile(allData[0]); + } else { + conversationsStore.downloadConversationsArchive(allData); + } exportedConversations = selectedConversations; showExportSummary = true; @@ -156,37 +160,21 @@ const input = document.createElement('input'); input.type = HtmlInputType.FILE; - input.accept = FileExtensionText.JSON; + input.accept = `${FileExtensionText.JSON},${FileExtensionText.JSONL},${FileExtensionText.ZIP}`; input.onchange = async (e) => { const file = (e.target as HTMLInputElement)?.files?.[0]; if (!file) return; try { - const text = await file.text(); - const parsedData = JSON.parse(text); - let importedData: ExportedConversations; + const importedData = await conversationsStore.parseImportFile(file); - if (Array.isArray(parsedData)) { - importedData = parsedData; - } else if ( - parsedData && - typeof parsedData === 'object' && - 'conv' in parsedData && - 'messages' in parsedData - ) { - // Single conversation object - importedData = [parsedData]; - } else { - throw new Error( - 'Invalid file format: expected array of conversations or single conversation object' - ); + if (importedData.length === 0) { + throw new Error('No conversations found in file'); } fullImportData = importedData; - availableConversations = importedData.map( - (item: { conv: DatabaseConversation; messages: DatabaseMessage[] }) => item.conv - ); + availableConversations = importedData.map((item) => item.conv); messageCountMap = createMessageCountMap(importedData); showImportDialog = true; } catch (err: unknown) { @@ -258,7 +246,7 @@ { + // `toolCalls` is stored as a JSON string; drop it when empty, otherwise parse it. + const { toolCalls, ...rest } = message; + const normalized = toolCalls ? { ...rest, toolCalls: JSON.parse(toolCalls) } : rest; + + return JSON.stringify({ type: 'message', message: normalized }); + }); + + return [sessionLine, ...messageLines].join('\n'); + } + + /** + * Parses the JSONL session format produced by {@link serializeSessionToJsonl}. + * A `type: 'session'` line starts a new session; following `type: 'message'` + * lines are appended to it. Supports multiple sessions in a single file. + * @param text - The JSONL file contents + * @returns The parsed conversations with their messages + */ + parseSessionsJsonl(text: string): ExportedConversation[] { + const sessions: ExportedConversation[] = []; + let current: ExportedConversation | null = null; + + for (const line of text.split('\n')) { + const trimmed = line.trim(); + if (!trimmed) continue; + + const record = JSON.parse(trimmed); + + if (record.type === 'session') { + // Drop the discriminator and harness marker; the rest is the conversation. + const conv = { ...record }; + delete conv.type; + delete conv.harness; + current = { conv: conv as DatabaseConversation, messages: [] }; + sessions.push(current); + } else if (record.type === 'message') { + if (!current) { + throw new Error('Invalid JSONL: message record before any session record'); + } + + const message = record.message as DatabaseMessage; + // `toolCalls` is parsed to an array on export; the DB stores it as a string. + if (message.toolCalls !== undefined && typeof message.toolCalls !== 'string') { + message.toolCalls = JSON.stringify(message.toolCalls); + } + current.messages.push(message); + } + // Ignore unknown record types for forward compatibility. + } + + return sessions; + } + + /** + * Parses an import file into conversations, accepting the current `.jsonl` and + * `.zip` formats as well as the legacy `.json` format. + * @param file - The user-selected file + * @returns The parsed conversations with their messages + */ + async parseImportFile(file: File): Promise { + const name = file.name.toLowerCase(); + + if (name.endsWith(FileExtensionText.ZIP)) { + const entries = unzipSync(new Uint8Array(await file.arrayBuffer())); + const sessions: ExportedConversation[] = []; + for (const [entryName, bytes] of Object.entries(entries)) { + if (!entryName.toLowerCase().endsWith(FileExtensionText.JSONL)) continue; + sessions.push(...this.parseSessionsJsonl(strFromU8(bytes))); + } + return sessions; + } + + const text = await file.text(); + + if (name.endsWith(FileExtensionText.JSONL)) { + return this.parseSessionsJsonl(text); + } + + // Legacy JSON format: an array of conversations or a single conversation object. + const parsed = JSON.parse(text); + if (Array.isArray(parsed)) { + return parsed; + } + if (parsed && typeof parsed === 'object' && 'conv' in parsed && 'messages' in parsed) { + return [parsed]; + } + throw new Error( + 'Invalid file format: expected array of conversations or single conversation object' + ); } /** * Triggers a browser download of the provided exported conversation data - * @param data - The exported conversation payload (either a single conversation or array of them) + * @param data - The exported conversation payload (a single conversation with its messages) * @param filename - Filename; if omitted, a deterministic name is generated */ - downloadConversationFile(data: ExportedConversations, filename?: string): void { - // Choose the first conversation or message - const conversation = - 'conv' in data ? data.conv : Array.isArray(data) ? data[0]?.conv : undefined; - const msgs = - 'messages' in data ? data.messages : Array.isArray(data) ? data[0]?.messages : undefined; + downloadConversationFile(data: ExportedConversation, filename?: string): void { + const { conv: conversation, messages: msgs } = data; if (!conversation) { console.error('Invalid data: missing conversation'); return; } - let downloadFilename: string; + const downloadFilename = filename ?? this.generateConversationFilename(conversation, msgs); - if (filename) { - downloadFilename = filename; - } else if (Array.isArray(data) && data.length > 1) { - downloadFilename = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations.json`; - } else { - downloadFilename = this.generateConversationFilename(conversation, msgs); + const jsonl = this.serializeSessionToJsonl(data); + const blob = new Blob([jsonl], { type: MimeTypeText.JSONL }); + this.triggerDownload(blob, downloadFilename); + } + + /** + * Triggers a browser download of multiple conversations as a `.zip`, one + * `.jsonl` file per conversation. + * @param data - The conversations to export + */ + downloadConversationsArchive(data: ExportedConversation[]): void { + if (data.length === 0) { + console.error('Invalid data: no conversations to export'); + return; } - const blob = new Blob([JSON.stringify(data, null, 2)], { type: 'application/json' }); + const usedNames = new SvelteSet(); + const files: Record = {}; + + for (const session of data) { + const baseName = this.generateConversationFilename(session.conv, session.messages); + + // Disambiguate any duplicate filenames within the archive. + let entryName = baseName; + let suffix = 1; + while (usedNames.has(entryName)) { + entryName = baseName.replace( + new RegExp(`${FileExtensionText.JSONL}$`), + `_${suffix++}${FileExtensionText.JSONL}` + ); + } + usedNames.add(entryName); + + files[entryName] = strToU8(this.serializeSessionToJsonl(session)); + } + + const archiveName = `${new Date().toISOString().split(ISO_DATE_TIME_SEPARATOR)[0]}_conversations${FileExtensionText.ZIP}`; + + const zipped = zipSync(files); + const blob = new Blob([zipped], { type: MimeTypeApplication.ZIP }); + this.triggerDownload(blob, archiveName); + } + + /** + * Triggers a browser download of a blob under the given filename. + */ + private triggerDownload(blob: Blob, filename: string): void { const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; - a.download = downloadFilename; + a.download = filename; document.body.appendChild(a); a.click(); document.body.removeChild(a); From d1759e4156cece09a6ee5231ae1219f461b16d23 Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Wed, 17 Jun 2026 22:20:01 +0800 Subject: [PATCH 018/292] [SYCL] Add conv_3d (#24691) * add conv_3d * optimize * update ops.md * restore test script * rm unused code * rm copyright notes --- docs/ops.md | 2 +- docs/ops/SYCL.csv | 516 +++++++++++++++---------------- ggml/src/ggml-sycl/backend.hpp | 1 + ggml/src/ggml-sycl/conv3d.cpp | 218 +++++++++++++ ggml/src/ggml-sycl/conv3d.hpp | 8 + ggml/src/ggml-sycl/ggml-sycl.cpp | 14 + 6 files changed, 500 insertions(+), 259 deletions(-) create mode 100644 ggml/src/ggml-sycl/conv3d.cpp create mode 100644 ggml/src/ggml-sycl/conv3d.hpp diff --git a/docs/ops.md b/docs/ops.md index d46f9b731..9e9442024 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -29,7 +29,7 @@ Legend: | CONT | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ | | CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | | CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | -| CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | +| CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | | COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | diff --git a/docs/ops/SYCL.csv b/docs/ops/SYCL.csv index 19a5346f8..116197d2f 100644 --- a/docs/ops/SYCL.csv +++ b/docs/ops/SYCL.csv @@ -4676,264 +4676,264 @@ "SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=1","support","0","no","SYCL" "SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=0","support","0","no","SYCL" "SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=1","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","0","no","SYCL" -"SYCL0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","0","no","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=1,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=3,KW=3,s0=2,s1=2,s2=2,p0=1,p1=1,p2=1,d0=2,d1=2,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=2,IC=3,ID=18,IH=22,IW=20,OC=4,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f16","support","1","yes","SYCL" +"SYCL0","CONV_3D","N=1,IC=4,ID=8,IH=8,IW=8,OC=8,KD=1,KH=1,KW=1,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f16","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_1D","ne_input=[1,1,1,1],ne_kernel=[1,1,1,1],s0=1,p0=0,d0=1","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_1D","ne_input=[1,1,1,1],ne_kernel=[1,1,1,1],s0=2,p0=0,d0=1","support","1","yes","SYCL" "SYCL0","CONV_TRANSPOSE_1D","ne_input=[1,1,1,1],ne_kernel=[1,1,1,1],s0=3,p0=0,d0=1","support","1","yes","SYCL" diff --git a/ggml/src/ggml-sycl/backend.hpp b/ggml/src/ggml-sycl/backend.hpp index a526d8e58..1f5a91272 100644 --- a/ggml/src/ggml-sycl/backend.hpp +++ b/ggml/src/ggml-sycl/backend.hpp @@ -17,6 +17,7 @@ #include "common.hpp" #include "concat.hpp" #include "conv.hpp" +#include "conv3d.hpp" #include "convert.hpp" #include "count-equal.hpp" #include "cpy.hpp" diff --git a/ggml/src/ggml-sycl/conv3d.cpp b/ggml/src/ggml-sycl/conv3d.cpp new file mode 100644 index 000000000..2fa29f930 --- /dev/null +++ b/ggml/src/ggml-sycl/conv3d.cpp @@ -0,0 +1,218 @@ +#include "conv3d.hpp" + +static inline int64_t ggml_sycl_conv3d_calc_patch_total(const ggml_tensor * dst, int32_t n) { + return (int64_t) n * dst->ne[0] * dst->ne[1] * dst->ne[2]; +} + +static inline int64_t ggml_sycl_conv3d_calc_knl_n_total(const ggml_tensor * src0, int32_t c) { + return (int64_t) src0->ne[0] * src0->ne[1] * src0->ne[2] * c; +} + +static inline void ggml_sycl_conv3d_write_output( + const ggml_tensor * dst, + const float * src, float * dst_data, + int64_t patch_total, int64_t oc, + int64_t dst_w, int64_t dst_h, int64_t dst_d, + dpct::queue_ptr stream) { + const int64_t dst_nb0 = dst->nb[0]; + const int64_t dst_nb1 = dst->nb[1]; + const int64_t dst_nb2 = dst->nb[2]; + const int64_t dst_nb3 = dst->nb[3]; + const int64_t total = patch_total * oc; + const int64_t block_size = 256; + const int64_t num_work_items = ((total + block_size - 1) / block_size) * block_size; + + stream->parallel_for(sycl::range<1>(num_work_items), [=](sycl::id<1> id) { + const int64_t i = id[0]; + if (i >= total) { + return; + } + + const int64_t patch_idx = i / oc; + const int64_t out_ch = i % oc; + const int64_t p_in_batch = patch_idx % (dst_w * dst_h * dst_d); + const int64_t batch_idx = patch_idx / (dst_w * dst_h * dst_d); + const int64_t dst_z = p_in_batch / (dst_w * dst_h); + const int64_t dst_y = (p_in_batch % (dst_w * dst_h)) / dst_w; + const int64_t dst_x = p_in_batch % dst_w; + const int64_t ocn_idx = batch_idx * oc + out_ch; + + const int64_t dst_offset = dst_x * dst_nb0 + dst_y * dst_nb1 + dst_z * dst_nb2 + ocn_idx * dst_nb3; + // `src` is a column-major (m x n) GEMM output where m == patch_total, n == oc. + // GEMM stores element (row, col) at index `row + col*m`, so compute index accordingly. + const int64_t src_index = patch_idx + out_ch * patch_total; + const float value = src[src_index]; + *(float *)((char *)dst_data + dst_offset) = value; + }); +} + +void ggml_sycl_op_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * src0 = dst->src[0]; + const ggml_tensor * src1 = dst->src[1]; + + GGML_ASSERT(src0->type == GGML_TYPE_F16 || src0->type == GGML_TYPE_F32); + GGML_ASSERT(src1->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + GGML_ASSERT(ggml_is_contiguous(src0)); + GGML_ASSERT(ggml_is_contiguous(src1)); + + const int32_t * opts = (const int32_t *) dst->op_params; + const int32_t s0 = opts[0]; + const int32_t s1 = opts[1]; + const int32_t s2 = opts[2]; + const int32_t p0 = opts[3]; + const int32_t p1 = opts[4]; + const int32_t p2 = opts[5]; + const int32_t d0 = opts[6]; + const int32_t d1 = opts[7]; + const int32_t d2 = opts[8]; + const int32_t c = opts[9]; + const int32_t n = opts[10]; + const int32_t oc = opts[11]; + + const int64_t knl_w = src0->ne[0]; + const int64_t knl_h = src0->ne[1]; + const int64_t knl_d = src0->ne[2]; + + const int64_t patch_total = ggml_sycl_conv3d_calc_patch_total(dst, n); + const int64_t knl_n_total = ggml_sycl_conv3d_calc_knl_n_total(src0, c); + + const size_t kernel_type_size = ggml_element_size(src0); + + ggml_sycl_pool_alloc gemm_output(ctx.pool()); + gemm_output.alloc((size_t) patch_total * oc); + + ggml_tensor dst_mat = {}; + dst_mat.type = GGML_TYPE_F32; + dst_mat.ne[0] = patch_total; + dst_mat.ne[1] = oc; + dst_mat.ne[2] = 1; + dst_mat.ne[3] = 1; + dst_mat.nb[0] = sizeof(float); + dst_mat.nb[1] = dst_mat.nb[0] * dst_mat.ne[0]; + dst_mat.nb[2] = dst_mat.nb[1]; + dst_mat.nb[3] = dst_mat.nb[2]; + dst_mat.data = gemm_output.get(); + dst_mat.buffer = dst->buffer; + dst_mat.extra = dst->extra; + + dpct::queue_ptr stream = ctx.stream(); + + // allocate packed arrays: A_packed (k x m), B_packed (k x n) + ggml_sycl_pool_alloc A_packed_alloc(ctx.pool()); + ggml_sycl_pool_alloc B_packed_alloc(ctx.pool()); + A_packed_alloc.alloc((size_t) knl_n_total * patch_total * sizeof(float)); + B_packed_alloc.alloc((size_t) knl_n_total * oc * sizeof(float)); + + float * A_packed = A_packed_alloc.get(); + float * B_packed = B_packed_alloc.get(); + + const int m = (int) patch_total; + const int n_gemm = (int) oc; + const int k = (int) knl_n_total; + + // Combined kernel: im2col -> pack A, and pack B simultaneously + const char * src1_base = (const char *) src1->data; + const int64_t src1_nb0 = src1->nb[0]; + const int64_t src1_nb1 = src1->nb[1]; + const int64_t src1_nb2 = src1->nb[2]; + const int64_t src1_nb3 = src1->nb[3]; + + // Compute correct strides for src0 as (knl_n_total, oc) matrix + const int64_t src0_packed_nb0 = kernel_type_size; + const int64_t src0_packed_nb1 = kernel_type_size * knl_n_total; + + const int64_t KW = knl_w; + const int64_t KH = knl_h; + const int64_t KD = knl_d; + const int64_t PW = dst->ne[0]; + const int64_t PH = dst->ne[1]; + const int64_t PD = dst->ne[2]; + + // Pack A (with inline im2col): for each (row, col) in k x m matrix + const int64_t A_total = (int64_t)k * m; + const int64_t A_block_size = 256; + const int64_t A_num_work = ((A_total + A_block_size - 1) / A_block_size) * A_block_size; + + stream->parallel_for(sycl::range<1>(A_num_work), [=](sycl::id<1> id) { + const int64_t t = id[0]; + if (t >= A_total) return; + + const int64_t row = t % k; + const int64_t col = t / k; + + // Inline im2col for this element + const int64_t k_index = row; + const int64_t patch_idx = col; + + const int64_t ic = k_index / (KD * KH * KW); + const int64_t rem = k_index - ic * (KD * KH * KW); + const int64_t kz = rem / (KH * KW); + const int64_t rem2 = rem - kz * (KH * KW); + const int64_t ky = rem2 / KW; + const int64_t kx = rem2 % KW; + + const int64_t p_in_batch = patch_idx % (PW * PH * PD); + const int64_t batch_idx = patch_idx / (PW * PH * PD); + const int64_t dst_z = p_in_batch / (PW * PH); + const int64_t dst_y = (p_in_batch % (PW * PH)) / PW; + const int64_t dst_x = p_in_batch % PW; + + const int64_t sx = dst_x * s0 + kx * d0 - p0; + const int64_t sy = dst_y * s1 + ky * d1 - p1; + const int64_t sz = dst_z * s2 + kz * d2 - p2; + + float val = 0.0f; + if (sx >= 0 && sx < src1->ne[0] && sy >= 0 && sy < src1->ne[1] && sz >= 0 && sz < src1->ne[2]) { + const int64_t channel_idx = batch_idx * c + ic; + const char * ptr = src1_base + sx * src1_nb0 + sy * src1_nb1 + sz * src1_nb2 + channel_idx * src1_nb3; + val = *(const float *) ptr; + } + A_packed[row + col * (int64_t)k] = val; + }); + + // Pack B: for each (row, col) in k x n_gemm matrix + const int64_t B_total = (int64_t)k * n_gemm; + const int64_t B_block_size = 256; + const int64_t B_num_work = ((B_total + B_block_size - 1) / B_block_size) * B_block_size; + + stream->parallel_for(sycl::range<1>(B_num_work), [=](sycl::id<1> id) { + const int64_t t = id[0]; + if (t >= B_total) return; + + const int64_t row = t % k; + const int64_t col = t / k; + const char * src_ptr = (const char *) src0->data + row * src0_packed_nb0 + col * src0_packed_nb1; + float v; + if (src0->type == GGML_TYPE_F32) { + v = *(const float *) src_ptr; + } else { + v = sycl::vec(*(const sycl::half *) src_ptr).convert()[0]; + } + B_packed[row + col * (int64_t)k] = v; + }); + + // GEMM: C = A^T * B where A is (k x m), B is (k x n), C is (m x n) + const float alpha = 1.0f; + const float beta = 0.0f; + const int lda = k; + const int ldb = k; + const int ldc = m; + + SYCL_CHECK(CHECK_TRY_ERROR(oneapi::mkl::blas::column_major::gemm( + *stream, oneapi::mkl::transpose::trans, oneapi::mkl::transpose::nontrans, + m, n_gemm, k, + dpct::get_value(&alpha, *stream), + (const float *) A_packed, lda, + (const float *) B_packed, ldb, + dpct::get_value(&beta, *stream), + (float *) dst_mat.data, ldc))); + + const float * gemm_data = (const float *) dst_mat.data; + float * dst_data = (float *) dst->data; + + ggml_sycl_conv3d_write_output(dst, gemm_data, dst_data, patch_total, oc, + dst->ne[0], dst->ne[1], dst->ne[2], stream); +} diff --git a/ggml/src/ggml-sycl/conv3d.hpp b/ggml/src/ggml-sycl/conv3d.hpp new file mode 100644 index 000000000..5852f393f --- /dev/null +++ b/ggml/src/ggml-sycl/conv3d.hpp @@ -0,0 +1,8 @@ +#ifndef GGML_SYCL_CONV3D_HPP +#define GGML_SYCL_CONV3D_HPP + +#include "common.hpp" + +void ggml_sycl_op_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV3D_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 057e0dccd..43c7e0a93 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4572,6 +4572,11 @@ static void ggml_sycl_im2col_3d(ggml_backend_sycl_context & ctx, ggml_tensor * d ggml_sycl_op_im2col_3d(ctx, dst); } +static void ggml_sycl_conv_3d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + ggml_sycl_op_conv_3d(ctx, dst); +} + static void ggml_sycl_sum(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/1); GGML_ASSERT(ggml_is_contiguous(dst->src[0])); @@ -4638,6 +4643,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_CONV_TRANSPOSE_1D: ggml_sycl_op_conv_transpose_1d(ctx, dst); break; + case GGML_OP_CONV_3D: + ggml_sycl_conv_3d(ctx, dst); + break; case GGML_OP_REPEAT: ggml_sycl_repeat(ctx, dst); break; @@ -5615,6 +5623,12 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g case GGML_OP_IM2COL_3D: case GGML_OP_UPSCALE: return true; + case GGML_OP_CONV_3D: + return op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) && + op->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous(op->src[0]) && + ggml_is_contiguous(op->src[1]); case GGML_OP_SUM: case GGML_OP_SUM_ROWS: case GGML_OP_MEAN: From 74a80dd9c052bd2d3e3e8134436ad05cc6396d1a Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Wed, 17 Jun 2026 22:21:34 +0800 Subject: [PATCH 019/292] [SYCL] add dev2dev memcpy by SYCL API (#24476) * add dev2dev memcpy by SYCL API * mv GGML_SYCL_DEV2DEV_MEMCPY to runntime table * update the detect method for p2p comm * fix the erro created during fix confilct --------- Co-authored-by: Neo Zhang --- docs/backend/SYCL.md | 2 ++ ggml/src/ggml-sycl/common.hpp | 6 ++++ ggml/src/ggml-sycl/dpct/helper.hpp | 22 +++++++++---- ggml/src/ggml-sycl/ggml-sycl.cpp | 52 ++++++++++++++++++++++-------- 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 97d4b5216..6617aa2a5 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -712,6 +712,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | Name | Value | Function | |-------------------|------------------|---------------------------------------------------------------------------------------------------------------------------| | GGML_SYCL_DEBUG | 0 (default) or 1 | Enable log function by macro: GGML_SYCL_DEBUG | +| GGML_SYCL_DEV2DEV_MEMCPY | 0 (default) or 1 | Choose the SYCL or L0 API in dev2dev memory copy.
Value:
* 0: SYCL API (default)
* 1: L0 API -- L0 API is found to lead to abnormal crash in some case. This debug flag is used to check the issue.| | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.| | GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features for Intel GPUs. (Recommended to 1 for Intel devices older than Gen 10) | | GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. | @@ -731,6 +732,7 @@ Pass these via `CXXFLAGS` or add a one-off `#define` to enable a flag on the spo | DEBUG_SYCL_POOL | Enable device memory pool logging on teardown. Useful for profiling allocations. | | DEBUG_SYCL_MALLOC | Enable verbose per-call logging of device pool alloc/free operations. | + ## Design Rule - Open to all contributors. diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 96586ea46..c87a4636e 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -62,6 +62,7 @@ extern int g_ggml_sycl_debug; extern int g_ggml_sycl_disable_optimize; extern int g_ggml_sycl_prioritize_dmmv; extern int g_ggml_sycl_enable_flash_attention; +extern int g_ggml_sycl_dev2dev_memcpy; #if defined(__clang__) && __has_builtin(__builtin_expect) @@ -126,6 +127,11 @@ enum ggml_sycl_backend_gpu_mode { SYCL_MUL_GPU_MODE }; +enum ggml_sycl_dev2dev_memcpy_mode { + DEV2DEV_MEMCPY_SYCL = 0, + DEV2DEV_MEMCPY_L0 = 1, +}; + static_assert(sizeof(sycl::half) == sizeof(ggml_fp16_t), "wrong fp16 size"); static void crash() { diff --git a/ggml/src/ggml-sycl/dpct/helper.hpp b/ggml/src/ggml-sycl/dpct/helper.hpp index 791d3cac5..664b8e969 100644 --- a/ggml/src/ggml-sycl/dpct/helper.hpp +++ b/ggml/src/ggml-sycl/dpct/helper.hpp @@ -13,14 +13,14 @@ #ifndef GGML_SYCL_DPCT_HELPER_HPP #define GGML_SYCL_DPCT_HELPER_HPP +#include +#include +#include + #include #include #include -#include - -#include "ggml.h" - #if defined(__linux__) #include #elif defined(_WIN64) @@ -43,6 +43,7 @@ #include #endif + #define DPCT_COMPATIBILITY_TEMP (900) #if defined(_MSC_VER) @@ -59,6 +60,13 @@ #define __dpct_noinline__ __attribute__((noinline)) #endif +#define DPCT_UNUSED(x) (void)(x) + +inline void _abort(const char * str) { + std::cerr << str << std::endl; + std::abort(); +} + inline std::string get_device_type_name(const sycl::device &Device) { auto DeviceType = Device.get_info(); switch (DeviceType) { @@ -1017,7 +1025,7 @@ namespace dpct if (backend == "opencl:cpu") return 4; if (backend == "opencl:acc") return 5; printf("convert_backend_index: can't handle backend=%s\n", backend.c_str()); - GGML_ABORT("fatal error"); + _abort("fatal error"); } static bool compare_backend(std::string &backend1, std::string &backend2) { return convert_backend_index(backend1) < convert_backend_index(backend2); @@ -1426,7 +1434,7 @@ namespace dpct if (!size) return sycl::event{}; return q.memcpy(to_ptr, from_ptr, size, dep_events); - GGML_UNUSED(direction); + DPCT_UNUSED(direction); } // Get actual copy range and make sure it will not exceed range. @@ -2092,7 +2100,7 @@ namespace dpct if (!size) return sycl::event{}; return q.memcpy(to_ptr, from_ptr, size, dep_events); - GGML_UNUSED(direction); + DPCT_UNUSED(direction); } // Get actual copy range and make sure it will not exceed range. diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 43c7e0a93..4c0567669 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -86,6 +86,7 @@ int g_ggml_sycl_use_async_mem_op = 0; int g_ggml_sycl_use_async_mem_op_requested = 1; int g_ggml_sycl_enable_level_zero = 0; int g_ggml_sycl_enable_flash_attention = 1; +int g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; int g_ggml_sycl_usm_system = 0; static ggml_sycl_device_info ggml_sycl_init() { @@ -272,6 +273,11 @@ static void ggml_check_sycl() try { g_ggml_sycl_enable_vmm = ggml_sycl_get_env("GGML_SYCL_ENABLE_VMM", 1); g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); + g_ggml_sycl_dev2dev_memcpy = ggml_sycl_get_env("GGML_SYCL_DEV2DEV_MEMCPY", DEV2DEV_MEMCPY_SYCL); + if (g_ggml_sycl_enable_level_zero == 0) { + g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; + } + #ifdef SYCL_FLASH_ATTN g_ggml_sycl_enable_flash_attention = ggml_sycl_get_env("GGML_SYCL_ENABLE_FLASH_ATTN", 1); #else @@ -324,8 +330,11 @@ static void ggml_check_sycl() try { #endif #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO GGML_LOG_INFO(" GGML_SYCL_ENABLE_LEVEL_ZERO: %d\n", g_ggml_sycl_enable_level_zero); + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d\n", g_ggml_sycl_dev2dev_memcpy); #else GGML_LOG_INFO(" GGML_SYCL_ENABLE_LEVEL_ZERO: Level Zero disabled by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d, enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO\n", + g_ggml_sycl_dev2dev_memcpy); #endif #if GGML_SYCL_DNNL GGML_LOG_INFO(" GGML_SYCL_DISABLE_DNN: %d\n", g_ggml_sycl_disable_dnn); @@ -598,27 +607,42 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) { static void dev2dev_memcpy(int device_dst, sycl::queue &q_dst, int device_src, sycl::queue &q_src, void *ptr_dst, const void *ptr_src, size_t size) { + #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO - // Use Level Zero direct copy for dGPU-to-dGPU transfers. - const bool l0_copy_supported = g_ggml_sycl_enable_level_zero && - ggml_sycl_is_l0_discrete_gpu(device_dst) && ggml_sycl_is_l0_discrete_gpu(device_src); - if (l0_copy_supported) { - auto ze_ctx = sycl::get_native(q_dst.get_context()); - auto ze_dev = sycl::get_native(q_dst.get_device()); - ze_command_queue_desc_t cq_desc = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, nullptr, 0, 0, - 0, ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL}; - ze_command_list_handle_t cl; - ze_result_t r = zeCommandListCreateImmediate(ze_ctx, ze_dev, &cq_desc, &cl); - if (r == ZE_RESULT_SUCCESS) { - r = zeCommandListAppendMemoryCopy(cl, ptr_dst, ptr_src, size, nullptr, 0, nullptr); - zeCommandListDestroy(cl); + if (g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_L0) { + // Use Level Zero direct copy for dGPU-to-dGPU transfers. + const bool l0_copy_supported = + ggml_sycl_is_l0_discrete_gpu(device_dst) && ggml_sycl_is_l0_discrete_gpu(device_src); + if (g_ggml_sycl_enable_level_zero && l0_copy_supported) { + auto ze_ctx = sycl::get_native(q_dst.get_context()); + auto ze_dev = sycl::get_native(q_dst.get_device()); + ze_command_queue_desc_t cq_desc = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, nullptr, 0, 0, + 0, ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS, ZE_COMMAND_QUEUE_PRIORITY_NORMAL}; + ze_command_list_handle_t cl; + ze_result_t r = zeCommandListCreateImmediate(ze_ctx, ze_dev, &cq_desc, &cl); if (r == ZE_RESULT_SUCCESS) { - return; + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by L0\n"); + r = zeCommandListAppendMemoryCopy(cl, ptr_dst, ptr_src, size, nullptr, 0, nullptr); + zeCommandListDestroy(cl); + if (r == ZE_RESULT_SUCCESS) { + return; + } } } } #endif + + if (g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_SYCL) { + if (q_dst.get_device().ext_oneapi_can_access_peer(q_src.get_device(), + sycl::ext::oneapi::peer_access::access_supported)) { + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by SYCL\n"); + SYCL_CHECK(CHECK_TRY_ERROR(q_dst.memcpy(ptr_dst, ptr_src, size).wait())); + return; + } + } + // Host-staged copy + GGML_SYCL_DEBUG("[SYCL] dev2dev memcpy by host forward\n"); char *host_buf = (char *)malloc(size); q_src.memcpy(host_buf, (const char *)ptr_src, size).wait(); q_dst.memcpy((char *)ptr_dst, host_buf, size).wait(); From 1a2dea29b9cf95416db358c5ffb02018a7cb04fb Mon Sep 17 00:00:00 2001 From: Ruixiang Wang Date: Wed, 17 Jun 2026 16:29:49 +0200 Subject: [PATCH 020/292] spec: fix segfault error on long prompts for eagle3 (#24707) --- src/llama-context.cpp | 2 +- src/llama-hparams.cpp | 4 ++++ src/llama-hparams.h | 7 +++++++ src/models/eagle3.cpp | 8 ++++---- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 168dbabd7..529bc4a5e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1382,7 +1382,7 @@ int llama_context::encode(const llama_batch & batch_inp) { const auto & hparams = model.hparams; // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim - const int64_t n_embd = hparams.n_embd_inp(); + const int64_t n_embd = hparams.n_embd_inp_enc(); const int64_t n_vocab = model.vocab.n_tokens(); // note: during encode, we always pass the full sequence starting from pos = 0 diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp index 2bf576873..9d0683d2f 100644 --- a/src/llama-hparams.cpp +++ b/src/llama-hparams.cpp @@ -104,6 +104,10 @@ uint32_t llama_hparams::n_embd_inp() const { return n_embd_inp; } +uint32_t llama_hparams::n_embd_inp_enc() const { + return n_embd_inp_enc_impl > 0 ? n_embd_inp_enc_impl : n_embd_inp(); +} + uint32_t llama_hparams::n_embd_out() const { return n_embd_out_impl > 0 ? n_embd_out_impl : n_embd; } diff --git a/src/llama-hparams.h b/src/llama-hparams.h index d045059a6..2eadeb214 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -189,6 +189,10 @@ struct llama_hparams { // input embedding dimension (0 = use n_embd) uint32_t n_embd_inp_impl = 0; + // encoder input embedding dimension (0 = use n_embd_inp()) + // e.g. the eagle3 encoder fuses target_layers * target_hidden features + uint32_t n_embd_inp_enc_impl = 0; + // output embedding dimension (0 = use n_embd) uint32_t n_embd_out_impl = 0; @@ -305,6 +309,9 @@ struct llama_hparams { // dimension of main + auxiliary input embeddings uint32_t n_embd_inp() const; + // dimension of the encoder input embeddings + uint32_t n_embd_inp_enc() const; + // dimension of output embeddings uint32_t n_embd_out() const; diff --git a/src/models/eagle3.cpp b/src/models/eagle3.cpp index 3321b3905..9d96fae59 100644 --- a/src/models/eagle3.cpp +++ b/src/models/eagle3.cpp @@ -19,7 +19,7 @@ void llama_model_eagle3::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_TARGET_HIDDEN_SIZE, n_embd_tgt); LLAMA_LOG_INFO("%s: EAGLE3 n_embd_tgt = %u (draft n_embd = %u)\n", __func__, n_embd_tgt, hparams.n_embd); - hparams.n_embd_inp_impl = (uint32_t) target_layer_ids.size() * n_embd_tgt; + hparams.n_embd_inp_enc_impl = (uint32_t) target_layer_ids.size() * n_embd_tgt; // eagle3 norm_before_residual (optional, default false) // compatible with Readhat eagle3 speculator model @@ -34,7 +34,7 @@ void llama_model_eagle3::load_arch_hparams(llama_model_loader & ml) { void llama_model_eagle3::load_arch_tensors(llama_model_loader &) { LLAMA_LOAD_LOCALS; - const int64_t n_embd_inp = hparams.n_embd_inp(); + const int64_t n_embd_inp = hparams.n_embd_inp_enc(); const int64_t n_embd_attn_input = 2 * n_embd; // Get vocab size from the d2t tensor in the GGUF file (optional - only needed if eagle3 has different vocab_size than target) @@ -109,8 +109,8 @@ ggml_tensor * llama_model_eagle3::graph::build_inp_embd_enc() const { // Input: Target model features (3 layers concatenated: low, mid, high) // Data will be provided via ubatch->embd in encode_eagle3_features() - auto inp_target = std::make_unique(hparams.n_embd_inp()); - inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32,hparams.n_embd_inp(), n_tokens); + auto inp_target = std::make_unique(hparams.n_embd_inp_enc()); + inp_target->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp_enc(), n_tokens); ggml_set_input(inp_target->embd); cur = inp_target->embd; From b4024af6c244e355cac1a0c3368c3bca623acf1b Mon Sep 17 00:00:00 2001 From: Dev-iL <6509619+Dev-iL@users.noreply.github.com> Date: Wed, 17 Jun 2026 17:30:26 +0300 Subject: [PATCH 021/292] llama : skip main_gpu validation when no devices are available (#23405) --- src/llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llama.cpp b/src/llama.cpp index a67fa8039..0de6048f2 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -249,7 +249,7 @@ static bool llama_prepare_model_devices(const llama_model_params & params, llama } // if using single GPU mode, remove all except the main GPU - if (params.split_mode == LLAMA_SPLIT_MODE_NONE) { + if (params.split_mode == LLAMA_SPLIT_MODE_NONE && !model->devices.empty()) { if (params.main_gpu < 0) { model->devices.clear(); } else { From eb05dd5fabac47205c8438f8979b5aeba3f51f34 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Wed, 17 Jun 2026 23:50:08 +0800 Subject: [PATCH 022/292] support audio to video for ltx 2.3 --- expose.h | 1 + koboldcpp.py | 9 +- otherarch/sdcpp/include/stable-diffusion.h | 1 + otherarch/sdcpp/sdtype_adapter.cpp | 49 ++++ .../sdcpp/src/model/vae/ltx_audio_vae.hpp | 264 +++++++++++++++++- otherarch/sdcpp/src/stable-diffusion.cpp | 153 +++++++++- otherarch/utils.cpp | 49 ++++ otherarch/utils.h | 3 +- 8 files changed, 519 insertions(+), 10 deletions(-) diff --git a/expose.h b/expose.h index c5f8dcf78..b6e0045cd 100644 --- a/expose.h +++ b/expose.h @@ -223,6 +223,7 @@ struct sd_generation_inputs const char * negative_prompt = nullptr; const char * init_images = ""; const char * mask = ""; + const char * audio_data = ""; const int extra_images_len = 0; const char ** extra_images = nullptr; const bool reverse_refimg = false; diff --git a/koboldcpp.py b/koboldcpp.py index 25ead4dfa..0459b5192 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -412,6 +412,7 @@ class sd_generation_inputs(ctypes.Structure): ("negative_prompt", ctypes.c_char_p), ("init_images", ctypes.c_char_p), ("mask", ctypes.c_char_p), + ("audio_data", ctypes.c_char_p), ("extra_images_len", ctypes.c_int), ("extra_images", ctypes.POINTER(ctypes.c_char_p)), ("reverse_refimg", ctypes.c_bool), @@ -1111,7 +1112,7 @@ def is_incomplete_utf8_sequence(byte_seq): #note, this will only flag INCOMPLETE def strip_base64_prefix(encoded_data): if not encoded_data: return "" - if encoded_data.startswith("data:image"): + if encoded_data.startswith("data:image") or encoded_data.startswith("data:audio"): encoded_data = encoded_data.split(',', 1)[-1] return encoded_data @@ -2814,6 +2815,11 @@ def sd_generate(genparams): extra_images_arr = genparams.get("extra_images", []) extra_images_arr = ([] if not extra_images_arr else extra_images_arr) extra_images_arr = [img for img in extra_images_arr if img not in (None, "")] + + audio_data = next((img for img in extra_images_arr if img.startswith("data:audio")), None) + extra_images_arr = [img for img in extra_images_arr if not img.startswith("data:audio")] + audio_data = strip_base64_prefix(audio_data) + extra_images_arr = extra_images_arr[:extra_images_max] lora_filenames, lora_multipliers = prepare_lora_multipliers(genparams.get("lora", [])) @@ -2840,6 +2846,7 @@ def sd_generate(genparams): inputs.negative_prompt = negative_prompt.encode("UTF-8") inputs.init_images = init_images.encode("UTF-8") inputs.mask = "".encode("UTF-8") if not mask else mask.encode("UTF-8") + inputs.audio_data = "".encode("UTF-8") if not audio_data else audio_data.encode("UTF-8") inputs.extra_images_len = len(extra_images_arr) inputs.extra_images = (ctypes.c_char_p * inputs.extra_images_len)() for n, estr in enumerate(extra_images_arr): diff --git a/otherarch/sdcpp/include/stable-diffusion.h b/otherarch/sdcpp/include/stable-diffusion.h index 1c04367b1..bfcd909cc 100644 --- a/otherarch/sdcpp/include/stable-diffusion.h +++ b/otherarch/sdcpp/include/stable-diffusion.h @@ -395,6 +395,7 @@ typedef struct { int64_t seed; int video_frames; int fps; + const sd_audio_t* input_audio; float vace_strength; sd_tiling_params_t vae_tiling_params; sd_cache_params_t cache; diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index 839f95141..4880c3ba8 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -939,6 +939,37 @@ static std::string raw_image_to_png_base64(const sd_image_t& img, std::string pa return result; } +static sd_audio_t load_audio_from_b64(const std::string& b64audio) { + sd_audio_t audio = {0, 0, 0, nullptr}; + if (b64audio.empty()) { + return audio; + } + + std::vector audio_data = kcpp_base64_decode(b64audio); + std::vector decoded_samples; + int sample_rate = 0; + int channels = 0; + if (!kcpp_decode_audio_file_from_buf(audio_data.data(), audio_data.size(), sample_rate, channels, decoded_samples)) { + printf("KCPP SD: failed to decode input audio\n"); + return audio; + } + + uint64_t sample_count = static_cast(decoded_samples.size() / static_cast(channels)); + size_t float_count = decoded_samples.size(); + float* samples = (float*)malloc(float_count * sizeof(float)); + if (samples == nullptr || sample_count == 0) { + free(samples); + return audio; + } + + std::memcpy(samples, decoded_samples.data(), float_count * sizeof(float)); + audio.sample_rate = static_cast(sample_rate); + audio.channels = static_cast(channels); + audio.sample_count = sample_count; + audio.data = samples; + return audio; +} + bool supports_reference_images(kcpp_sd::model_info info) { bool supported = (info.is_wan || info.is_ltx || info.is_qwenimg || info.is_flux2 || info.is_kontext || photomaker_enabled); @@ -955,6 +986,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) std::string img2img_data = std::string(inputs.init_images); std::string img2img_mask = std::string(inputs.mask); + std::string input_audio_data = std::string(inputs.audio_data ? inputs.audio_data : ""); std::vector extra_image_data; for(int i=0;isample_steps<=1 && sd_params->strength<=0 && is_img2img && vid_req_frames<=1 && extra_image_data.size()==0); sd_audio_t* generated_audio = nullptr; + sd_audio_t input_audio = {0, 0, 0, nullptr}; if(is_vid_model) { @@ -1302,6 +1335,13 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) vid_gen_params.video_frames = vid_req_frames; vid_gen_params.fps = vid_fps; vid_gen_params.vae_tiling_params = params.vae_tiling_params; + if (!input_audio_data.empty()) { + input_audio = load_audio_from_b64(input_audio_data); + if (input_audio.data == nullptr) { + return sd_generation.error("KCPP SD: load audio from base64 failed!"); + } + vid_gen_params.input_audio = &input_audio; + } if (wan_imgs.size() > 0) { if (wan_imgs.size() >= 2) { vid_gen_params.init_image = wan_imgs[0]; @@ -1327,6 +1367,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) << "\nFRAMES:" << vid_gen_params.video_frames << "\nCTRL_FRM:" << vid_gen_params.control_frames_size << "\nINIT_IMGS:" << wan_imgs.size() + << "\nINPUT_AUDIO:" << (vid_gen_params.input_audio ? "true" : "false") << "\n\n"; printf("%s", ss.str().c_str()); } @@ -1429,6 +1470,10 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) } if (!is_passthrough && results == NULL) { + if (input_audio.data) { + free(input_audio.data); + input_audio.data = nullptr; + } return sd_generation.error("KCPP SD generate failed!"); } @@ -1576,6 +1621,10 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) free_sd_audio(generated_audio); generated_audio = nullptr; } + if (input_audio.data) { + free(input_audio.data); + input_audio.data = nullptr; + } free(results); diff --git a/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp b/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp index 997c57a5b..7a3fb25a2 100644 --- a/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp +++ b/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp @@ -1,6 +1,7 @@ #ifndef __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__ #define __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__ +#include #include #include #include @@ -21,6 +22,10 @@ namespace LTXV { int latent_channels = 8; int latent_frequency_bins = 16; int audio_channels = 2; + bool has_encoder = false; + int encoder_channels = 128; + std::vector encoder_channel_multipliers = {1, 2, 4}; + int encoder_num_res_blocks = 2; int decoder_channels = 128; std::vector decoder_channel_multipliers = {1, 2, 4}; int decoder_num_res_blocks = 2; @@ -74,6 +79,7 @@ namespace LTXV { const TensorStorage* decoder_conv_in = require("audio_vae.decoder.conv_in.conv.weight"); const TensorStorage* decoder_conv_out = require("audio_vae.decoder.conv_out.conv.weight"); + const TensorStorage* encoder_conv_in = require("audio_vae.encoder.conv_in.conv.weight"); const TensorStorage* latent_std = require("audio_vae.per_channel_statistics.std-of-means"); const TensorStorage* vocoder_conv_pre = require("vocoder.vocoder.conv_pre.weight"); const TensorStorage* vocoder_conv_post = require("vocoder.vocoder.conv_post.weight"); @@ -98,6 +104,40 @@ namespace LTXV { return config; } + if (encoder_conv_in != nullptr) { + config.has_encoder = true; + config.audio_channels = static_cast(encoder_conv_in->ne[2]); + config.encoder_channels = static_cast(encoder_conv_in->ne[3]); + + std::vector> encoder_level_channels; + for (const auto& pair : tensor_storage_map) { + const std::string& name = pair.first; + const std::string prefix = "audio_vae.encoder.down."; + const std::string suffix = ".block.0.conv1.conv.weight"; + if (!starts_with(name, prefix) || !ends_with(name, suffix)) { + continue; + } + std::string level_str = name.substr(prefix.size(), name.size() - prefix.size() - suffix.size()); + int level = std::stoi(level_str); + encoder_level_channels.push_back({level, static_cast(pair.second.ne[3])}); + } + std::sort(encoder_level_channels.begin(), encoder_level_channels.end()); + if (!encoder_level_channels.empty()) { + config.encoder_channel_multipliers.clear(); + for (const auto& level_channel : encoder_level_channels) { + config.encoder_channel_multipliers.push_back(level_channel.second / std::max(1, config.encoder_channels)); + } + } + + int encoder_block_count = 0; + while (tensor_storage_map.find("audio_vae.encoder.down.0.block." + std::to_string(encoder_block_count) + ".conv1.conv.weight") != tensor_storage_map.end()) { + ++encoder_block_count; + } + if (encoder_block_count > 0) { + config.encoder_num_res_blocks = encoder_block_count; + } + } + std::vector> level_channels; for (const auto& pair : tensor_storage_map) { const std::string& name = pair.first; @@ -171,16 +211,89 @@ namespace LTXV { if (config.audio_channels != 2 || config.latent_channels != 8 || config.mel_bins != 64) { return config; } - LOG_DEBUG("ltx_audio_vae: sample_rate = %d, mel_bins = %d, latent_channels = %d, latent_frequency_bins = %d, has_bwe = %s", + LOG_DEBUG("ltx_audio_vae: sample_rate = %d, mel_bins = %d, latent_channels = %d, latent_frequency_bins = %d, has_encoder = %s, has_bwe = %s", config.sample_rate, config.mel_bins, config.latent_channels, config.latent_frequency_bins, + config.has_encoder ? "true" : "false", config.has_bwe ? "true" : "false"); return config; } }; + static double ltx_audio_hz_to_mel(double freq) { + constexpr double min_log_hz = 1000.0; + constexpr double min_log_mel = 15.0; + constexpr double logstep = 0.06875177742094912; // log(6.4) / 27 + constexpr double f_sp = 200.0 / 3.0; + if (freq < min_log_hz) { + return freq / f_sp; + } + return min_log_mel + std::log(freq / min_log_hz) / logstep; + } + + static double ltx_audio_mel_to_hz(double mel) { + constexpr double min_log_hz = 1000.0; + constexpr double min_log_mel = 15.0; + constexpr double logstep = 0.06875177742094912; // log(6.4) / 27 + constexpr double f_sp = 200.0 / 3.0; + if (mel < min_log_mel) { + return mel * f_sp; + } + return min_log_hz * std::exp(logstep * (mel - min_log_mel)); + } + + static sd::Tensor build_encoder_stft_basis(int n_fft) { + constexpr double kPi = 3.14159265358979323846; + const int n_freqs = n_fft / 2 + 1; + sd::Tensor basis({n_fft, 1, n_freqs * 2}); + for (int k = 0; k < n_freqs; ++k) { + for (int n = 0; n < n_fft; ++n) { + double window = 0.5 - 0.5 * std::cos(2.0 * kPi * n / static_cast(n_fft)); + double phase = 2.0 * kPi * k * n / static_cast(n_fft); + basis.index(n, 0, k) = static_cast(std::cos(phase) * window); + basis.index(n, 0, k + n_freqs) = static_cast(-std::sin(phase) * window); + } + } + return basis; + } + + static sd::Tensor build_encoder_mel_basis(int sample_rate, int n_fft, int n_mels) { + const int n_freqs = n_fft / 2 + 1; + sd::Tensor basis({n_freqs, n_mels}); + std::vector fft_freqs(n_freqs); + for (int i = 0; i < n_freqs; ++i) { + fft_freqs[i] = (static_cast(sample_rate) * 0.5) * static_cast(i) / static_cast(n_freqs - 1); + } + + std::vector mel_f(n_mels + 2); + const double min_mel = ltx_audio_hz_to_mel(0.0); + const double max_mel = ltx_audio_hz_to_mel(static_cast(sample_rate) * 0.5); + for (int i = 0; i < n_mels + 2; ++i) { + double mel = min_mel + (max_mel - min_mel) * static_cast(i) / static_cast(n_mels + 1); + mel_f[i] = ltx_audio_mel_to_hz(mel); + } + + for (int m = 0; m < n_mels; ++m) { + double lower = mel_f[m]; + double center = mel_f[m + 1]; + double upper = mel_f[m + 2]; + double enorm = 2.0 / std::max(upper - lower, 1e-12); + for (int f = 0; f < n_freqs; ++f) { + double freq = fft_freqs[f]; + double value = 0.0; + if (freq > lower && freq <= center) { + value = (freq - lower) / std::max(center - lower, 1e-12); + } else if (freq > center && freq < upper) { + value = (upper - freq) / std::max(upper - center, 1e-12); + } + basis.index(f, m) = static_cast(value * enorm); + } + } + return basis; + } + static ggml_tensor* compute_log_mel_spectrogram(GGMLRunnerContext* runner_ctx, ggml_tensor* waveform, ggml_tensor* forward_basis, @@ -478,6 +591,31 @@ namespace LTXV { } }; + struct AudioDownsample2D : public GGMLBlock { + AudioDownsample2D(int64_t channels) { + blocks["conv"] = std::make_shared(channels, + channels, + std::pair{3, 3}, + std::pair{2, 2}, + std::pair{0, 0}); + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) { + auto conv = std::dynamic_pointer_cast(blocks["conv"]); + x = ggml_ext_pad_ext(ctx->ggml_ctx, + x, + 0, + 1, + 2, + 0, + 0, + 0, + 0, + 0); + return conv->forward(ctx, x); + } + }; + struct AudioResnetBlock2D : public GGMLBlock { int64_t in_channels; int64_t out_channels; @@ -514,6 +652,86 @@ namespace LTXV { } }; + struct AudioEncoder : public GGMLBlock { + LTXAudioVAEConfig config; + + explicit AudioEncoder(const LTXAudioVAEConfig& config) + : config(config) { + int block_in = config.encoder_channels; + blocks["conv_in"] = std::make_shared(config.audio_channels, block_in, std::pair{3, 3}); + + for (int level = 0; level < static_cast(config.encoder_channel_multipliers.size()); ++level) { + int block_out = config.encoder_channels * config.encoder_channel_multipliers[level]; + for (int block_idx = 0; block_idx < config.encoder_num_res_blocks; ++block_idx) { + blocks["down." + std::to_string(level) + ".block." + std::to_string(block_idx)] = + std::make_shared(block_in, block_out); + block_in = block_out; + } + if (level != static_cast(config.encoder_channel_multipliers.size()) - 1) { + blocks["down." + std::to_string(level) + ".downsample"] = std::make_shared(block_in); + } + } + + blocks["mid.block_1"] = std::make_shared(block_in, block_in); + blocks["mid.block_2"] = std::make_shared(block_in, block_in); + blocks["norm_out"] = std::make_shared(); + blocks["conv_out"] = std::make_shared(block_in, config.latent_channels * 2, std::pair{3, 3}); + } + + ggml_tensor* normalize_latent(GGMLRunnerContext* ctx, + ggml_tensor* latent, + ggml_tensor* mean, + ggml_tensor* stddev) { + latent = ggml_ext_slice(ctx->ggml_ctx, latent, 2, 0, config.latent_channels); + latent = ggml_permute(ctx->ggml_ctx, latent, 0, 2, 1, 3); + latent = ggml_cont(ctx->ggml_ctx, latent); + latent = ggml_reshape_4d(ctx->ggml_ctx, latent, config.latent_frequency_bins * config.latent_channels, latent->ne[2], 1, latent->ne[3]); + + mean = ggml_reshape_4d(ctx->ggml_ctx, mean, mean->ne[0], 1, 1, 1); + stddev = ggml_reshape_4d(ctx->ggml_ctx, stddev, stddev->ne[0], 1, 1, 1); + latent = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, latent, mean), stddev); + + latent = ggml_reshape_4d(ctx->ggml_ctx, + latent, + config.latent_frequency_bins, + config.latent_channels, + latent->ne[1], + latent->ne[3]); + latent = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, latent, 0, 2, 1, 3)); + return latent; + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, + ggml_tensor* spectrogram, + ggml_tensor* mean, + ggml_tensor* stddev) { + auto conv_in = std::dynamic_pointer_cast(blocks["conv_in"]); + auto mid_block_1 = std::dynamic_pointer_cast(blocks["mid.block_1"]); + auto mid_block_2 = std::dynamic_pointer_cast(blocks["mid.block_2"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + auto conv_out = std::dynamic_pointer_cast(blocks["conv_out"]); + + auto x = conv_in->forward(ctx, spectrogram); + for (int level = 0; level < static_cast(config.encoder_channel_multipliers.size()); ++level) { + for (int block_idx = 0; block_idx < config.encoder_num_res_blocks; ++block_idx) { + auto block = std::dynamic_pointer_cast(blocks["down." + std::to_string(level) + ".block." + std::to_string(block_idx)]); + x = block->forward(ctx, x); + } + if (level != static_cast(config.encoder_channel_multipliers.size()) - 1) { + auto downsample = std::dynamic_pointer_cast(blocks["down." + std::to_string(level) + ".downsample"]); + x = downsample->forward(ctx, x); + } + } + + x = mid_block_1->forward(ctx, x); + x = mid_block_2->forward(ctx, x); + x = norm_out->forward(ctx, x); + x = ggml_silu_inplace(ctx->ggml_ctx, x); + x = conv_out->forward(ctx, x); + return normalize_latent(ctx, x, mean, stddev); + } + }; + struct Conv1D : public UnaryBlock { int64_t in_channels; int64_t out_channels; @@ -914,6 +1132,9 @@ namespace LTXV { explicit LTXAudioVAE(const LTXAudioVAEConfig& config) : config(config) { + if (config.has_encoder) { + blocks["audio_vae.encoder"] = std::make_shared(config); + } blocks["audio_vae.decoder"] = std::make_shared(config); blocks["vocoder.vocoder"] = std::make_shared(config); if (config.has_bwe) { @@ -993,6 +1214,18 @@ namespace LTXV { return waveform; } + + ggml_tensor* encode(GGMLRunnerContext* ctx, + ggml_tensor* waveform, + ggml_tensor* stft_basis, + ggml_tensor* mel_basis) { + GGML_ASSERT(config.has_encoder); + auto encoder = std::dynamic_pointer_cast(blocks["audio_vae.encoder"]); + auto mean = params["audio_vae.per_channel_statistics.mean-of-means"]; + auto stddev = params["audio_vae.per_channel_statistics.std-of-means"]; + auto mel = compute_log_mel_spectrogram(ctx, waveform, stft_basis, mel_basis, config.mel_hop_length); + return encoder->forward(ctx, mel, mean, stddev); + } }; struct LTXAudioVAERunner : public GGMLRunner { @@ -1000,6 +1233,8 @@ namespace LTXV { LTXAudioVAE model; std::string weight_prefix; sd::Tensor bwe_skip_filter_tensor; + sd::Tensor encoder_stft_basis_tensor; + sd::Tensor encoder_mel_basis_tensor; LTXAudioVAERunner(ggml_backend_t backend, const String2TensorStorage& tensor_storage_map, @@ -1014,6 +1249,10 @@ namespace LTXV { const int bwe_ratio = config.bwe_output_sample_rate / config.bwe_input_sample_rate; bwe_skip_filter_tensor = sd::Tensor::from_vector(build_hann_resample_filter(bwe_ratio)); } + if (config.has_encoder) { + encoder_stft_basis_tensor = build_encoder_stft_basis(config.n_fft); + encoder_mel_basis_tensor = build_encoder_mel_basis(config.sample_rate, config.n_fft, config.mel_bins); + } } void get_param_tensors(std::map& tensors) { @@ -1046,6 +1285,29 @@ namespace LTXV { return result; } + sd::Tensor encode(int n_threads, + const sd::Tensor& waveform_tensor) { + if (!config.has_encoder || waveform_tensor.empty() || + encoder_stft_basis_tensor.empty() || encoder_mel_basis_tensor.empty()) { + return {}; + } + int64_t t0 = ggml_time_ms(); + auto get_graph = [&]() -> ggml_cgraph* { + auto waveform = make_input(waveform_tensor); + auto stft_basis = make_input(encoder_stft_basis_tensor); + auto mel_basis = make_input(encoder_mel_basis_tensor); + ggml_cgraph* gf = new_graph_custom(655360); + auto runner_ctx = GGMLRunner::get_context(); + auto latent = model.encode(&runner_ctx, waveform, stft_basis, mel_basis); + ggml_build_forward_expand(gf, latent); + return gf; + }; + auto result = restore_trailing_singleton_dims(GGMLRunner::compute(get_graph, n_threads, false, false, false), 4); + int64_t t1 = ggml_time_ms(); + LOG_INFO("ltx audio vae encode completed, taking %.2fs", (t1 - t0) * 1.0f / 1000); + return result; + } + void test(const std::string& input_path) { auto z = sd::load_tensor_from_file_as_tensor(input_path); GGML_ASSERT(!z.empty()); diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index 68a4d6aae..765d59fd3 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -1342,9 +1343,6 @@ public: ignore_tensors.insert("model.diffusion_model.__32x32__"); ignore_tensors.insert("model.diffusion_model.__index_timestep_zero__"); - if (audio_vae_model) { - ignore_tensors.insert("audio_vae.encoder"); - } if (version == VERSION_OVIS_IMAGE) { ignore_tensors.insert("text_encoders.llm.vision_model."); ignore_tensors.insert("text_encoders.llm.visual_tokenizer."); @@ -3205,6 +3203,7 @@ void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) { sd_vid_gen_params->seed = -1; sd_vid_gen_params->video_frames = 6; sd_vid_gen_params->fps = 16; + sd_vid_gen_params->input_audio = nullptr; sd_vid_gen_params->moe_boundary = 0.875f; sd_vid_gen_params->vace_strength = 1.f; sd_vid_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr}; @@ -3304,6 +3303,77 @@ static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd, return audio; } +static sd_audio_t* clone_sd_audio(const sd_audio_t* src) { + if (src == nullptr || src->data == nullptr || src->sample_rate == 0 || src->channels == 0 || src->sample_count == 0) { + return nullptr; + } + + sd_audio_t* audio = (sd_audio_t*)malloc(sizeof(sd_audio_t)); + if (audio == nullptr) { + return nullptr; + } + + audio->sample_rate = src->sample_rate; + audio->channels = src->channels; + audio->sample_count = src->sample_count; + size_t sample_bytes = static_cast(src->sample_count) * static_cast(src->channels) * sizeof(float); + audio->data = (float*)malloc(sample_bytes); + if (audio->data == nullptr) { + free(audio); + return nullptr; + } + + std::memcpy(audio->data, src->data, sample_bytes); + return audio; +} + +static sd::Tensor sd_audio_to_ltx_waveform_tensor(const sd_audio_t* audio, + int target_sample_rate, + int target_channels) { + if (audio == nullptr || audio->data == nullptr || audio->sample_rate == 0 || + audio->channels == 0 || audio->sample_count == 0 || target_sample_rate <= 0 || + target_channels <= 0) { + return {}; + } + + uint64_t out_samples_u64 = (audio->sample_count * static_cast(target_sample_rate) + + static_cast(audio->sample_rate) - 1) / + static_cast(audio->sample_rate); + if (out_samples_u64 == 0 || out_samples_u64 > static_cast(std::numeric_limits::max())) { + return {}; + } + + int64_t out_samples = static_cast(out_samples_u64); + sd::Tensor waveform({out_samples, target_channels, 1, 1}); + const double src_rate = static_cast(audio->sample_rate); + const double dst_rate = static_cast(target_sample_rate); + const int src_channels = static_cast(audio->channels); + + auto src_value = [&](uint64_t sample, int channel) -> float { + int src_channel = channel; + if (src_channels == 1) { + src_channel = 0; + } else if (channel >= src_channels) { + src_channel = src_channels - 1; + } + return audio->data[static_cast(sample) * static_cast(src_channels) + static_cast(src_channel)]; + }; + + for (int64_t t = 0; t < out_samples; ++t) { + double src_pos = static_cast(t) * src_rate / dst_rate; + uint64_t i0 = static_cast(std::floor(src_pos)); + uint64_t i1 = std::min(i0 + 1, audio->sample_count - 1); + float frac = static_cast(src_pos - static_cast(i0)); + for (int ch = 0; ch < target_channels; ++ch) { + float v0 = src_value(i0, ch); + float v1 = src_value(i1, ch); + waveform.index(t, ch, 0, 0) = v0 + (v1 - v0) * frac; + } + } + + return waveform; +} + void free_sd_audio(sd_audio_t* audio) { if (audio == nullptr) { return; @@ -3872,7 +3942,8 @@ static sd::Tensor pack_ltxav_audio_and_video_latents(const sd::Tensor pack_ltxav_audio_and_video_denoise_mask(const sd::Tensor& video_mask, const sd::Tensor& video_latent, - const sd::Tensor& audio_latent) { + const sd::Tensor& audio_latent, + float audio_mask_value = 1.f) { if (video_mask.empty() || audio_latent.empty()) { return video_mask; } @@ -3915,7 +3986,7 @@ static sd::Tensor pack_ltxav_audio_and_video_denoise_mask(const sd::Tenso std::vector audio_mask_shape = video_latent.shape(); audio_mask_shape[3] = extra_ch; - auto audio_mask = sd::Tensor::ones(audio_mask_shape); + auto audio_mask = sd::full(audio_mask_shape, audio_mask_value); return sd::ops::concat(video_mask_full, audio_mask, 3); } @@ -4957,6 +5028,44 @@ static std::optional prepare_video_generation_latents(sd if (sd_version_is_ltxav(sd_ctx->sd->version)) { latents.audio_length = get_ltxav_num_audio_latents(request->frames, request->fps); latents.audio_latent = make_ltxav_empty_audio_latent(latents.audio_length); + if (sd_vid_gen_params->input_audio != nullptr && + sd_vid_gen_params->input_audio->data != nullptr && + sd_vid_gen_params->input_audio->sample_count > 0) { + if (sd_ctx->sd->audio_vae_model == nullptr || !sd_ctx->sd->audio_vae_model->config.has_encoder) { + LOG_ERROR("LTX A2V requires an audio VAE with encoder weights"); + return std::nullopt; + } + + int64_t audio_encode_start = ggml_time_ms(); + auto waveform = sd_audio_to_ltx_waveform_tensor(sd_vid_gen_params->input_audio, + sd_ctx->sd->audio_vae_model->config.sample_rate, + sd_ctx->sd->audio_vae_model->config.audio_channels); + if (waveform.empty()) { + LOG_ERROR("failed to convert source audio for LTX A2V encoding"); + return std::nullopt; + } + + auto encoded_audio_latent = sd_ctx->sd->audio_vae_model->encode(sd_ctx->sd->n_threads, waveform); + if (encoded_audio_latent.empty()) { + LOG_ERROR("LTX A2V audio latent encoding failed"); + return std::nullopt; + } + + latents.audio_latent = resize_ltxav_audio_latent(encoded_audio_latent, latents.audio_length); + if (latents.audio_latent.empty()) { + LOG_ERROR("failed to resize encoded LTX A2V audio latent"); + return std::nullopt; + } + + int64_t audio_encode_end = ggml_time_ms(); + LOG_INFO("encoded LTX A2V source audio latent %dx%dx%dx%d -> length %d, taking %.2fs", + (int)encoded_audio_latent.shape()[0], + (int)encoded_audio_latent.shape()[1], + (int)encoded_audio_latent.shape()[2], + (int)encoded_audio_latent.shape()[3], + latents.audio_length, + (audio_encode_end - audio_encode_start) * 1.0f / 1000); + } } if (sd_version_is_ltxav(sd_ctx->sd->version)) { @@ -5234,10 +5343,17 @@ static std::optional prepare_video_generation_latents(sd } if (sd_version_is_ltxav(sd_ctx->sd->version) && !latents.audio_latent.empty()) { + bool has_input_audio = sd_vid_gen_params->input_audio != nullptr && + sd_vid_gen_params->input_audio->data != nullptr && + sd_vid_gen_params->input_audio->sample_count > 0; + if (has_input_audio && latents.denoise_mask.empty()) { + latents.denoise_mask = make_ltxav_video_denoise_mask(latents.init_latent, 1.f); + } if (!latents.denoise_mask.empty()) { latents.denoise_mask = pack_ltxav_audio_and_video_denoise_mask(latents.denoise_mask, latents.init_latent, - latents.audio_latent); + latents.audio_latent, + has_input_audio ? 0.f : 1.f); } latents.init_latent = pack_ltxav_audio_and_video_latents(latents.init_latent, latents.audio_latent); } @@ -5509,8 +5625,14 @@ static bool apply_ltxv_refine_image_conditioning(sd_ctx_t* sd_ctx, } if (!audio_latent.empty()) { + bool has_input_audio = sd_vid_gen_params->input_audio != nullptr && + sd_vid_gen_params->input_audio->data != nullptr && + sd_vid_gen_params->input_audio->sample_count > 0; *latent = pack_ltxav_audio_and_video_latents(video_latent, audio_latent); - *denoise_mask = pack_ltxav_audio_and_video_denoise_mask(video_mask, video_latent, audio_latent); + *denoise_mask = pack_ltxav_audio_and_video_denoise_mask(video_mask, + video_latent, + audio_latent, + has_input_audio ? 0.f : 1.f); } else { *latent = std::move(video_latent); *denoise_mask = std::move(video_mask); @@ -5542,6 +5664,9 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, int64_t t0 = ggml_time_ms(); sd_ctx->sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params; GenerationRequest request(sd_ctx, sd_vid_gen_params); + bool has_input_audio = sd_vid_gen_params->input_audio != nullptr && + sd_vid_gen_params->input_audio->data != nullptr && + sd_vid_gen_params->input_audio->sample_count > 0; bool latent_upscale_enabled = request.hires.enabled; GenerationRequest hires_request = request; if (latent_upscale_enabled) { @@ -5759,6 +5884,17 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, &hires_video_positions)) { return false; } + if (has_input_audio && hires_denoise_mask.empty() && x_t.shape()[3] > sd_ctx->sd->get_latent_channel()) { + int latent_channels = sd_ctx->sd->get_latent_channel(); + auto video_latent = sd::ops::slice(x_t, 3, 0, latent_channels); + auto audio_latent = unpack_ltxav_audio_latent(x_t, latents.audio_length, latent_channels); + if (!audio_latent.empty()) { + hires_denoise_mask = pack_ltxav_audio_and_video_denoise_mask(make_ltxav_video_denoise_mask(video_latent, 1.f), + video_latent, + audio_latent, + 0.f); + } + } noise = sd::Tensor::randn_like(x_t, sd_ctx->sd->rng); W = hires_request.width / hires_request.vae_scale_factor; @@ -5827,6 +5963,9 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx, sd_audio_t* generated_audio = nullptr; if (sd_version_is_ltxav(sd_ctx->sd->version) && + has_input_audio) { + generated_audio = clone_sd_audio(sd_vid_gen_params->input_audio); + } else if (sd_version_is_ltxav(sd_ctx->sd->version) && latents.audio_length > 0 && sd_ctx->sd->audio_vae_model != nullptr) { if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) { diff --git a/otherarch/utils.cpp b/otherarch/utils.cpp index 108c31747..2e2170ac1 100644 --- a/otherarch/utils.cpp +++ b/otherarch/utils.cpp @@ -949,6 +949,55 @@ bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int ta return true; } +bool kcpp_decode_audio_file_from_buf(const unsigned char * buf_in, size_t len, int & sample_rate, int & channels, std::vector & pcmf32_interleaved) { + sample_rate = 0; + channels = 0; + pcmf32_interleaved.clear(); + + if (!buf_is_audio_file((const char *)buf_in, len)) + { + return false; + } + + ma_result result; + ma_decoder_config decoder_config = ma_decoder_config_init(ma_format_f32, 0, 0); + ma_decoder decoder; + + result = ma_decoder_init_memory(buf_in, len, &decoder_config, &decoder); + if (result != MA_SUCCESS) { + return false; + } + + ma_uint32 decoded_channels = 0; + ma_uint32 decoded_sample_rate = 0; + result = ma_decoder_get_data_format(&decoder, nullptr, &decoded_channels, &decoded_sample_rate, nullptr, 0); + if (result != MA_SUCCESS || decoded_channels == 0 || decoded_sample_rate == 0) { + ma_decoder_uninit(&decoder); + return false; + } + + ma_uint64 frame_count; + ma_uint64 frames_read; + result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count); + if (result != MA_SUCCESS) { + ma_decoder_uninit(&decoder); + return false; + } + + pcmf32_interleaved.resize(static_cast(frame_count) * static_cast(decoded_channels)); + result = ma_decoder_read_pcm_frames(&decoder, pcmf32_interleaved.data(), frame_count, &frames_read); + ma_decoder_uninit(&decoder); + if (result != MA_SUCCESS) { + pcmf32_interleaved.clear(); + return false; + } + + pcmf32_interleaved.resize(static_cast(frames_read) * static_cast(decoded_channels)); + sample_rate = static_cast(decoded_sample_rate); + channels = static_cast(decoded_channels); + return !pcmf32_interleaved.empty(); +} + //this version is specifically required for ace-step bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector & pcm, int & T_audio) { ma_result result; diff --git a/otherarch/utils.h b/otherarch/utils.h index 1c292db3e..ea489449a 100644 --- a/otherarch/utils.h +++ b/otherarch/utils.h @@ -61,6 +61,7 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector< std::vector split_string(const std::string& input, const std::string& separator); bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector & pcmf32_mono); +bool kcpp_decode_audio_file_from_buf(const unsigned char * buf_in, size_t len, int & sample_rate, int & channels, std::vector & pcmf32_interleaved); bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector & pcm, int & T_audio); typedef struct ggml_backend_device * ggml_backend_dev_t; @@ -112,4 +113,4 @@ struct wav_ulaw_header { std::string save_ulaw_wav8_base64(const std::vector &data, int sample_rate); std::string save_wav16_base64(const std::vector &data, int sample_rate); std::string save_stereo_wav16_base64(const std::vector & raw_audio, int T_audio, int sample_rate); -std::string save_stereo_mp3_base64(const std::vector & raw_audio,int T_audio,int sample_rate); \ No newline at end of file +std::string save_stereo_mp3_base64(const std::vector & raw_audio,int T_audio,int sample_rate); From 11cd91658b51777533da930be1d962f7eff00190 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Wed, 17 Jun 2026 23:53:06 +0800 Subject: [PATCH 023/292] minor linting --- otherarch/sdcpp/examples/common/media_io.cpp | 115 ++++++++++++++++++ otherarch/sdcpp/examples/common/media_io.h | 2 + .../sdcpp/src/model/vae/ltx_audio_vae.hpp | 14 +-- otherarch/sdcpp/src/stable-diffusion.cpp | 14 +-- 4 files changed, 131 insertions(+), 14 deletions(-) diff --git a/otherarch/sdcpp/examples/common/media_io.cpp b/otherarch/sdcpp/examples/common/media_io.cpp index 506c67f4d..f0fdf374e 100644 --- a/otherarch/sdcpp/examples/common/media_io.cpp +++ b/otherarch/sdcpp/examples/common/media_io.cpp @@ -191,6 +191,20 @@ uint32_t read_u32_le_bytes(const uint8_t* data) { (static_cast(data[3]) << 24); } +uint16_t read_u16_le_bytes(const uint8_t* p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8); +} + +int32_t read_s24_le_bytes(const uint8_t* p) { + int32_t value = static_cast(p[0]) | + (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16); + if (value & 0x00800000) { + value |= 0xff000000; + } + return value; +} + int stbi_ext_write_png_to_func(stbi_write_func* func, void* context, int x, @@ -1374,3 +1388,104 @@ bool write_wav_to_file(const std::string& path, file.write(reinterpret_cast(pcm.data()), static_cast(pcm.size() * sizeof(int16_t))); return file.good(); } + +sd_audio_t load_pcm_wav_from_file(const std::string& path) { + sd_audio_t audio = {0, 0, 0, nullptr}; + if (path.empty()) { + return audio; + } + + std::vector wav; + if (!read_binary_file_bytes(path.c_str(), wav)) { + LOG_ERROR("load WAV from '%s' failed", path.c_str()); + return audio; + } + if (wav.size() < 44 || std::memcmp(wav.data(), "RIFF", 4) != 0 || std::memcmp(wav.data() + 8, "WAVE", 4) != 0) { + LOG_ERROR("input audio file '%s' is not a RIFF/WAVE file", path.c_str()); + return audio; + } + + uint16_t format = 0; + uint16_t channels = 0; + uint32_t sample_rate = 0; + uint16_t bits_per_sample = 0; + const uint8_t* data = nullptr; + uint32_t data_size = 0; + + size_t pos = 12; + while (pos + 8 <= wav.size()) { + const uint8_t* chunk = wav.data() + pos; + uint32_t chunk_size = read_u32_le_bytes(chunk + 4); + size_t chunk_data = pos + 8; + if (chunk_data + chunk_size > wav.size()) { + break; + } + + if (std::memcmp(chunk, "fmt ", 4) == 0 && chunk_size >= 16) { + format = read_u16_le_bytes(wav.data() + chunk_data); + channels = read_u16_le_bytes(wav.data() + chunk_data + 2); + sample_rate = read_u32_le_bytes(wav.data() + chunk_data + 4); + bits_per_sample = read_u16_le_bytes(wav.data() + chunk_data + 14); + } else if (std::memcmp(chunk, "data", 4) == 0) { + data = wav.data() + chunk_data; + data_size = chunk_size; + } + pos = chunk_data + chunk_size + (chunk_size & 1); + } + + if (data == nullptr || data_size == 0 || channels == 0 || sample_rate == 0) { + LOG_ERROR("input WAV '%s' is missing fmt/data chunks", path.c_str()); + return audio; + } + if (format != 1 && format != 3) { + LOG_ERROR("unsupported WAV format %u in '%s', only PCM and float WAV are supported", + static_cast(format), + path.c_str()); + return audio; + } + + uint16_t bytes_per_sample = static_cast((bits_per_sample + 7) / 8); + uint32_t frame_bytes = static_cast(bytes_per_sample) * channels; + if (bytes_per_sample == 0 || frame_bytes == 0 || data_size < frame_bytes) { + LOG_ERROR("invalid WAV sample format in '%s'", path.c_str()); + return audio; + } + + uint64_t sample_count = data_size / frame_bytes; + size_t float_count = static_cast(sample_count) * channels; + float* samples = (float*)malloc(float_count * sizeof(float)); + if (samples == nullptr) { + return audio; + } + + for (uint64_t i = 0; i < sample_count; ++i) { + for (uint16_t ch = 0; ch < channels; ++ch) { + const uint8_t* src = data + i * frame_bytes + ch * bytes_per_sample; + float sample = 0.f; + if (format == 3 && bits_per_sample == 32) { + std::memcpy(&sample, src, sizeof(float)); + } else if (format == 1 && bits_per_sample == 8) { + sample = (static_cast(src[0]) - 128) / 128.f; + } else if (format == 1 && bits_per_sample == 16) { + sample = static_cast(read_u16_le_bytes(src)) / 32768.f; + } else if (format == 1 && bits_per_sample == 24) { + sample = read_s24_le_bytes(src) / 8388608.f; + } else if (format == 1 && bits_per_sample == 32) { + sample = static_cast(read_u32_le_bytes(src)) / 2147483648.f; + } else { + LOG_ERROR("unsupported WAV bit depth %u in '%s'", + static_cast(bits_per_sample), + path.c_str()); + free(samples); + return audio; + } + samples[i * channels + ch] = std::clamp(sample, -1.0f, 1.0f); + } + } + + audio.sample_rate = sample_rate; + audio.channels = channels; + audio.sample_count = sample_count; + audio.data = samples; + return audio; +} diff --git a/otherarch/sdcpp/examples/common/media_io.h b/otherarch/sdcpp/examples/common/media_io.h index 0f7679d7f..df2fd019b 100644 --- a/otherarch/sdcpp/examples/common/media_io.h +++ b/otherarch/sdcpp/examples/common/media_io.h @@ -110,4 +110,6 @@ bool write_wav_to_file(const std::string& path, uint32_t channels, uint32_t sample_rate); +sd_audio_t load_pcm_wav_from_file(const std::string& path); + #endif // __MEDIA_IO_H__ diff --git a/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp b/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp index 7a3fb25a2..49847445d 100644 --- a/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp +++ b/otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp @@ -250,9 +250,9 @@ namespace LTXV { sd::Tensor basis({n_fft, 1, n_freqs * 2}); for (int k = 0; k < n_freqs; ++k) { for (int n = 0; n < n_fft; ++n) { - double window = 0.5 - 0.5 * std::cos(2.0 * kPi * n / static_cast(n_fft)); - double phase = 2.0 * kPi * k * n / static_cast(n_fft); - basis.index(n, 0, k) = static_cast(std::cos(phase) * window); + double window = 0.5 - 0.5 * std::cos(2.0 * kPi * n / static_cast(n_fft)); + double phase = 2.0 * kPi * k * n / static_cast(n_fft); + basis.index(n, 0, k) = static_cast(std::cos(phase) * window); basis.index(n, 0, k + n_freqs) = static_cast(-std::sin(phase) * window); } } @@ -276,12 +276,12 @@ namespace LTXV { } for (int m = 0; m < n_mels; ++m) { - double lower = mel_f[m]; + double lower = mel_f[m]; double center = mel_f[m + 1]; - double upper = mel_f[m + 2]; - double enorm = 2.0 / std::max(upper - lower, 1e-12); + double upper = mel_f[m + 2]; + double enorm = 2.0 / std::max(upper - lower, 1e-12); for (int f = 0; f < n_freqs; ++f) { - double freq = fft_freqs[f]; + double freq = fft_freqs[f]; double value = 0.0; if (freq > lower && freq <= center) { value = (freq - lower) / std::max(center - lower, 1e-12); diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index 765d59fd3..e2a80a035 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -3345,8 +3345,8 @@ static sd::Tensor sd_audio_to_ltx_waveform_tensor(const sd_audio_t* audio int64_t out_samples = static_cast(out_samples_u64); sd::Tensor waveform({out_samples, target_channels, 1, 1}); - const double src_rate = static_cast(audio->sample_rate); - const double dst_rate = static_cast(target_sample_rate); + const double src_rate = static_cast(audio->sample_rate); + const double dst_rate = static_cast(target_sample_rate); const int src_channels = static_cast(audio->channels); auto src_value = [&](uint64_t sample, int channel) -> float { @@ -3365,8 +3365,8 @@ static sd::Tensor sd_audio_to_ltx_waveform_tensor(const sd_audio_t* audio uint64_t i1 = std::min(i0 + 1, audio->sample_count - 1); float frac = static_cast(src_pos - static_cast(i0)); for (int ch = 0; ch < target_channels; ++ch) { - float v0 = src_value(i0, ch); - float v1 = src_value(i1, ch); + float v0 = src_value(i0, ch); + float v1 = src_value(i1, ch); waveform.index(t, ch, 0, 0) = v0 + (v1 - v0) * frac; } } @@ -5037,9 +5037,9 @@ static std::optional prepare_video_generation_latents(sd } int64_t audio_encode_start = ggml_time_ms(); - auto waveform = sd_audio_to_ltx_waveform_tensor(sd_vid_gen_params->input_audio, - sd_ctx->sd->audio_vae_model->config.sample_rate, - sd_ctx->sd->audio_vae_model->config.audio_channels); + auto waveform = sd_audio_to_ltx_waveform_tensor(sd_vid_gen_params->input_audio, + sd_ctx->sd->audio_vae_model->config.sample_rate, + sd_ctx->sd->audio_vae_model->config.audio_channels); if (waveform.empty()) { LOG_ERROR("failed to convert source audio for LTX A2V encoding"); return std::nullopt; From 4b4d13ae721e5eb79b749ca2c6feefd157f90ed7 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 17 Jun 2026 18:04:58 +0200 Subject: [PATCH 024/292] server: (router) add model management API (#23976) * wip * server: (router) add SSE realtime updates API * nits * wip * add download API * add download api * update docs * add delete endpoint * fix std::terminate * fix crash * fix 2 * add tests * nits --- common/download.cpp | 84 +++++ common/download.h | 7 + common/hf-cache.cpp | 15 + common/hf-cache.h | 3 + tools/server/README-dev.md | 18 + tools/server/README.md | 115 +++++++ tools/server/server-http.cpp | 17 + tools/server/server-http.h | 1 + tools/server/server-models.cpp | 433 +++++++++++++++++++++++-- tools/server/server-models.h | 89 +++-- tools/server/server-queue.cpp | 11 + tools/server/server-queue.h | 6 +- tools/server/server-task.h | 12 + tools/server/server.cpp | 8 + tools/server/tests/unit/test_router.py | 96 ++++++ tools/server/tests/utils.py | 3 + 16 files changed, 855 insertions(+), 63 deletions(-) diff --git a/common/download.cpp b/common/download.cpp index 40f6eb780..c3c8ff49b 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -997,3 +997,87 @@ std::vector common_list_cached_models() { return result; } + +bool common_download_remove(const std::string & hf_repo_with_tag) { + namespace fs = std::filesystem; + + auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag); + + if (tag.empty()) { + return hf_cache::remove_cached_repo(repo_id); + } + + std::string tag_upper = tag; + for (char & c : tag_upper) { + c = (char) std::toupper((unsigned char) c); + } + + auto files = hf_cache::get_cached_files(repo_id); + if (files.empty()) { + return false; + } + + // collect snapshot entries whose tag matches + std::vector to_remove; + for (const auto & f : files) { + auto split = get_gguf_split_info(f.path); + if (split.tag == tag_upper) { + to_remove.emplace_back(f.local_path); + } + } + + if (to_remove.empty()) { + return false; + } + + // resolve blob paths from symlinks before deleting snapshot entries + std::vector blobs_to_check; + for (const auto & p : to_remove) { + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + blobs_to_check.push_back((p.parent_path() / target).lexically_normal()); + } + } + } + + // remove snapshot entries + for (const auto & p : to_remove) { + std::error_code ec; + fs::remove(p, ec); + if (ec) { + LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str()); + } + } + + if (blobs_to_check.empty()) { + return true; + } + + // collect blobs still referenced by remaining snapshot entries + std::unordered_set still_referenced; + for (const auto & f : hf_cache::get_cached_files(repo_id)) { + fs::path p(f.local_path); + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + still_referenced.insert((p.parent_path() / target).lexically_normal().string()); + } + } + } + + // remove orphaned blobs + for (const auto & blob : blobs_to_check) { + if (still_referenced.find(blob.string()) == still_referenced.end()) { + std::error_code ec; + fs::remove(blob, ec); + if (ec) { + LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str()); + } + } + } + + return true; +} diff --git a/common/download.h b/common/download.h index ebeedd605..237179764 100644 --- a/common/download.h +++ b/common/download.h @@ -115,3 +115,10 @@ int common_download_file_single(const std::string & url, // resolve and download model from Docker registry // return local path to downloaded model file std::string common_docker_resolve_model(const std::string & docker); + +// Remove a cached model from disk +// input format: "user/model" or "user/model:tag" +// - if tag is omitted, removes the entire repo cache directory +// - if tag is present, removes only files matching that tag (and orphaned blobs) +// returns true if anything was removed +bool common_download_remove(const std::string & hf_repo_with_tag); diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index ba7417a12..f1dacaa47 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -495,4 +495,19 @@ std::string finalize_file(const hf_file & file) { return file.final_path; } +bool remove_cached_repo(const std::string & repo_id) { + if (!is_valid_repo_id(repo_id)) { + LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str()); + return false; + } + fs::path repo_path = get_repo_path(repo_id); + std::error_code ec; + auto removed = fs::remove_all(repo_path, ec); + if (ec) { + LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str()); + return false; + } + return removed > 0; +} + } // namespace hf_cache diff --git a/common/hf-cache.h b/common/hf-cache.h index 23fa0adb7..42c9c6ce3 100644 --- a/common/hf-cache.h +++ b/common/hf-cache.h @@ -29,4 +29,7 @@ hf_files get_cached_files(const std::string & repo_id = {}); // Create snapshot path (link or move/copy) and return it std::string finalize_file(const hf_file & file); +// Remove the entire cached directory for a repo, returns true if removed +bool remove_cached_repo(const std::string & repo_id); + } // namespace hf_cache diff --git a/tools/server/README-dev.md b/tools/server/README-dev.md index 0ff334724..4c4103123 100644 --- a/tools/server/README-dev.md +++ b/tools/server/README-dev.md @@ -180,6 +180,24 @@ That requires `JSON.stringify` when formatted to message content: } ``` +### Model management API (router mode) + +Model management API was added via PR [#23976](https://github.com/ggml-org/llama.cpp/pull/23976) + +The main goal of this API is to allow downloading models and/or removing models from the web UI. It relies on the model cache infrastructure under the hood to manage the list of models dynamically. + +Instead of building everything from the ground up (like what most AI agents will do when you ask them to implement a similar feature), we built on top of existing, already well-engineered components inside the codebase: +- Model cache infrastructure as mentioned above (`common/download.h`) +- Server response queue (`server-queue.h`). We use this feature to broadcast events to SSE clients. +- Server router thread management (`server-models.h`). We re-use the same thread model that is used for managing subprocess life cycle, except that we don't create a new subprocess, but launch the download right inside the thread. + +The flow for downloading a new model: +- POST request comes in --> `post_router_models` --> validation +- `server_models::download()` is called + - Sets up a new thread `inst.th` and runs the download inside +- If a stop request comes in, set `stop_download` to `true` +- Otherwise, upon completion, we call `load_models()` to refresh the list of models + ### Notable Related PRs - Initial server implementation: https://github.com/ggml-org/llama.cpp/pull/1443 diff --git a/tools/server/README.md b/tools/server/README.md index b41491059..88a507e2c 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1778,6 +1778,20 @@ The `status` object can be: } ``` +Note: for "downloading" state, there can be multiple files be downloading in parallel + +```json +"status": { + "value": "downloading", + "progress": { + "https://...model.gguf": { + "done": 195963406, + "total": 219307424 + } + } +} +``` + ### POST `/models/load`: Load a model Load a model @@ -1820,6 +1834,107 @@ Response: } ``` +### GET `/models/sse`: Real-time events + +Example events: + +```js +{ + "model": "...", + "event": "model_status", + "data": { + "status": "loading" + } +} + +{ + "model": "...", + "event": "download_progress", + "data": { + // note: there can be multiple files being downloaded in parallel + "https://...model.gguf": { + "done": 195963406, + "total": 219307424 + } + } +} + +{ + "model": "...", + "event": "download_finished", + "data": { + "status": "loading" + } +} + +{ + "model": "...", + "event": "model_remove" +} + +// special event: reload of the list of all models +{ + "model": "*", + "event": "models_reload" +} +``` + +### POST `/models`: Download new model + +Trigger a new download (non-blocking), the progress can be tracked via SSE endpoint `/models/sse` + +To cancel model downloading, send an event to `/models/unload` + +Download procedure: +- Send POST request to `/models` +- Subscribe to `/models/sse` for updates +- On downloading completed, you will receive either `download_finished` or `download_failed` event +- Call GET `/models` to trigger model list update. If the download success, you should see the new model in the list + +Payload: + +```json +{ + "model": "ggml-org/gemma-3-4b-it-GGUF:Q4_K_M", +} +``` + +Response (download is started in the background): + +```json +{ + "success": true +} +``` + +Response (error, cannot start the download): + +```json +{ + "error": { + "code": 400, + "message": "model validation failed, unable to download", + "type": "invalid_request_error" + } +} +``` + +### DELETE `/models`: Delete a model from cache + +IMPORTANT: only model stored in cache can be deleted. You cannot delete models in a preset. + +Model name must be passed via query param: `?model={name}` + +If delete success, it will send an SSE event of type `model_remove` + +Response: + +```json +{ + "success": true +} +``` + ## API errors `llama-server` returns errors in the same format as OAI: https://github.com/openai/openai-openapi diff --git a/tools/server/server-http.cpp b/tools/server/server-http.cpp index 3c58fcfec..5defee1f5 100644 --- a/tools/server/server-http.cpp +++ b/tools/server/server-http.cpp @@ -588,6 +588,23 @@ void server_http_context::post(const std::string & path, const server_http_conte }); } +void server_http_context::del(const std::string & path, const server_http_context::handler_t & handler) const { + handlers.emplace(path, handler); + pimpl->srv->Delete(path_prefix + path, [handler](const httplib::Request & req, httplib::Response & res) { + server_http_req_ptr request = std::make_unique(server_http_req{ + get_params(req), + get_headers(req), + req.path, + build_query_string(req), + req.body, + {}, + req.is_connection_closed + }); + server_http_res_ptr response = handler(*request); + process_handler_response(std::move(request), response, res); + }); +} + // // Vertex AI Prediction protocol (AIP_PREDICT_ROUTE) // https://cloud.google.com/vertex-ai/docs/predictions/custom-container-requirements diff --git a/tools/server/server-http.h b/tools/server/server-http.h index 25c7f1062..6b4a4b87a 100644 --- a/tools/server/server-http.h +++ b/tools/server/server-http.h @@ -86,6 +86,7 @@ struct server_http_context { void get(const std::string & path, const handler_t & handler) const; void post(const std::string & path, const handler_t & handler) const; + void del(const std::string & path, const handler_t & handler) const; // Register the Google Cloud Platform (Vertex AI) compat (AIP_PREDICT_ROUTE env var, or /predict) // Must be called AFTER all other API routes are registered diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 49b0e423f..ff9a0df12 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -51,6 +52,21 @@ extern char **environ; // ref: https://github.com/ggml-org/llama.cpp/issues/17862 #define CHILD_ADDR "127.0.0.1" +struct server_subproc { + std::optional sproc; // empty while in DOWNLOADING state + std::atomic stop_download{false}; // flag to signal download cancellation + + subprocess_s & get() { + GGML_ASSERT(sproc.has_value() && "subprocess not initialized"); + return sproc.value(); + } + + bool is_alive() { + return sproc.has_value() && subprocess_alive(&sproc.value()); + } +}; + + static std::filesystem::path get_server_exec_path() { #if defined(_WIN32) wchar_t buf[32768] = { 0 }; // Large buffer to handle long paths @@ -272,12 +288,25 @@ void server_models::add_model(server_model_meta && meta) { meta.update_caps(); std::string name = meta.name; mapping[name] = instance_t{ - /* subproc */ std::make_shared(), + /* subproc */ std::make_shared(), /* th */ std::thread(), /* meta */ std::move(meta) }; } +void server_models::notify_sse(const std::string & event, const std::string & model_id, const json & data) { + std::unique_ptr result = std::make_unique(); + result->data = { + {"model", model_id}, + {"event", event}, + }; + if (!data.is_null()) { + result->data["data"] = data; + } + SRV_DBG("notifying SSE clients about event '%s' for model '%s': %s\n", event.c_str(), model_id.c_str(), safe_json_to_str(result->data).c_str()); + sse.broadcast(std::move(result)); +} + void server_models::load_models() { // Phase 1: load presets from all sources — pure I/O, no lock needed // 1. cached models @@ -304,19 +333,27 @@ void server_models::load_models() { // note: if a model exists in both cached and local, local takes precedence common_presets final_presets; - for (const auto & [name, preset] : cached_models) final_presets[name] = preset; - for (const auto & [name, preset] : local_models) final_presets[name] = preset; + std::unordered_map source_map; + for (const auto & [name, preset] : cached_models) { + final_presets[name] = preset; + source_map[name] = SERVER_MODEL_SOURCE_CACHE; + } + for (const auto & [name, preset] : local_models) { + final_presets[name] = preset; + source_map[name] = SERVER_MODEL_SOURCE_MODELS_DIR; + } for (const auto & [name, custom] : custom_presets) { if (final_presets.find(name) != final_presets.end()) { final_presets[name].merge(custom); } else { final_presets[name] = custom; } + source_map[name] = SERVER_MODEL_SOURCE_PRESET; } - // server base preset from CLI args takes highest precedence - for (auto & [name, preset] : final_presets) { - preset.merge(base_preset); - } + + auto get_source = [&](const std::string & name) { + return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET; + }; // Helpers that read `mapping` — must be called while holding the lock. std::unordered_set custom_names; @@ -366,12 +403,15 @@ void server_models::load_models() { // (unload, load) or when joining threads (the monitoring thread calls update_status // which locks the mutex, so joining while holding it would deadlock). std::unique_lock lk(mutex); + + need_reload = false; bool is_first_load = mapping.empty(); if (is_first_load) { // FIRST LOAD: add all models, then unlock for autoloading for (const auto & [name, preset] : final_presets) { server_model_meta meta{ + /* source */ get_source(name), /* preset */ preset, /* name */ name, /* aliases */ {}, @@ -384,7 +424,7 @@ void server_models::load_models() { /* exit_code */ 0, /* stop_timeout */ DEFAULT_STOP_TIMEOUT, /* multimodal */ mtmd_caps{false, false}, - /* need_download */ false, + // /* need_download */ false, }; add_model(std::move(meta)); } @@ -453,6 +493,9 @@ void server_models::load_models() { } } for (auto & [name, inst] : mapping) { + if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + continue; // downloading models are not from config sources, leave them alone + } if (final_presets.find(name) == final_presets.end() && !inst.meta.is_running() && inst.th.joinable()) { threads_to_join.push_back(std::move(inst.th)); } @@ -465,7 +508,15 @@ void server_models::load_models() { // erase models no longer in any source for (auto it = mapping.begin(); it != mapping.end(); ) { - if (final_presets.find(it->first) == final_presets.end()) { + if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + ++it; // download thread is still busy, skip + } else if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADED) { + // download finished, safe to erase + if (it->second.th.joinable()) { + it->second.th.join(); + } + it = mapping.erase(it); + } else if (final_presets.find(it->first) == final_presets.end()) { SRV_INF("(reload) removing model name=%s (no longer in source)\n", it->first.c_str()); GGML_ASSERT(!it->second.th.joinable()); // must have been joined above it = mapping.erase(it); @@ -526,6 +577,7 @@ void server_models::load_models() { for (const auto & [name, preset] : final_presets) { if (mapping.find(name) == mapping.end()) { server_model_meta meta{ + /* source */ get_source(name), /* preset */ preset, /* name */ name, /* aliases */ {}, @@ -538,7 +590,7 @@ void server_models::load_models() { /* exit_code */ 0, /* stop_timeout */ DEFAULT_STOP_TIMEOUT, /* multimodal */ mtmd_caps{false, false}, - /* need_download */ false, + // /* need_download */ false, }; add_model(std::move(meta)); newly_added.push_back(name); @@ -571,6 +623,8 @@ void server_models::load_models() { SRV_INF("(reload) loading new model %s\n", name.c_str()); load(name); } + + notify_sse("models_reload", "*"); } } @@ -597,7 +651,13 @@ bool server_models::has_model(const std::string & name) { } std::optional server_models::get_meta(const std::string & name) { - std::lock_guard lk(mutex); + std::unique_lock lk(mutex); + if (need_reload) { + lk.unlock(); + load_models(); + lk.lock(); + } + auto it = mapping.find(name); if (it != mapping.end()) { return it->second.meta; @@ -683,7 +743,13 @@ static std::vector to_char_ptr_array(const std::vector & ve } std::vector server_models::get_all_meta() { - std::lock_guard lk(mutex); + std::unique_lock lk(mutex); + if (need_reload) { + lk.unlock(); + load_models(); + lk.lock(); + } + std::vector result; result.reserve(mapping.size()); for (const auto & [name, inst] : mapping) { @@ -770,7 +836,7 @@ void server_models::load(const std::string & name) { throw std::runtime_error("failed to get a port number"); } - inst.subproc = std::make_shared(); + inst.subproc = std::make_shared(); { SRV_INF("spawning server instance with name=%s on port %d\n", inst.meta.name.c_str(), inst.meta.port); @@ -792,19 +858,20 @@ void server_models::load(const std::string & name) { // TODO @ngxson : maybe separate stdout and stderr in the future // so that we can use stdout for commands and stderr for logging int options = subprocess_option_no_window | subprocess_option_combined_stdout_stderr; - int result = subprocess_create_ex(argv.data(), options, envp.data(), inst.subproc.get()); + inst.subproc->sproc.emplace(); + int result = subprocess_create_ex(argv.data(), options, envp.data(), &inst.subproc->get()); if (result != 0) { throw std::runtime_error("failed to spawn server instance"); } - inst.stdin_file = subprocess_stdin(inst.subproc.get()); + inst.stdin_file = subprocess_stdin(&inst.subproc->get()); } // start a thread to manage the child process // captured variables are guaranteed to be destroyed only after the thread is joined inst.th = std::thread([this, name, child_proc = inst.subproc, port = inst.meta.port, stop_timeout = inst.meta.stop_timeout]() { - FILE * stdin_file = subprocess_stdin(child_proc.get()); - FILE * stdout_file = subprocess_stdout(child_proc.get()); // combined stdout/stderr + FILE * stdin_file = subprocess_stdin(&child_proc->get()); + FILE * stdout_file = subprocess_stdout(&child_proc->get()); // combined stdout/stderr std::thread log_thread([&]() { // read stdout/stderr and forward to main server log @@ -834,14 +901,14 @@ void server_models::load(const std::string & name) { return this->stopping_models.find(name) != this->stopping_models.end(); }; auto should_wake = [&]() { - return is_stopping() || !subprocess_alive(child_proc.get()); + return is_stopping() || !child_proc->is_alive(); }; { std::unique_lock lk(this->mutex); this->cv_stop.wait(lk, should_wake); } // child may have already exited (e.g. crashed) — skip shutdown sequence - if (!subprocess_alive(child_proc.get())) { + if (!child_proc->is_alive()) { return; } SRV_INF("stopping model instance name=%s\n", name.c_str()); @@ -859,7 +926,7 @@ void server_models::load(const std::string & name) { if (elapsed >= stop_timeout * 1000) { // timeout, force kill SRV_WRN("force-killing model instance name=%s after %d seconds timeout\n", name.c_str(), stop_timeout); - subprocess_terminate(child_proc.get()); + subprocess_terminate(&child_proc->get()); return; } this->cv_stop.wait_for(lk, std::chrono::seconds(1)); @@ -884,8 +951,8 @@ void server_models::load(const std::string & name) { // get the exit code int exit_code = 0; - subprocess_join(child_proc.get(), &exit_code); - subprocess_destroy(child_proc.get()); + subprocess_join(&child_proc->get(), &exit_code); + subprocess_destroy(&child_proc->get()); // update status and exit code this->update_status(name, SERVER_MODEL_STATUS_UNLOADED, exit_code); @@ -896,30 +963,118 @@ void server_models::load(const std::string & name) { { auto & old_instance = mapping[name]; // old process should have exited already, but just in case, we clean it up here - if (subprocess_alive(old_instance.subproc.get())) { + if (old_instance.subproc->is_alive()) { SRV_WRN("old process for model name=%s is still alive, this is unexpected\n", name.c_str()); - subprocess_terminate(old_instance.subproc.get()); // force kill + subprocess_terminate(&old_instance.subproc->get()); // force kill } if (old_instance.th.joinable()) { old_instance.th.join(); } } + notify_sse("model_status", name, { + {"status", server_model_status_to_string(inst.meta.status)}, + }); + mapping[name] = std::move(inst); cv.notify_all(); } +// callback for model downloading functionality +struct server_models_download_res : public common_download_callback { + common_params_model model; + common_download_opts opts; + + std::function should_stop; + std::function on_progress; + + bool is_ok = false; + + bool run() { + try { + common_download_model(model, opts); + is_ok = true; + } catch (const std::exception & e) { + SRV_ERR("download failed for model name=%s: %s\n", model.name.c_str(), e.what()); + is_ok = false; + } + return is_ok; + } + void on_start(const common_download_progress & p) override { + on_progress(p); + } + void on_update(const common_download_progress & p) override { + on_progress(p); + } + void on_done(const common_download_progress &, bool ok) override { + is_ok = ok; + } + bool is_cancelled() const override { + return should_stop(); + } +}; + +void server_models::download(common_params_model && model, common_download_opts && opts) { + std::string name = model.name; + GGML_ASSERT(name == model.hf_repo); + + std::unique_lock lk(mutex); + if (mapping.find(name) != mapping.end()) { + throw std::runtime_error("model name=" + name + " already exists"); + } + + instance_t inst; + inst.meta.name = name; + inst.meta.status = SERVER_MODEL_STATUS_DOWNLOADING; + inst.subproc = std::make_shared(); + + auto dl = std::make_unique(); + dl->model = model; // copy + dl->opts = opts; // copy + + dl->should_stop = [sp = inst.subproc]() { + return sp->stop_download.load(std::memory_order_relaxed); + }; + + dl->on_progress = [this, name](const common_download_progress & p) { + update_download_progress(name, p, false); + }; + + inst.th = std::thread([this, dl = std::move(dl)]() { + dl->opts.callback = dl.get(); + bool ok = dl->run(); + SRV_INF("download finished for model name=%s with status=%s\n", + dl->model.name.c_str(), ok ? "success" : "failure"); + update_download_progress(dl->model.name, {}, true, ok); + // need_reload is set inside update_download_progress under the mutex; + // the next load_models() call will clean up this instance + }); + + mapping[name] = std::move(inst); + notify_sse("status_update", name, { + {"status", server_model_status_to_string(SERVER_MODEL_STATUS_DOWNLOADING)}, + }); + cv.notify_all(); +} + void server_models::unload(const std::string & name) { - std::lock_guard lk(mutex); + std::unique_lock lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { - if (it->second.meta.is_running()) { + if (it->second.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + SRV_INF("cancelling download for model name=%s\n", name.c_str()); + it->second.subproc->stop_download.store(true, std::memory_order_relaxed); + // for convenience, we wait the status change here + wait(lk, name, [](const server_model_meta & new_meta) { + return new_meta.status != SERVER_MODEL_STATUS_DOWNLOADING; + }); + } else if (it->second.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); stopping_models.insert(name); if (it->second.meta.status == SERVER_MODEL_STATUS_LOADING) { // special case: if model is in loading state, unloading means force-killing it SRV_WRN("model name=%s is still loading, force-killing\n", name.c_str()); - subprocess_terminate(it->second.subproc.get()); + subprocess_terminate(&it->second.subproc->get()); } cv_stop.notify_all(); // status change will be handled by the managing thread @@ -934,7 +1089,10 @@ void server_models::unload_all() { { std::lock_guard lk(mutex); for (auto & [name, inst] : mapping) { - if (inst.meta.is_running()) { + if (inst.meta.status == SERVER_MODEL_STATUS_DOWNLOADING) { + SRV_INF("cancelling download for model name=%s\n", name.c_str()); + inst.subproc->stop_download.store(true, std::memory_order_relaxed); + } else if (inst.meta.is_running()) { SRV_INF("stopping model instance name=%s\n", name.c_str()); stopping_models.insert(name); cv_stop.notify_all(); @@ -959,6 +1117,17 @@ void server_models::update_status(const std::string & name, server_model_status meta.status = status; meta.exit_code = exit_code; } + // broadcast status change to SSE + { + json data = { + {"status", server_model_status_to_string(status)}, + }; + if (status == SERVER_MODEL_STATUS_UNLOADED) { + data["exit_code"] = exit_code; + } + // note: notify_sse doesn't acquire the lock, so no deadlock here + notify_sse("status_change", name, data); + } cv.notify_all(); } @@ -985,12 +1154,82 @@ void server_models::update_loaded_info(const std::string & name, std::string & r cv.notify_all(); } -void server_models::wait_until_loading_finished(const std::string & name) { - std::unique_lock lk(mutex); - cv.wait(lk, [this, &name]() { +void server_models::update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok) { + json curr; + { + std::lock_guard lk(mutex); auto it = mapping.find(name); if (it != mapping.end()) { - return it->second.meta.status != SERVER_MODEL_STATUS_LOADING; + if (done) { + // mark the instance to be erased on next load_models() call + it->second.meta.status = SERVER_MODEL_STATUS_DOWNLOADED; + need_reload = true; + } else { + json & info = it->second.meta.loaded_info; + if (!info.contains("progress")) { + info["progress"] = json{}; + } + info["progress"][progress.url] = { + {"done", progress.downloaded}, + {"total", progress.total}, + }; + curr = it->second.meta.loaded_info; // copy + } + } + } + if (done) { + cv.notify_all(); // notify in case unload() is waiting for download to be cancelled + notify_sse(ok ? "download_finished" : "download_failed", name, {}); + } else { + notify_sse("download_progress", name, curr); + } +} + +bool server_models::remove(const std::string & name) { + auto meta = get_meta(name); + + if (!meta.has_value()) { + throw std::runtime_error("model name=" + name + " is not found"); + } + if (meta->source != SERVER_MODEL_SOURCE_CACHE) { + throw std::runtime_error("model name=" + name + " is not removable (not from cache)"); + } + + unload(name); // cancel download or stop running instance + { + std::unique_lock lk(mutex); + // a cancelled download lands on DOWNLOADED; a stopped instance lands on UNLOADED + wait(lk, name, [](const server_model_meta & new_meta) { + return new_meta.status == SERVER_MODEL_STATUS_UNLOADED + || new_meta.status == SERVER_MODEL_STATUS_DOWNLOADED; + }); + // join before erasing - after status reaches UNLOADED/DOWNLOADED the thread no + // longer acquires this mutex, so joining while holding it is safe + if (mapping[name].th.joinable()) { + mapping[name].th.join(); + } + // remove the model from disk (hold lock to prevent concurrent load) + bool ok = common_download_remove(name); + if (ok) { + mapping.erase(name); + } + SRV_INF("removing model name=%s from cache (%s)\n", name.c_str(), ok ? "succeeded" : "failed"); + notify_sse("model_remove", name, {}); + return ok; + } +} + +void server_models::wait(const std::string & name, std::function predicate) { + std::unique_lock lk(mutex); + wait(lk, name, predicate); +} + +void server_models::wait(std::unique_lock & lk, const std::string & name, std::function predicate) { + cv.wait(lk, [this, &name, &predicate]() { + auto it = mapping.find(name); + if (it != mapping.end()) { + return predicate(it->second.meta); + } return false; }); @@ -1014,10 +1253,15 @@ bool server_models::ensure_model_ready(const std::string & name) { // wait for loading to complete SRV_INF("waiting until model name=%s is fully loaded...\n", name.c_str()); - wait_until_loading_finished(name); + wait(name, [&meta](const server_model_meta & new_meta) { + if (new_meta.status != SERVER_MODEL_STATUS_LOADING) { + meta = new_meta; // update meta for final check after wait + return true; + } + return false; + }); // check final status - meta = get_meta(name); if (!meta.has_value() || meta->is_failed()) { throw std::runtime_error("model name=" + name + " failed to load"); } @@ -1111,6 +1355,42 @@ void server_models::notify_router_sleeping_state(bool is_sleeping) { // server_models_routes // +// RAII wrapper similar to server_response_reader, but doesn't use server_queue +static std::atomic sse_client_id_counter = 0; +struct server_models_sse_client { + server_response & queue_results; + int client_id; + server_models_sse_client(server_response & q) + : queue_results(q), client_id(sse_client_id_counter.fetch_add(1, std::memory_order_relaxed)) { + SRV_DBG("new SSE client connected, assigned client_id=%d\n", client_id); + queue_results.add_waiting_task_id(client_id); + } + ~server_models_sse_client() { + SRV_DBG("SSE client disconnected, removing client_id=%d\n", client_id); + queue_results.remove_waiting_task_id(client_id); + } + + // return nullptr if should_stop() is true before receiving a result + // note: if one error is received, it will stop further processing and return error result + server_task_result_ptr next(const std::function & should_stop) { + while (true) { + static const int http_polling_seconds = 1; // check should_stop every 1 second + server_task_result_ptr result = queue_results.recv_with_timeout({client_id}, http_polling_seconds); + if (result == nullptr) { + // timeout, check stop condition + if (should_stop()) { + return nullptr; + } + // continue waiting otherwise + } else { + SRV_DBG("recv result for client_id=%d: %s\n", client_id, safe_json_to_str(result->to_json()).c_str()); + return result; + } + } + // should not reach here + } +}; + static void res_ok(std::unique_ptr & res, const json & response_data) { res->status = 200; res->data = safe_json_to_str(response_data); @@ -1274,7 +1554,9 @@ void server_models_routes::init_routes() { {"created", t}, // for OAI-compat {"status", status}, {"architecture", architecture}, - {"need_download", meta.need_download}, + {"source", server_model_source_to_string(meta.source)}, + {"can_remove", meta.source == SERVER_MODEL_SOURCE_CACHE}, + // {"need_download", meta.need_download}, // TODO: add other fields, may require reading GGUF metadata }; @@ -1312,6 +1594,87 @@ void server_models_routes::init_routes() { res_ok(res, {{"success", true}}); return res; }; + + this->get_router_models_sse = [this](const server_http_req & req) { + auto res = std::make_unique(); + res->status = 200; + res->content_type = "text/event-stream"; + auto sse_client = std::make_shared(models.sse); + res->next = [this, sse_client, &req](std::string & output) -> bool { + auto result = sse_client->next([&]() { + return stopping.load(std::memory_order_relaxed) || req.should_stop(); + }); + if (result == nullptr) { + return false; // client disconnected or should_stop + } + output = "data: " + safe_json_to_str(result->to_json()) + "\n\n"; + return true; // listen for the next event + }; + return res; + }; + + this->post_router_models = [this](const server_http_req & req) { + auto res = std::make_unique(); + + json body = json::parse(req.body); + std::string name = json_value(body, "model", std::string()); + if (name.empty()) { + throw std::invalid_argument("model must be a non-empty string"); + } + + common_params_model model; + common_download_opts opts; + + model.name = name; + model.hf_repo = name; + opts.bearer_token = params.hf_token; + opts.download_mmproj = true; + opts.download_mtp = true; + + // first, only check if the model is valid and can be downloaded + opts.skip_download = true; + bool ok = false; + try { + auto validation = common_download_model(model, opts); + ok = !validation.model_path.empty(); + } catch (const common_skip_download_exception &) { + // model is valid and will be downloaded + ok = true; + } catch (...) { + SRV_ERR("unknown error while validating model '%s'\n", name.c_str()); + // other exceptions will be handled by the outer ex_wrapper() + throw; + } + + if (!ok) { + throw std::invalid_argument("model validation failed, unable to download"); + } + + // then, proceed with the actual download + opts.skip_download = false; + SRV_INF("starting download for model '%s'\n", name.c_str()); + models.download(std::move(model), std::move(opts)); + + res_ok(res, {{"success", true}}); + return res; + }; + + this->del_router_models = [this](const server_http_req & req) { + auto res = std::make_unique(); + + std::string name = req.get_param("model"); + if (name.empty()) { + throw std::invalid_argument("model must be a non-empty string"); + } + + bool ok = models.remove(name); + if (!ok) { + throw std::runtime_error("failed to remove model '" + name + "'"); + } + + res_ok(res, {{"success", true}}); + return res; + }; } diff --git a/tools/server/server-models.h b/tools/server/server-models.h index 2198589a7..319c4352e 100644 --- a/tools/server/server-models.h +++ b/tools/server/server-models.h @@ -1,9 +1,11 @@ #pragma once #include "common.h" +#include "download.h" #include "preset.h" #include "server-common.h" #include "server-http.h" +#include "server-queue.h" #include #include @@ -14,6 +16,8 @@ /** * state diagram: * + * DOWNLOADING ──► DOWNLOADED ──► (replaced by new instance) + * * UNLOADED ──► LOADING ──► LOADED ◄──── SLEEPING * ▲ │ │ ▲ * └───failed───┘ │ │ @@ -22,39 +26,43 @@ */ enum server_model_status { // TODO: also add downloading state when the logic is added + SERVER_MODEL_STATUS_DOWNLOADING, + SERVER_MODEL_STATUS_DOWNLOADED, SERVER_MODEL_STATUS_UNLOADED, SERVER_MODEL_STATUS_LOADING, SERVER_MODEL_STATUS_LOADED, SERVER_MODEL_STATUS_SLEEPING }; -static server_model_status server_model_status_from_string(const std::string & status_str) { - if (status_str == "unloaded") { - return SERVER_MODEL_STATUS_UNLOADED; - } - if (status_str == "loading") { - return SERVER_MODEL_STATUS_LOADING; - } - if (status_str == "loaded") { - return SERVER_MODEL_STATUS_LOADED; - } - if (status_str == "sleeping") { - return SERVER_MODEL_STATUS_SLEEPING; - } - throw std::runtime_error("invalid server model status"); -} +enum server_model_source { + SERVER_MODEL_SOURCE_PRESET, + SERVER_MODEL_SOURCE_MODELS_DIR, + SERVER_MODEL_SOURCE_CACHE, +}; static std::string server_model_status_to_string(server_model_status status) { switch (status) { - case SERVER_MODEL_STATUS_UNLOADED: return "unloaded"; - case SERVER_MODEL_STATUS_LOADING: return "loading"; - case SERVER_MODEL_STATUS_LOADED: return "loaded"; - case SERVER_MODEL_STATUS_SLEEPING: return "sleeping"; - default: return "unknown"; + case SERVER_MODEL_STATUS_DOWNLOADING: return "downloading"; + case SERVER_MODEL_STATUS_DOWNLOADED: return "downloaded"; + case SERVER_MODEL_STATUS_UNLOADED: return "unloaded"; + case SERVER_MODEL_STATUS_LOADING: return "loading"; + case SERVER_MODEL_STATUS_LOADED: return "loaded"; + case SERVER_MODEL_STATUS_SLEEPING: return "sleeping"; + default: return "unknown"; + } +} + +static std::string server_model_source_to_string(server_model_source source) { + switch (source) { + case SERVER_MODEL_SOURCE_PRESET: return "preset"; + case SERVER_MODEL_SOURCE_MODELS_DIR: return "models_dir"; + case SERVER_MODEL_SOURCE_CACHE: return "cache"; + default: return "unknown"; } } struct server_model_meta { + server_model_source source = SERVER_MODEL_SOURCE_CACHE; common_preset preset; std::string name; std::set aliases; // additional names that resolve to this model @@ -63,11 +71,11 @@ struct server_model_meta { server_model_status status = SERVER_MODEL_STATUS_UNLOADED; int64_t last_used = 0; // for LRU unloading std::vector args; // args passed to the model instance, will be populated by render_args() - json loaded_info; // info to be reflected via /v1/models endpoint + json loaded_info; // info to be reflected via /v1/models endpoint ; if in DOWNLOADING state, it should contain download progress info int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED) int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown mtmd_caps multimodal; // multimodal capabilities - bool need_download = false; // whether the model needs to be downloaded before loading + // bool need_download = false; // whether the model needs to be downloaded before loading // TODO @ngxson: implement this bool is_ready() const { return status == SERVER_MODEL_STATUS_LOADED; @@ -85,12 +93,15 @@ struct server_model_meta { void update_caps(); }; -struct subprocess_s; +struct server_models_routes; +struct server_subproc; // defined in server-models.cpp struct server_models { + friend struct server_models_routes; + private: struct instance_t { - std::shared_ptr subproc; // shared between main thread and monitoring thread + std::shared_ptr subproc; // shared between main thread and monitoring thread std::thread th; server_model_meta meta; FILE * stdin_file = nullptr; @@ -107,6 +118,9 @@ private: // set to true while load_models() is executing a reload; load() will wait until clear bool is_reloading = false; + // if true, the next get_meta() will trigger a reload of model list + bool need_reload = false; + common_preset_context ctx_preset; common_params base_params; @@ -122,9 +136,14 @@ private: // not thread-safe, caller must hold mutex void add_model(server_model_meta && meta); + // notify SSE clients + void notify_sse(const std::string & event, const std::string & model_id, const json & data = nullptr); + public: server_models(const common_params & params, int argc, char ** argv); + server_response sse; // for real-time updates via SSE endpoint + // (re-)load the list of models from various sources and prepare the metadata mapping // - if this is called the first time, simply populate the metadata // - if this is called subsequently (e.g. when refreshing from disk): @@ -147,13 +166,24 @@ public: void unload(const std::string & name); void unload_all(); + // download a new model, progress is reported via SSE + // to stop the download, call unload() + void download(common_params_model && model, common_download_opts && opts); + // update the status of a model instance (thread-safe) void update_status(const std::string & name, server_model_status status, int exit_code); void update_loaded_info(const std::string & name, std::string & raw_info); + void update_download_progress(const std::string & name, const common_download_progress & progress, bool done, bool ok = true); + + // remove a cache model from disk and update the list (thread-safe) + // note: only cache models can be removed; returns false if the model doesn't exist or is not a cache model + bool remove(const std::string & name); // wait until the model instance is fully loaded (thread-safe) + // note: predicate is called while holding the lock // return when the model no longer in "loading" state - void wait_until_loading_finished(const std::string & name); + void wait(const std::string & name, std::function predicate); + void wait(std::unique_lock & lk, const std::string & name, std::function predicate); // ensure the model is in ready state (thread-safe) // return false if model is ready @@ -176,8 +206,9 @@ public: struct server_models_routes { common_params params; - json ui_settings = json::object(); // Primary: new name - json webui_settings = json::object(); // Deprecated: use ui_settings (kept for compat) + json ui_settings = json::object(); // Primary: new name + json webui_settings = json::object(); // Deprecated: use ui_settings (kept for compat) + std::atomic stopping = false; // for graceful disconnecting SSE clients during shutdown server_models models; server_models_routes(const common_params & params, int argc, char ** argv) : params(params), models(params, argc, argv) { @@ -206,6 +237,10 @@ struct server_models_routes { server_http_context::handler_t get_router_models; server_http_context::handler_t post_router_models_load; server_http_context::handler_t post_router_models_unload; + // management API + server_http_context::handler_t get_router_models_sse; + server_http_context::handler_t post_router_models; + server_http_context::handler_t del_router_models; }; /** diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cpp index 32cfe7830..5d37c3453 100644 --- a/tools/server/server-queue.cpp +++ b/tools/server/server-queue.cpp @@ -331,6 +331,17 @@ void server_response::send(server_task_result_ptr && result) { } } +void server_response::broadcast(server_task_result_ptr && result) { + std::unique_lock lock(mutex_results); + for (const auto & id_task : waiting_task_ids) { + RES_DBG("task id = %d pushed to result queue\n", id_task); + server_task_result_ptr res_copy(result->clone()); + res_copy->id = id_task; // override id with target task id + queue_results.emplace_back(std::move(res_copy)); + } + condition_results.notify_all(); +} + void server_response::terminate() { running = false; condition_results.notify_all(); diff --git a/tools/server/server-queue.h b/tools/server/server-queue.h index 35f010401..0b674d6ff 100644 --- a/tools/server/server-queue.h +++ b/tools/server/server-queue.h @@ -154,11 +154,15 @@ public: // Send a new result to a waiting id_task void send(server_task_result_ptr && result); + // broadcast a new result to all waiting tasks + // (used by router mode) + void broadcast(server_task_result_ptr && result); + // terminate the waiting loop void terminate(); }; -// utility class to make working with server_queue and server_response easier +// RAII wrapper to make working with server_queue and server_response easier // it provides a generator-like API for server responses // support pooling connection state and aggregating multiple results struct server_response_reader { diff --git a/tools/server/server-task.h b/tools/server/server-task.h index bdadcff76..1a03d5f26 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -312,6 +312,9 @@ struct server_task_result { } virtual json to_json() = 0; virtual ~server_task_result() = default; + virtual server_task_result * clone() const { + GGML_ABORT("not implemented for this task type"); + } }; // using shared_ptr for polymorphism of server_task_result @@ -649,3 +652,12 @@ struct server_prompt_cache { void update(); }; + +// used exclusively by router mode +struct server_task_result_router : server_task_result { + json data; + virtual json to_json() override { return data; } + virtual server_task_result * clone() const override { + return new server_task_result_router(*this); + } +}; diff --git a/tools/server/server.cpp b/tools/server/server.cpp index da635c625..0364d7d3b 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -174,8 +174,11 @@ int llama_server(int argc, char ** argv) { routes.get_props = models_routes->get_router_props; routes.get_models = models_routes->get_router_models; + ctx_http.post("/models", ex_wrapper(models_routes->post_router_models)); ctx_http.post("/models/load", ex_wrapper(models_routes->post_router_models_load)); ctx_http.post("/models/unload", ex_wrapper(models_routes->post_router_models_unload)); + ctx_http.get ("/models/sse", ex_wrapper(models_routes->get_router_models_sse)); + ctx_http.del ("/models", ex_wrapper(models_routes->del_router_models)); } ctx_http.get ("/health", ex_wrapper(routes.get_health)); // public endpoint (no API key check) @@ -261,6 +264,7 @@ int llama_server(int argc, char ** argv) { clean_up = [&models_routes]() { SRV_INF("%s: cleaning up before exit...\n", __func__); if (models_routes.has_value()) { + models_routes->stopping.store(true); // maybe redundant, but just to be safe models_routes->models.unload_all(); } llama_backend_free(); @@ -274,6 +278,10 @@ int llama_server(int argc, char ** argv) { ctx_http.is_ready.store(true); shutdown_handler = [&](int) { + if (models_routes.has_value()) { + // important to disconnect any SSE clients + models_routes->stopping.store(true); + } ctx_http.stop(); }; diff --git a/tools/server/tests/unit/test_router.py b/tools/server/tests/unit/test_router.py index c93b92b0b..11c77ca7a 100644 --- a/tools/server/tests/unit/test_router.py +++ b/tools/server/tests/unit/test_router.py @@ -1,3 +1,4 @@ +import threading import pytest from utils import * @@ -253,3 +254,98 @@ def test_router_reload_models(): assert "model-reload-c" in ids, "newly added model should appear" finally: os.remove(preset_path) + + +MODEL_DOWNLOAD_ID = "ggml-org/test-model-router-download:F16" +MODEL_DOWNLOAD_TIMEOUT = 300 + + +def _listen_sse(server: ServerProcess, collected: list, stop: threading.Event): + """Collect /models/sse events into `collected` until `stop` is set.""" + url = f"http://{server.server_host}:{server.server_port}/models/sse" + try: + with requests.get(url, stream=True, timeout=MODEL_DOWNLOAD_TIMEOUT) as resp: + for line_bytes in resp.iter_lines(): + if stop.is_set(): + break + line = line_bytes.decode("utf-8") + if line.startswith("data: "): + collected.append(json.loads(line[6:])) + except Exception: + pass + + +def _wait_for_sse_event(collected: list, event_type: str, model: str, timeout: int) -> bool: + deadline = time.time() + timeout + while time.time() < deadline: + if any(e.get("event") == event_type and e.get("model") == model for e in collected): + return True + time.sleep(0.5) + return False + + +def test_router_download_model(): + """Case 1: download a model, verify SSE events and GET /models.""" + global server + server.start() + + # Ensure the model is not present before we start + server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + + sse_events: list = [] + stop = threading.Event() + sse_thread = threading.Thread( + target=_listen_sse, args=(server, sse_events, stop), daemon=True + ) + sse_thread.start() + + # Trigger the download + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 200 + assert res.body.get("success") is True + + # Wait for download_finished SSE event + finished = _wait_for_sse_event( + sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT + ) + stop.set() + + assert finished, "Never received download_finished SSE event" + assert any( + e.get("event") == "download_progress" and e.get("model") == MODEL_DOWNLOAD_ID + for e in sse_events + ), "No download_progress events received" + + # Model should now appear in GET /models + ids = _get_model_ids(is_reload=False) + assert MODEL_DOWNLOAD_ID in ids, f"{MODEL_DOWNLOAD_ID} not found in /models after download" + + +def test_router_delete_model(): + """Case 2: delete the downloaded model, verify it disappears from GET /models.""" + global server + server.start() + + # Ensure the model exists (download it if needed) + if MODEL_DOWNLOAD_ID not in _get_model_ids(is_reload=False): + res = server.make_request("POST", "/models", data={"model": MODEL_DOWNLOAD_ID}) + assert res.status_code == 200 + sse_events: list = [] + stop = threading.Event() + threading.Thread( + target=_listen_sse, args=(server, sse_events, stop), daemon=True + ).start() + finished = _wait_for_sse_event( + sse_events, "download_finished", MODEL_DOWNLOAD_ID, MODEL_DOWNLOAD_TIMEOUT + ) + stop.set() + assert finished, "Model did not finish downloading before delete test" + + # Delete the model + del_res = server.make_request("DELETE", f"/models?model={MODEL_DOWNLOAD_ID}") + assert del_res.status_code == 200 + assert del_res.body.get("success") is True + + # Model should no longer appear in GET /models + ids = _get_model_ids(is_reload=False) + assert MODEL_DOWNLOAD_ID not in ids, f"{MODEL_DOWNLOAD_ID} still present after deletion" diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py index c5dba1c13..c50c9a0f5 100644 --- a/tools/server/tests/utils.py +++ b/tools/server/tests/utils.py @@ -340,6 +340,9 @@ class ServerProcess: elif method == "POST": response = requests.post(url, headers=headers, json=data, timeout=timeout) parse_body = True + elif method == "DELETE": + response = requests.delete(url, headers=headers, timeout=timeout) + parse_body = True elif method == "OPTIONS": response = requests.options(url, headers=headers, timeout=timeout) else: From e3c9601d37199077bb4ebc92f7beefbcd9faf794 Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Thu, 18 Jun 2026 00:15:39 +0800 Subject: [PATCH 025/292] updated sdui --- embd_res/kcpp_sdui.embd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index 1fe8dcb96..50807c23b 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -506,9 +506,9 @@ gl_FragColor.rgb *= color.a; `?(s++,f[s]=0):f[s]++;f[0]>0&&(this.insertCharStyleObject(c.lineIndex,c.charIndex,f[0],d),d=d&&d.slice(f[0]+1)),s&&this.insertNewlineStyleObject(c.lineIndex,c.charIndex+f[0],s);for(var h=1;h0?this.insertCharStyleObject(c.lineIndex+h,0,f[h],d):d&&this.styles[c.lineIndex+h]&&d[0]&&(this.styles[c.lineIndex+h][0]=d[0]),d=d&&d.slice(f[h]+1);f[h]>0&&this.insertCharStyleObject(c.lineIndex+h,0,f[h],d)},setSelectionStartEndWithShift:function(a,u,d){d<=a?(u===a?this._selectionDirection="left":this._selectionDirection==="right"&&(this._selectionDirection="left",this.selectionEnd=a),this.selectionStart=d):d>a&&da?this.selectionStart=a:this.selectionStart<0&&(this.selectionStart=0),this.selectionEnd>a?this.selectionEnd=a:this.selectionEnd<0&&(this.selectionEnd=0)}})}(),t.util.object.extend(t.IText.prototype,{initDoubleClickSimulation:function(){this.__lastClickTime=+new Date,this.__lastLastClickTime=+new Date,this.__lastPointer={},this.on("mousedown",this.onMouseDown)},onMouseDown:function(o){if(!!this.canvas){this.__newClickTime=+new Date;var a=o.pointer;this.isTripleClick(a)&&(this.fire("tripleclick",o),this._stopEvent(o.e)),this.__lastLastClickTime=this.__lastClickTime,this.__lastClickTime=this.__newClickTime,this.__lastPointer=a,this.__lastIsEditing=this.isEditing,this.__lastSelected=this.selected}},isTripleClick:function(o){return this.__newClickTime-this.__lastClickTime<500&&this.__lastClickTime-this.__lastLastClickTime<500&&this.__lastPointer.x===o.x&&this.__lastPointer.y===o.y},_stopEvent:function(o){o.preventDefault&&o.preventDefault(),o.stopPropagation&&o.stopPropagation()},initCursorSelectionHandlers:function(){this.initMousedownHandler(),this.initMouseupHandler(),this.initClicks()},doubleClickHandler:function(o){!this.isEditing||this.selectWord(this.getSelectionStartFromPointer(o.e))},tripleClickHandler:function(o){!this.isEditing||this.selectLine(this.getSelectionStartFromPointer(o.e))},initClicks:function(){this.on("mousedblclick",this.doubleClickHandler),this.on("tripleclick",this.tripleClickHandler)},_mouseDownHandler:function(o){!this.canvas||!this.editable||o.e.button&&o.e.button!==1||(this.__isMousedown=!0,this.selected&&(this.inCompositionMode=!1,this.setCursorByClick(o.e)),this.isEditing&&(this.__selectionStartOnMouseDown=this.selectionStart,this.selectionStart===this.selectionEnd&&this.abortCursorAnimation(),this.renderCursorOrSelection()))},_mouseDownHandlerBefore:function(o){!this.canvas||!this.editable||o.e.button&&o.e.button!==1||(this.selected=this===this.canvas._activeObject)},initMousedownHandler:function(){this.on("mousedown",this._mouseDownHandler),this.on("mousedown:before",this._mouseDownHandlerBefore)},initMouseupHandler:function(){this.on("mouseup",this.mouseUpHandler)},mouseUpHandler:function(o){if(this.__isMousedown=!1,!(!this.editable||this.group||o.transform&&o.transform.actionPerformed||o.e.button&&o.e.button!==1)){if(this.canvas){var a=this.canvas._activeObject;if(a&&a!==this)return}this.__lastSelected&&!this.__corner?(this.selected=!1,this.__lastSelected=!1,this.enterEditing(o.e),this.selectionStart===this.selectionEnd?this.initDelayedCursor(!0):this.renderCursorOrSelection()):this.selected=!0}},setCursorByClick:function(o){var a=this.getSelectionStartFromPointer(o),u=this.selectionStart,d=this.selectionEnd;o.shiftKey?this.setSelectionStartEndWithShift(u,d,a):(this.selectionStart=a,this.selectionEnd=a),this.isEditing&&(this._fireSelectionChanged(),this._updateTextarea())},getSelectionStartFromPointer:function(o){for(var a=this.getLocalPointer(o),u=0,d=0,c=0,f=0,s=0,h,v,m=0,y=this._textLines.length;m0&&(f+=this._textLines[m-1].length+this.missingNewlineOffset(m-1));h=this._getLineLeftOffset(s),d=h*this.scaleX,v=this._textLines[s],this.direction==="rtl"&&(a.x=this.width*this.scaleX-a.x+d);for(var b=0,_=v.length;b<_&&(u=d,d+=this.__charBounds[s][b].kernedWidth*this.scaleX,d<=a.x);b++)f++;return this._getNewSelectionStartFromOffset(a,u,d,f,_)},_getNewSelectionStartFromOffset:function(o,a,u,d,c){var f=o.x-a,s=u-o.x,h=s>f||s<0?0:1,v=d+h;return this.flipX&&(v=c-v),v>this._text.length&&(v=this._text.length),v}}),t.util.object.extend(t.IText.prototype,{initHiddenTextarea:function(){this.hiddenTextarea=t.document.createElement("textarea"),this.hiddenTextarea.setAttribute("autocapitalize","off"),this.hiddenTextarea.setAttribute("autocorrect","off"),this.hiddenTextarea.setAttribute("autocomplete","off"),this.hiddenTextarea.setAttribute("spellcheck","false"),this.hiddenTextarea.setAttribute("data-fabric-hiddentextarea",""),this.hiddenTextarea.setAttribute("wrap","off");var o=this._calcTextareaPosition();this.hiddenTextarea.style.cssText="position: absolute; top: "+o.top+"; left: "+o.left+"; z-index: -999; opacity: 0; width: 1px; height: 1px; font-size: 1px; padding\uFF70top: "+o.fontSize+";",this.hiddenTextareaContainer?this.hiddenTextareaContainer.appendChild(this.hiddenTextarea):t.document.body.appendChild(this.hiddenTextarea),t.util.addListener(this.hiddenTextarea,"keydown",this.onKeyDown.bind(this)),t.util.addListener(this.hiddenTextarea,"keyup",this.onKeyUp.bind(this)),t.util.addListener(this.hiddenTextarea,"input",this.onInput.bind(this)),t.util.addListener(this.hiddenTextarea,"copy",this.copy.bind(this)),t.util.addListener(this.hiddenTextarea,"cut",this.copy.bind(this)),t.util.addListener(this.hiddenTextarea,"paste",this.paste.bind(this)),t.util.addListener(this.hiddenTextarea,"compositionstart",this.onCompositionStart.bind(this)),t.util.addListener(this.hiddenTextarea,"compositionupdate",this.onCompositionUpdate.bind(this)),t.util.addListener(this.hiddenTextarea,"compositionend",this.onCompositionEnd.bind(this)),!this._clickHandlerInitialized&&this.canvas&&(t.util.addListener(this.canvas.upperCanvasEl,"click",this.onClick.bind(this)),this._clickHandlerInitialized=!0)},keysMap:{9:"exitEditing",27:"exitEditing",33:"moveCursorUp",34:"moveCursorDown",35:"moveCursorRight",36:"moveCursorLeft",37:"moveCursorLeft",38:"moveCursorUp",39:"moveCursorRight",40:"moveCursorDown"},keysMapRtl:{9:"exitEditing",27:"exitEditing",33:"moveCursorUp",34:"moveCursorDown",35:"moveCursorLeft",36:"moveCursorRight",37:"moveCursorRight",38:"moveCursorUp",39:"moveCursorLeft",40:"moveCursorDown"},ctrlKeysMapUp:{67:"copy",88:"cut"},ctrlKeysMapDown:{65:"selectAll"},onClick:function(){this.hiddenTextarea&&this.hiddenTextarea.focus()},onKeyDown:function(o){if(!!this.isEditing){var a=this.direction==="rtl"?this.keysMapRtl:this.keysMap;if(o.keyCode in a)this[a[o.keyCode]](o);else if(o.keyCode in this.ctrlKeysMapDown&&(o.ctrlKey||o.metaKey))this[this.ctrlKeysMapDown[o.keyCode]](o);else return;o.stopImmediatePropagation(),o.preventDefault(),o.keyCode>=33&&o.keyCode<=40?(this.inCompositionMode=!1,this.clearContextTop(),this.renderCursorOrSelection()):this.canvas&&this.canvas.requestRenderAll()}},onKeyUp:function(o){if(!this.isEditing||this._copyDone||this.inCompositionMode){this._copyDone=!1;return}if(o.keyCode in this.ctrlKeysMapUp&&(o.ctrlKey||o.metaKey))this[this.ctrlKeysMapUp[o.keyCode]](o);else return;o.stopImmediatePropagation(),o.preventDefault(),this.canvas&&this.canvas.requestRenderAll()},onInput:function(o){var a=this.fromPaste;if(this.fromPaste=!1,o&&o.stopPropagation(),!!this.isEditing){var u=this._splitTextIntoLines(this.hiddenTextarea.value).graphemeText,d=this._text.length,c=u.length,f,s,h=c-d,v=this.selectionStart,m=this.selectionEnd,y=v!==m,b,_,w;if(this.hiddenTextarea.value===""){this.styles={},this.updateFromTextArea(),this.fire("changed"),this.canvas&&(this.canvas.fire("text:changed",{target:this}),this.canvas.requestRenderAll());return}var S=this.fromStringToGraphemeSelection(this.hiddenTextarea.selectionStart,this.hiddenTextarea.selectionEnd,this.hiddenTextarea.value),x=v>S.selectionStart;y?(f=this._text.slice(v,m),h+=m-v):c0&&(d=this.__charBounds[o][a-1],u+=d.left+d.width),u},getDownCursorOffset:function(o,a){var u=this._getSelectionForOffset(o,a),d=this.get2DCursorLocation(u),c=d.lineIndex;if(c===this._textLines.length-1||o.metaKey||o.keyCode===34)return this._text.length-u;var f=d.charIndex,s=this._getWidthBeforeCursor(c,f),h=this._getIndexOnLine(c+1,s),v=this._textLines[c].slice(f);return v.length+h+1+this.missingNewlineOffset(c)},_getSelectionForOffset:function(o,a){return o.shiftKey&&this.selectionStart!==this.selectionEnd&&a?this.selectionEnd:this.selectionStart},getUpCursorOffset:function(o,a){var u=this._getSelectionForOffset(o,a),d=this.get2DCursorLocation(u),c=d.lineIndex;if(c===0||o.metaKey||o.keyCode===33)return-u;var f=d.charIndex,s=this._getWidthBeforeCursor(c,f),h=this._getIndexOnLine(c-1,s),v=this._textLines[c].slice(0,f),m=this.missingNewlineOffset(c-1);return-this._textLines[c-1].length+h-v.length+(1-m)},_getIndexOnLine:function(o,a){for(var u=this._textLines[o],d=this._getLineLeftOffset(o),c=d,f=0,s,h,v=0,m=u.length;va){h=!0;var y=c-s,b=c,_=Math.abs(y-a),w=Math.abs(b-a);f=w<_?v:v-1;break}return h||(f=u.length-1),f},moveCursorDown:function(o){this.selectionStart>=this._text.length&&this.selectionEnd>=this._text.length||this._moveCursorUpOrDown("Down",o)},moveCursorUp:function(o){this.selectionStart===0&&this.selectionEnd===0||this._moveCursorUpOrDown("Up",o)},_moveCursorUpOrDown:function(o,a){var u="get"+o+"CursorOffset",d=this[u](a,this._selectionDirection==="right");a.shiftKey?this.moveCursorWithShift(d):this.moveCursorWithoutShift(d),d!==0&&(this.setSelectionInBoundaries(),this.abortCursorAnimation(),this._currentCursorOpacity=1,this.initDelayedCursor(),this._fireSelectionChanged(),this._updateTextarea())},moveCursorWithShift:function(o){var a=this._selectionDirection==="left"?this.selectionStart+o:this.selectionEnd+o;return this.setSelectionStartEndWithShift(this.selectionStart,this.selectionEnd,a),o!==0},moveCursorWithoutShift:function(o){return o<0?(this.selectionStart+=o,this.selectionEnd=this.selectionStart):(this.selectionEnd+=o,this.selectionStart=this.selectionEnd),o!==0},moveCursorLeft:function(o){this.selectionStart===0&&this.selectionEnd===0||this._moveCursorLeftOrRight("Left",o)},_move:function(o,a,u){var d;if(o.altKey)d=this["findWordBoundary"+u](this[a]);else if(o.metaKey||o.keyCode===35||o.keyCode===36)d=this["findLineBoundary"+u](this[a]);else return this[a]+=u==="Left"?-1:1,!0;if(typeof d<"u"&&this[a]!==d)return this[a]=d,!0},_moveLeft:function(o,a){return this._move(o,a,"Left")},_moveRight:function(o,a){return this._move(o,a,"Right")},moveCursorLeftWithoutShift:function(o){var a=!0;return this._selectionDirection="left",this.selectionEnd===this.selectionStart&&this.selectionStart!==0&&(a=this._moveLeft(o,"selectionStart")),this.selectionEnd=this.selectionStart,a},moveCursorLeftWithShift:function(o){if(this._selectionDirection==="right"&&this.selectionStart!==this.selectionEnd)return this._moveLeft(o,"selectionEnd");if(this.selectionStart!==0)return this._selectionDirection="left",this._moveLeft(o,"selectionStart")},moveCursorRight:function(o){this.selectionStart>=this._text.length&&this.selectionEnd>=this._text.length||this._moveCursorLeftOrRight("Right",o)},_moveCursorLeftOrRight:function(o,a){var u="moveCursor"+o+"With";this._currentCursorOpacity=1,a.shiftKey?u+="Shift":u+="outShift",this[u](a)&&(this.abortCursorAnimation(),this.initDelayedCursor(),this._fireSelectionChanged(),this._updateTextarea())},moveCursorRightWithShift:function(o){if(this._selectionDirection==="left"&&this.selectionStart!==this.selectionEnd)return this._moveRight(o,"selectionStart");if(this.selectionEnd!==this._text.length)return this._selectionDirection="right",this._moveRight(o,"selectionEnd")},moveCursorRightWithoutShift:function(o){var a=!0;return this._selectionDirection="right",this.selectionStart===this.selectionEnd?(a=this._moveRight(o,"selectionStart"),this.selectionEnd=this.selectionStart):this.selectionStart=this.selectionEnd,a},removeChars:function(o,a){typeof a>"u"&&(a=o+1),this.removeStyleFromTo(o,a),this._text.splice(o,a-o),this.text=this._text.join(""),this.set("dirty",!0),this._shouldClearDimensionCache()&&(this.initDimensions(),this.setCoords()),this._removeExtraneousStyles()},insertChars:function(o,a,u,d){typeof d>"u"&&(d=u),d>u&&this.removeStyleFromTo(u,d);var c=t.util.string.graphemeSplit(o);this.insertNewStyleBlock(c,u,a),this._text=[].concat(this._text.slice(0,u),c,this._text.slice(d)),this.text=this._text.join(""),this.set("dirty",!0),this._shouldClearDimensionCache()&&(this.initDimensions(),this.setCoords()),this._removeExtraneousStyles()}}),function(){var o=t.util.toFixed,a=/ +/g;t.util.object.extend(t.Text.prototype,{_toSVG:function(){var u=this._getSVGLeftTopOffsets(),d=this._getSVGTextAndBg(u.textTop,u.textLeft);return this._wrapSVGTextAndBg(d)},toSVG:function(u){return this._createBaseSVGMarkup(this._toSVG(),{reviver:u,noStyle:!0,withShadow:!0})},_getSVGLeftTopOffsets:function(){return{textLeft:-this.width/2,textTop:-this.height/2,lineTop:this.getHeightOfLine(0)}},_wrapSVGTextAndBg:function(u){var d=!0,c=this.getSvgTextDecoration(this);return[u.textBgRects.join(""),' ",u.textSpans.join(""),` `]},_getSVGTextAndBg:function(u,d){var c=[],f=[],s=u,h;this._setSVGBg(f);for(var v=0,m=this._textLines.length;v",t.util.string.escapeXml(u),""].join("")},_setSVGTextLineText:function(u,d,c,f){var s=this.getHeightOfLine(d),h=this.textAlign.indexOf("justify")!==-1,v,m,y="",b,_,w=0,S=this._textLines[d],x;f+=s*(1-this._fontSizeFraction)/this.lineHeight;for(var T=0,E=S.length-1;T<=E;T++)x=T===E||this.charSpacing,y+=S[T],b=this.__charBounds[d][T],w===0?(c+=b.kernedWidth-b.width,w+=b.width):w+=b.kernedWidth,h&&!x&&this._reSpaceAndTab.test(S[T])&&(x=!0),x||(v=v||this.getCompleteStyleDeclaration(d,T),m=this.getCompleteStyleDeclaration(d,T+1),x=t.util.hasStyleChanged(v,m,!0)),x&&(_=this._getStyleDeclaration(d,T)||{},u.push(this._createTextCharSpan(y,_,c,f)),y="",v=m,c+=w,w=0)},_pushTextBgRect:function(u,d,c,f,s,h){var v=t.Object.NUM_FRACTION_DIGITS;u.push(" `)},_setSVGTextLineBg:function(u,d,c,f){for(var s=this._textLines[d],h=this.getHeightOfLine(d)/this.lineHeight,v=0,m=0,y,b,_=this.getValueOfPropertyAt(d,0,"textBackgroundColor"),w=0,S=s.length;wthis.width&&this._set("width",this.dynamicMinWidth),this.textAlign.indexOf("justify")!==-1&&this.enlargeSpaces(),this.height=this.calcTextHeight(),this.saveState({propertySet:"_dimensionAffectingProps"}))},_generateStyleMap:function(u){for(var d=0,c=0,f=0,s={},h=0;h0?(c=0,f++,d++):!this.splitByGrapheme&&this._reSpaceAndTab.test(u.graphemeText[f])&&h>0&&(c++,f++),s[h]={line:d,offset:c},f+=u.graphemeLines[h].length,c+=u.graphemeLines[h].length;return s},styleHas:function(u,d){if(this._styleMap&&!this.isWrapping){var c=this._styleMap[d];c&&(d=c.line)}return a.Text.prototype.styleHas.call(this,u,d)},isEmptyStyles:function(u){if(!this.styles)return!0;var d=0,c=u+1,f,s,h=!1,v=this._styleMap[u],m=this._styleMap[u+1];v&&(u=v.line,d=v.offset),m&&(c=m.line,h=c===u,f=m.offset),s=typeof u>"u"?this.styles:{line:this.styles[u]};for(var y in s)for(var b in s[y])if(b>=d&&(!h||bc&&!E?(v.push(m),m=[],s=S,E=!0):s+=I,!E&&!h&&m.push(w),m=m.concat(b),x=h?0:this._measureWord([w],d,_),_++,E=!1,S>T&&(T=S);return N&&v.push(m),T+D>this.dynamicMinWidth&&(this.dynamicMinWidth=T-I+D),v},isEndOfWrapping:function(u){return!this._styleMap[u+1]||this._styleMap[u+1].line!==this._styleMap[u].line},missingNewlineOffset:function(u){return this.splitByGrapheme?this.isEndOfWrapping(u)?1:0:1},_splitTextIntoLines:function(u){for(var d=a.Text.prototype._splitTextIntoLines.call(this,u),c=this._wrapText(d.lines,this.width),f=new Array(c.length),s=0;s{const e=()=>({canvas:void 0,brush:void 0,visibleImageLayer:void 0,imageLayer:void 0,visibleDrawLayer:void 0,drawLayer:void 0,cropPreviewLayer:void 0,maskPathColor:"",maskBackgroundColor:"",imageScale:1,undoHistory:[],redoHistory:[],drawing:!1}),t=oe({...e(),maskPathColor:"white",maskBackgroundColor:"black"}),n=oe({...e(),maskPathColor:"black",maskBackgroundColor:"white"}),r=ee(()=>Jt().generatorType==="Inpainting"),i=ee(()=>r.value?t.value:n.value),l=ee(()=>Jt().currentImageProps),g=ee({get:()=>i.value.drawing&&!r.value,set:P=>i.value.drawing=P}),o=oe(512),a=oe(512),u=oe(!1),d=oe(30),c=oe(!1),f=new Kn.fabric.Circle({radius:d.value,left:0,originX:"center",originY:"center",angle:0,fill:"",stroke:"red",strokeWidth:3,opacity:0}),s=oe("Erase"),h=oe("rgb(0, 0, 0, 1)");function v(){!i.value.canvas||i.value.canvas.renderAll()}function m(){u.value=!u.value,s.value=u.value?"Draw":"Erase"}function y(P=null){!i.value.canvas||(i.value.brush=i.value.canvas.freeDrawingBrush,i.value.brush.color=P||i.value.brush.color,i.value.brush.width=d.value)}async function b({history:P,erase:Q=!1,draw:j=!1}={}){if(!P||!i.value.drawLayer||!i.value.visibleDrawLayer||!i.value.imageLayer||!i.value.visibleImageLayer||!i.value.canvas)return;P.path.selectable=!1,P.path.opacity=1,P.drawPath=await M(P.path),P.visibleDrawPath=await M(P.path),Q?(P.visibleDrawPath.globalCompositeOperation="destination-out",P.drawPath.stroke=i.value.maskBackgroundColor):(P.visibleDrawPath.globalCompositeOperation="source-over",P.drawPath.stroke=j?h.value:i.value.maskPathColor);let A=await M(P.drawPath);A=A.scale(i.value.imageScale),A.left=A.left+P.drawPath.left*(i.value.imageScale-1),A.top=A.top+P.drawPath.top*(i.value.imageScale-1),j?(i.value.imageLayer.add(A),i.value.visibleImageLayer.addWithUpdate(P.visibleDrawPath)):(i.value.drawLayer.add(A),i.value.visibleDrawLayer.addWithUpdate(P.visibleDrawPath)),i.value.canvas.remove(P.path),v()}function _(){if(i.value.undoHistory.length===0)return;const P=i.value.undoHistory.pop();b({history:P,erase:!1,draw:g.value}),i.value.redoHistory.push(P)}function w(){if(i.value.redoHistory.length===0||!i.value.drawLayer||!i.value.visibleDrawLayer||!i.value.imageLayer||!i.value.visibleImageLayer||!i.value.canvas)return;const P=i.value.redoHistory.pop();i.value.undoHistory.push(P),g.value?(i.value.imageLayer.remove(P.drawPath),i.value.visibleImageLayer.remove(P.visibleDrawPath)):(i.value.drawLayer.remove(P.drawPath),i.value.visibleDrawLayer.remove(P.visibleDrawPath)),delete P.drawPath,delete P.visibleDrawPath,v()}function S(P){i.value.canvas=new Kn.fabric.Canvas(P,{isDrawingMode:!1,width:o.value,height:a.value,backgroundColor:"white"}),i.value.canvas.selection=!1,i.value.canvas.freeDrawingCursor="crosshair",y(i.value.maskPathColor),i.value.canvas.on("mouse:move",R),i.value.canvas.on("path:created",U),v()}function x(P,Q,j,A){let q=A,G=A;return Q>j?(P.scaleToWidth(A),q=A*(a.value/o.value)):(P.scaleToHeight(A),G=A*(o.value/a.value)),{newHeight:q,newWidth:G}}function T(P){const Q=Jt();if(re(),P.selectable=!1,o.value=P.width,a.value=P.height,o.value>Q.maxDimensions||a.value>Q.maxDimensions){const{newHeight:G,newWidth:te}=x(P,o.value,a.value,Q.maxDimensions);o.value=te,a.value=G}if(o.value{i.value.imageLayer=F({image:G,layerHeight:q,layerWidth:A})}),P.cloneAsImage(G=>{if(!i.value.canvas)return;if(o.value!==j||a.value!==j){const{newHeight:pe,newWidth:ve}=x(G,o.value,a.value,j);o.value=ve,a.value=pe}i.value.canvas.setWidth(o.value),i.value.canvas.setHeight(a.value),i.value.canvas.isDrawingMode=!0,i.value.visibleDrawLayer=V(),i.value.visibleImageLayer=V({image:G}),i.value.drawLayer=F({layerHeight:q,layerWidth:A});const te=o.value*i.value.imageScale,fe=a.value*i.value.imageScale;Q.params.width=te-te%64,Q.params.height=fe-fe%64,i.value.visibleDrawLayer.set("opacity",.8),i.value.canvas.add(i.value.visibleImageLayer),i.value.canvas.add(i.value.visibleDrawLayer),i.value.canvas.add(f),c.value=!0,N(),I()})}function E(P,Q,j,A,q,G){const te=G?G.width:P.width,fe=G?G.height:P.height;if(q==="Original")return P;const pe=document.createElement("canvas");pe.width=Q,pe.height=j;const ve=pe.getContext("2d");ve.fillStyle=A,ve.fillRect(0,0,pe.width,pe.height);let H=0,W=0,k=P.width,X=P.height,Y=0,$=0,B=pe.width,K=pe.height;switch(q){case"Stretch":break;case"ScaleAndCrop":{const ce=pe.width/te,ie=pe.height/fe;ce>ie?(X=pe.height/ce,W=(P.height-X)/2):(k=pe.width/ie,H=(P.width-k)/2);break}case"ScaleAndPad":{const ce=pe.width/te,ie=pe.height/fe;ceQ?(k=Q,H=ce-Q/2):(B=te,Y=Q/2-ce),fe>j?(X=j,W=ie-j/2):(K=fe,$=j/2-ie);break}}return ve.drawImage(P,H,W,k,X,Y,$,B,K),pe}function I(){const P=Jt(),Q=zt();if(!i.value.imageLayer||!i.value.drawLayer)return;const j=P.params.width,A=P.params.height,q=i.value.imageLayer.backgroundColor||"#FFFFFF",G=Q.imageResizeMode,te=i.value.imageLayer.toCanvasElement(),fe=E(te,j,A,q,G,te);if(l.value.sourceImage=fe.toDataURL("image/jpeg",1),l.value.maskImage=void 0,i.value.redoHistory.length>0&&!g.value){const ve=i.value.drawLayer.toCanvasElement(),H=E(ve,j,A,q,G,te);l.value.maskImage=H.toDataURL("image/jpeg",1).split(",")[1]}}let D;function N(){if(!i.value.canvas)return;const P=Jt();i.value.cropPreviewLayer&&(i.value.canvas.remove(i.value.cropPreviewLayer),i.value.cropPreviewLayer=void 0),c.value&&(i.value.cropPreviewLayer=V({layerWidth:P.params.width/i.value.imageScale,layerHeight:P.params.height/i.value.imageScale,fill:"rgba(100, 0, 0, 0.5)"}),i.value.canvas.centerObject(i.value.cropPreviewLayer),i.value.canvas.add(i.value.cropPreviewLayer),D&&clearTimeout(D),D=setTimeout(()=>{c.value=!1,N(),D=void 0},5e3))}function L(P,Q){const j="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAA1JREFUGFdj+P///38ACfsD/QVDRcoAAAAASUVORK5CYII=";Kn.fabric.Image.fromURL(j,A=>{A.set({height:P,width:Q});const q=A.toDataURL({format:"png"});l.value.sourceImage=q,g.value=!0,T(A)})}function F({image:P,layerWidth:Q,layerHeight:j}={}){const A=new Kn.fabric.Canvas(null);return A.selection=!1,A.backgroundColor=i.value.maskBackgroundColor,A.setHeight(j||a.value),A.setWidth(Q||o.value),P&&A.add(P),A}function O(){return i.value.imageLayer?{layerHeight:i.value.imageLayer.getHeight(),layerWidth:i.value.imageLayer.getWidth()}:{}}function V({image:P,layerWidth:Q,layerHeight:j,fill:A,abosolute:q}={}){const G=P||new Kn.fabric.Rect({width:Q||o.value,height:j||a.value,left:0,top:0,fill:A||"transparent",absolutePositioned:q||!0,selectable:!1});return new Kn.fabric.Group([G],{selectable:!1,absolutePositioned:q||!0})}function re(){!i.value.canvas||(i.value.visibleImageLayer&&(i.value.canvas.remove(i.value.visibleImageLayer),i.value.visibleImageLayer=void 0),i.value.visibleDrawLayer&&(i.value.canvas.remove(i.value.visibleDrawLayer),i.value.visibleDrawLayer=void 0),i.value.imageLayer=void 0,i.value.drawLayer=void 0,i.value.redoHistory=[],i.value.undoHistory=[],i.value.canvas.isDrawingMode=!1)}function J(){if(!!i.value.canvas){if(i.value.visibleDrawLayer&&(i.value.canvas.remove(i.value.visibleDrawLayer),i.value.visibleDrawLayer=void 0),g.value){const P=Jt();L(P.params.height||512,P.params.width||512)}i.value.drawLayer=void 0,i.value.redoHistory=[],i.value.undoHistory=[],i.value.visibleDrawLayer=V(),i.value.drawLayer=F(O()),i.value.visibleDrawLayer.set("opacity",.8),i.value.canvas.add(i.value.visibleDrawLayer)}}function ue(){var Q;I();const P=document.createElement("a");if(g.value){P.href="data:image/png;base64,"+((Q=l.value.sourceImage)==null?void 0:Q.split(",")[1]),P.download="image_drawing.png",P.click();return}P.href="data:image/png;base64,"+l.value.maskImage,P.download="image_mask.png",P.click()}async function M(P){return new Promise((Q,j)=>{try{P.clone(Q)}catch(A){j(A)}})}async function U(P){const Q={path:P.path};b({history:Q,erase:u.value,draw:g.value}),i.value.redoHistory.push(Q)}function R(P){if(!i.value.canvas)return;const Q=i.value.canvas.getPointer(P.e);f.left=Q.x,f.top=Q.y,f.opacity=.8,u.value?(f.set("strokeWidth",3),f.set("fill",""),y("red")):(f.set("strokeWidth",0),g.value?(f.set("fill",h.value),y(h.value)):(f.set("fill","white"),y("white"))),f.set("radius",d.value/2),v()}return{showCropPreview:c,erasing:u,switchToolText:s,brushSize:d,drawColor:h,drawing:g,imageProps:i,updateCropPreview:N,createNewCanvas:S,downloadMask:ue,resetCanvas:re,resetDrawing:J,flipErase:m,undoAction:w,redoAction:_,newImage:T,newBlankImage:L,setBrush:y,saveImages:I}});const XH={},KH={xmlns:"http://www.w3.org/2000/svg","xmlns:xlink":"http://www.w3.org/1999/xlink",viewBox:"0 0 20 20"},GH=ne("g",{fill:"none"},[ne("path",{d:"M11.197 2.44a1.5 1.5 0 0 1 2.121 0l4.243 4.242a1.5 1.5 0 0 1 0 2.121L9.364 17H14.5a.5.5 0 0 1 0 1H7.82a1.496 1.496 0 0 1-1.14-.437L2.437 13.32a1.5 1.5 0 0 1 0-2.121l8.76-8.76zm1.414.706a.5.5 0 0 0-.707 0L5.538 9.512l4.95 4.95l6.366-6.366a.5.5 0 0 0 0-.707L12.61 3.146zM9.781 15.17l-4.95-4.95l-1.687 1.687a.5.5 0 0 0 0 .707l4.243 4.243a.5.5 0 0 0 .707 0l1.687-1.687z",fill:"currentColor"})],-1),qH=[GH];function ZH(e,t){return z(),se("svg",KH,qH)}const JH=qt(XH,[["render",ZH]]);async function QH(e,t){const n=document.createElement("canvas"),r=n.getContext("2d"),i=new Image;return i.src=e,await new Promise(g=>i.onload=g),n.width=i.width,n.height=i.height,r==null||r.drawImage(i,0,0),n.toDataURL(t)}async function eW(e,t){const n=e.split(";base64,"),r=t!=null?t:n[0].split(":")[1],i=window.atob(r===n[0].split(":")[1]?n[1]:(await QH(e,r)).split(",")[1]),l=new Uint8Array(i.length);for(let g=0;g{const r=new FileReader;r.onload=()=>t(r.result),r.onerror=i=>n(i),r.readAsDataURL(e)})}const gm=e=>(ui("data-v-7dc3b5bb"),e=e(),ci(),e),tW=gm(()=>ne("div",null,[He("Drop file here, paste an image OR "),ne("em",null,"click to upload")],-1)),nW={key:0},rW=gm(()=>ne("div",{class:"center-horizontal",style:{"margin-top":"5px"}},"OR",-1)),iW={class:"canvas-container"},aW=gm(()=>ne("canvas",{id:"canvas"},null,-1)),oW={class:"action-buttons",style:{left:"10px",right:"unset"}},sW={class:"action-buttons"},lW=Ee({__name:"CustomCanvas",setup(e){const t=Jt(),n=Ut(),r=Ls(),i=oe();async function l(a){if(!a.raw.type.includes("image")){n.raiseError("Uploaded file needs to be a image!",!1),i.value.clearFiles();return}const u=await Tc(a.raw);t.currentImageProps.sourceImage=u,r.drawing=!1,Kn.fabric.Image.fromURL(u,r.newImage)}function g(){t.currentImageProps.sourceImage="",r.resetCanvas()}nt(()=>{r.createNewCanvas("canvas"),t.currentImageProps.sourceImage&&Kn.fabric.Image.fromURL(t.currentImageProps.sourceImage,r.newImage),window.addEventListener("paste",o)}),Yt(()=>{window.removeEventListener("paste",o)});async function o(a){var d;const u=(d=a.clipboardData)==null?void 0:d.items;if(!!u)for(let c=0;c(z(),se(je,null,[C(t).currentImageProps.sourceImage?ye("",!0):(z(),be(C(Yp),{key:0,drag:"",ref_key:"upload",ref:i,"auto-upload":!1,onChange:l,limit:1,multiple:""},{tip:he(()=>[C(t).generatorType==="Img2Img"?(z(),se("div",nW,[rW,ne("div",{class:"center-both",style:{cursor:"pointer","text-decoration":"underline","font-size":"1rem"},onClick:u[0]||(u[0]=d=>C(r).newBlankImage(C(t).params.height||512,C(t).params.width||512))},[le(C(Le),{size:20,style:{"margin-right":"2px"}},{default:he(()=>[le(S2)]),_:1}),He("draw something")])])):ye("",!0)]),default:he(()=>[le(C(Le),{size:100},{default:he(()=>[le(C(dp))]),_:1}),tW]),_:1},512)),Et(ne("div",null,[ne("div",iW,[aW,ne("div",oW,[le(C(at),{onClick:u[1]||(u[1]=d=>C(r).undoAction()),icon:C(y_),plain:"",disabled:C(r).imageProps.redoHistory.length===0},null,8,["icon","disabled"]),le(C(at),{onClick:u[2]||(u[2]=d=>C(r).redoAction()),icon:C(b_),plain:"",disabled:C(r).imageProps.undoHistory.length===0},null,8,["icon","disabled"])]),ne("div",sW,[le(C(at),{onClick:u[3]||(u[3]=d=>C(r).resetDrawing()),icon:C(Mr),plain:""},null,8,["icon"]),le(C(at),{onClick:g,icon:C(Ol),plain:""},null,8,["icon"]),le(C(at),{onClick:u[4]||(u[4]=d=>C(r).downloadMask()),icon:C(Ks),plain:""},null,8,["icon"]),le(C(at),{onClick:u[5]||(u[5]=d=>C(r).flipErase()),icon:C(r).erasing?C(o3):JH,plain:""},null,8,["icon"]),C(r).drawing?(z(),be(C(EB),{key:0,modelValue:C(r).drawColor,"onUpdate:modelValue":u[6]||(u[6]=d=>C(r).drawColor=d),"show-alpha":""},null,8,["modelValue"])):ye("",!0)]),le(C(jp),{"label-width":"110px",style:{"margin-top":"10px"}},{default:he(()=>[le(bn,{style:{"margin-bottom":"5px"},label:"Brush Size",prop:"brushSize",modelValue:C(r).brushSize,"onUpdate:modelValue":u[7]||(u[7]=d=>C(r).brushSize=d),min:10,max:100,step:10,change:C(r).setBrush},null,8,["modelValue","change"])]),_:1})])],512),[[Ht,C(t).currentImageProps.sourceImage]])],64))}});const K0=qt(lW,[["__scopeId","data-v-7dc3b5bb"]]),uW={class:"centerIcons"},cW={class:"stackedIcons"},fW=Ee({__name:"StackedIcon",props:{iconOne:null,iconTwo:null,size:null},setup(e){const t=e;return Fx(n=>({"2ad037ca":e.size+"px"})),(n,r)=>(z(),se("div",uW,[ne("div",cW,[le(C(Le),{class:"firstIcon",size:e.size},{default:he(()=>[(z(),be(kt(t.iconOne)))]),_:1},8,["size"]),le(C(Le),{class:"secondIcon",size:e.size},{default:he(()=>[(z(),be(kt(t.iconTwo)))]),_:1},8,["size"])])]))}});const dW=qt(fW,[["__scopeId","data-v-74586a39"]]),hW={key:1,style:{width:"40px"}},bu=Ee({__name:"GeneratorMenuItem",props:{index:null,iconOne:null,iconTwo:null,isMobile:{type:Boolean}},setup(e){const t=e;return(n,r)=>(z(),be(C(zn),{content:e.index,placement:e.isMobile?"bottom":"right",enterable:!1,"hide-after":100},{default:he(()=>[le(C(Up),{index:e.index,style:{height:"60px",display:"flex","justify-content":"center"}},{default:he(()=>[e.iconTwo?(z(),be(dW,{key:0,iconOne:e.iconOne,iconTwo:e.iconTwo,size:40},null,8,["iconOne","iconTwo"])):(z(),se("div",hW,[le(C(Le),{style:{width:"35px"},size:40},{default:he(()=>[(z(),be(kt(t.iconOne)))]),_:1})]))]),_:1},8,["index"])]),_:1},8,["content","placement"]))}});/*! pako 2.1.0 https://github.com/nodeca/pako @license (MIT AND Zlib) */const pW=4,G0=0,q0=1,mW=2;function Go(e){let t=e.length;for(;--t>=0;)e[t]=0}const gW=0,k2=1,vW=2,yW=3,bW=258,vm=29,Vl=256,pl=Vl+1+vm,Co=30,ym=19,E2=2*pl+1,_a=15,Jf=16,_W=7,bm=256,O2=16,A2=17,P2=18,Eh=new Uint8Array([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0]),Xu=new Uint8Array([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13]),wW=new Uint8Array([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7]),I2=new Uint8Array([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),CW=512,Gr=new Array((pl+2)*2);Go(Gr);const Rs=new Array(Co*2);Go(Rs);const ml=new Array(CW);Go(ml);const gl=new Array(bW-yW+1);Go(gl);const _m=new Array(vm);Go(_m);const kc=new Array(Co);Go(kc);function Qf(e,t,n,r,i){this.static_tree=e,this.extra_bits=t,this.extra_base=n,this.elems=r,this.max_length=i,this.has_stree=e&&e.length}let M2,L2,R2;function ed(e,t){this.dyn_tree=e,this.max_code=0,this.stat_desc=t}const D2=e=>e<256?ml[e]:ml[256+(e>>>7)],vl=(e,t)=>{e.pending_buf[e.pending++]=t&255,e.pending_buf[e.pending++]=t>>>8&255},En=(e,t,n)=>{e.bi_valid>Jf-n?(e.bi_buf|=t<>Jf-e.bi_valid,e.bi_valid+=n-Jf):(e.bi_buf|=t<{En(e,n[t*2],n[t*2+1])},$2=(e,t)=>{let n=0;do n|=e&1,e>>>=1,n<<=1;while(--t>0);return n>>>1},SW=e=>{e.bi_valid===16?(vl(e,e.bi_buf),e.bi_buf=0,e.bi_valid=0):e.bi_valid>=8&&(e.pending_buf[e.pending++]=e.bi_buf&255,e.bi_buf>>=8,e.bi_valid-=8)},xW=(e,t)=>{const n=t.dyn_tree,r=t.max_code,i=t.stat_desc.static_tree,l=t.stat_desc.has_stree,g=t.stat_desc.extra_bits,o=t.stat_desc.extra_base,a=t.stat_desc.max_length;let u,d,c,f,s,h,v=0;for(f=0;f<=_a;f++)e.bl_count[f]=0;for(n[e.heap[e.heap_max]*2+1]=0,u=e.heap_max+1;ua&&(f=a,v++),n[d*2+1]=f,!(d>r)&&(e.bl_count[f]++,s=0,d>=o&&(s=g[d-o]),h=n[d*2],e.opt_len+=h*(f+s),l&&(e.static_len+=h*(i[d*2+1]+s)));if(v!==0){do{for(f=a-1;e.bl_count[f]===0;)f--;e.bl_count[f]--,e.bl_count[f+1]+=2,e.bl_count[a]--,v-=2}while(v>0);for(f=a;f!==0;f--)for(d=e.bl_count[f];d!==0;)c=e.heap[--u],!(c>r)&&(n[c*2+1]!==f&&(e.opt_len+=(f-n[c*2+1])*n[c*2],n[c*2+1]=f),d--)}},B2=(e,t,n)=>{const r=new Array(_a+1);let i=0,l,g;for(l=1;l<=_a;l++)i=i+n[l-1]<<1,r[l]=i;for(g=0;g<=t;g++){let o=e[g*2+1];o!==0&&(e[g*2]=$2(r[o]++,o))}},TW=()=>{let e,t,n,r,i;const l=new Array(_a+1);for(n=0,r=0;r>=7;r{let t;for(t=0;t{e.bi_valid>8?vl(e,e.bi_buf):e.bi_valid>0&&(e.pending_buf[e.pending++]=e.bi_buf),e.bi_buf=0,e.bi_valid=0},Z0=(e,t,n,r)=>{const i=t*2,l=n*2;return e[i]{const r=e.heap[n];let i=n<<1;for(;i<=e.heap_len&&(i{let r,i,l=0,g,o;if(e.sym_next!==0)do r=e.pending_buf[e.sym_buf+l++]&255,r+=(e.pending_buf[e.sym_buf+l++]&255)<<8,i=e.pending_buf[e.sym_buf+l++],r===0?Tr(e,i,t):(g=gl[i],Tr(e,g+Vl+1,t),o=Eh[g],o!==0&&(i-=_m[g],En(e,i,o)),r--,g=D2(r),Tr(e,g,n),o=Xu[g],o!==0&&(r-=kc[g],En(e,r,o)));while(l{const n=t.dyn_tree,r=t.stat_desc.static_tree,i=t.stat_desc.has_stree,l=t.stat_desc.elems;let g,o,a=-1,u;for(e.heap_len=0,e.heap_max=E2,g=0;g>1;g>=1;g--)td(e,n,g);u=l;do g=e.heap[1],e.heap[1]=e.heap[e.heap_len--],td(e,n,1),o=e.heap[1],e.heap[--e.heap_max]=g,e.heap[--e.heap_max]=o,n[u*2]=n[g*2]+n[o*2],e.depth[u]=(e.depth[g]>=e.depth[o]?e.depth[g]:e.depth[o])+1,n[g*2+1]=n[o*2+1]=u,e.heap[1]=u++,td(e,n,1);while(e.heap_len>=2);e.heap[--e.heap_max]=e.heap[1],xW(e,t),B2(n,a,e.bl_count)},Q0=(e,t,n)=>{let r,i=-1,l,g=t[0*2+1],o=0,a=7,u=4;for(g===0&&(a=138,u=3),t[(n+1)*2+1]=65535,r=0;r<=n;r++)l=g,g=t[(r+1)*2+1],!(++o{let r,i=-1,l,g=t[0*2+1],o=0,a=7,u=4;for(g===0&&(a=138,u=3),r=0;r<=n;r++)if(l=g,g=t[(r+1)*2+1],!(++o{let t;for(Q0(e,e.dyn_ltree,e.l_desc.max_code),Q0(e,e.dyn_dtree,e.d_desc.max_code),Oh(e,e.bl_desc),t=ym-1;t>=3&&e.bl_tree[I2[t]*2+1]===0;t--);return e.opt_len+=3*(t+1)+5+5+4,t},EW=(e,t,n,r)=>{let i;for(En(e,t-257,5),En(e,n-1,5),En(e,r-4,4),i=0;i{let t=4093624447,n;for(n=0;n<=31;n++,t>>>=1)if(t&1&&e.dyn_ltree[n*2]!==0)return G0;if(e.dyn_ltree[9*2]!==0||e.dyn_ltree[10*2]!==0||e.dyn_ltree[13*2]!==0)return q0;for(n=32;n{ty||(TW(),ty=!0),e.l_desc=new ed(e.dyn_ltree,M2),e.d_desc=new ed(e.dyn_dtree,L2),e.bl_desc=new ed(e.bl_tree,R2),e.bi_buf=0,e.bi_valid=0,F2(e)},N2=(e,t,n,r)=>{En(e,(gW<<1)+(r?1:0),3),z2(e),vl(e,n),vl(e,~n),n&&e.pending_buf.set(e.window.subarray(t,t+n),e.pending),e.pending+=n},PW=e=>{En(e,k2<<1,3),Tr(e,bm,Gr),SW(e)},IW=(e,t,n,r)=>{let i,l,g=0;e.level>0?(e.strm.data_type===mW&&(e.strm.data_type=OW(e)),Oh(e,e.l_desc),Oh(e,e.d_desc),g=kW(e),i=e.opt_len+3+7>>>3,l=e.static_len+3+7>>>3,l<=i&&(i=l)):i=l=n+5,n+4<=i&&t!==-1?N2(e,t,n,r):e.strategy===pW||l===i?(En(e,(k2<<1)+(r?1:0),3),J0(e,Gr,Rs)):(En(e,(vW<<1)+(r?1:0),3),EW(e,e.l_desc.max_code+1,e.d_desc.max_code+1,g+1),J0(e,e.dyn_ltree,e.dyn_dtree)),F2(e),r&&z2(e)},MW=(e,t,n)=>(e.pending_buf[e.sym_buf+e.sym_next++]=t,e.pending_buf[e.sym_buf+e.sym_next++]=t>>8,e.pending_buf[e.sym_buf+e.sym_next++]=n,t===0?e.dyn_ltree[n*2]++:(e.matches++,t--,e.dyn_ltree[(gl[n]+Vl+1)*2]++,e.dyn_dtree[D2(t)*2]++),e.sym_next===e.sym_end);var LW=AW,RW=N2,DW=IW,$W=MW,BW=PW,FW={_tr_init:LW,_tr_stored_block:RW,_tr_flush_block:DW,_tr_tally:$W,_tr_align:BW};const zW=(e,t,n,r)=>{let i=e&65535|0,l=e>>>16&65535|0,g=0;for(;n!==0;){g=n>2e3?2e3:n,n-=g;do i=i+t[r++]|0,l=l+i|0;while(--g);i%=65521,l%=65521}return i|l<<16|0};var yl=zW;const NW=()=>{let e,t=[];for(var n=0;n<256;n++){e=n;for(var r=0;r<8;r++)e=e&1?3988292384^e>>>1:e>>>1;t[n]=e}return t},jW=new Uint32Array(NW()),VW=(e,t,n,r)=>{const i=jW,l=r+n;e^=-1;for(let g=r;g>>8^i[(e^t[g])&255];return e^-1};var Zt=VW,$a={2:"need dictionary",1:"stream end",0:"","-1":"file error","-2":"stream error","-3":"data error","-4":"insufficient memory","-5":"buffer error","-6":"incompatible version"},qo={Z_NO_FLUSH:0,Z_PARTIAL_FLUSH:1,Z_SYNC_FLUSH:2,Z_FULL_FLUSH:3,Z_FINISH:4,Z_BLOCK:5,Z_TREES:6,Z_OK:0,Z_STREAM_END:1,Z_NEED_DICT:2,Z_ERRNO:-1,Z_STREAM_ERROR:-2,Z_DATA_ERROR:-3,Z_MEM_ERROR:-4,Z_BUF_ERROR:-5,Z_NO_COMPRESSION:0,Z_BEST_SPEED:1,Z_BEST_COMPRESSION:9,Z_DEFAULT_COMPRESSION:-1,Z_FILTERED:1,Z_HUFFMAN_ONLY:2,Z_RLE:3,Z_FIXED:4,Z_DEFAULT_STRATEGY:0,Z_BINARY:0,Z_TEXT:1,Z_UNKNOWN:2,Z_DEFLATED:8};const{_tr_init:UW,_tr_stored_block:Ah,_tr_flush_block:HW,_tr_tally:$i,_tr_align:WW}=FW,{Z_NO_FLUSH:Bi,Z_PARTIAL_FLUSH:YW,Z_FULL_FLUSH:XW,Z_FINISH:Gn,Z_BLOCK:ny,Z_OK:rn,Z_STREAM_END:ry,Z_STREAM_ERROR:Pr,Z_DATA_ERROR:KW,Z_BUF_ERROR:nd,Z_DEFAULT_COMPRESSION:GW,Z_FILTERED:qW,Z_HUFFMAN_ONLY:_u,Z_RLE:ZW,Z_FIXED:JW,Z_DEFAULT_STRATEGY:QW,Z_UNKNOWN:eY,Z_DEFLATED:lf}=qo,tY=9,nY=15,rY=8,iY=29,aY=256,Ph=aY+1+iY,oY=30,sY=19,lY=2*Ph+1,uY=15,lt=3,Mi=258,Ir=Mi+lt+1,cY=32,$o=42,wm=57,Ih=69,Mh=73,Lh=91,Rh=103,wa=113,ms=666,wn=1,Zo=2,Ba=3,Jo=4,fY=3,Ca=(e,t)=>(e.msg=$a[t],t),iy=e=>e*2-(e>4?9:0),Ai=e=>{let t=e.length;for(;--t>=0;)e[t]=0},dY=e=>{let t,n,r,i=e.w_size;t=e.hash_size,r=t;do n=e.head[--r],e.head[r]=n>=i?n-i:0;while(--t);t=i,r=t;do n=e.prev[--r],e.prev[r]=n>=i?n-i:0;while(--t)};let hY=(e,t,n)=>(t<{const t=e.state;let n=t.pending;n>e.avail_out&&(n=e.avail_out),n!==0&&(e.output.set(t.pending_buf.subarray(t.pending_out,t.pending_out+n),e.next_out),e.next_out+=n,t.pending_out+=n,e.total_out+=n,e.avail_out-=n,t.pending-=n,t.pending===0&&(t.pending_out=0))},jn=(e,t)=>{HW(e,e.block_start>=0?e.block_start:-1,e.strstart-e.block_start,t),e.block_start=e.strstart,$n(e.strm)},mt=(e,t)=>{e.pending_buf[e.pending++]=t},us=(e,t)=>{e.pending_buf[e.pending++]=t>>>8&255,e.pending_buf[e.pending++]=t&255},Dh=(e,t,n,r)=>{let i=e.avail_in;return i>r&&(i=r),i===0?0:(e.avail_in-=i,t.set(e.input.subarray(e.next_in,e.next_in+i),n),e.state.wrap===1?e.adler=yl(e.adler,t,i,n):e.state.wrap===2&&(e.adler=Zt(e.adler,t,i,n)),e.next_in+=i,e.total_in+=i,i)},j2=(e,t)=>{let n=e.max_chain_length,r=e.strstart,i,l,g=e.prev_length,o=e.nice_match;const a=e.strstart>e.w_size-Ir?e.strstart-(e.w_size-Ir):0,u=e.window,d=e.w_mask,c=e.prev,f=e.strstart+Mi;let s=u[r+g-1],h=u[r+g];e.prev_length>=e.good_match&&(n>>=2),o>e.lookahead&&(o=e.lookahead);do if(i=t,!(u[i+g]!==h||u[i+g-1]!==s||u[i]!==u[r]||u[++i]!==u[r+1])){r+=2,i++;do;while(u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&rg){if(e.match_start=t,g=l,l>=o)break;s=u[r+g-1],h=u[r+g]}}while((t=c[t&d])>a&&--n!==0);return g<=e.lookahead?g:e.lookahead},Bo=e=>{const t=e.w_size;let n,r,i;do{if(r=e.window_size-e.lookahead-e.strstart,e.strstart>=t+(t-Ir)&&(e.window.set(e.window.subarray(t,t+t-r),0),e.match_start-=t,e.strstart-=t,e.block_start-=t,e.insert>e.strstart&&(e.insert=e.strstart),dY(e),r+=t),e.strm.avail_in===0)break;if(n=Dh(e.strm,e.window,e.strstart+e.lookahead,r),e.lookahead+=n,e.lookahead+e.insert>=lt)for(i=e.strstart-e.insert,e.ins_h=e.window[i],e.ins_h=Fi(e,e.ins_h,e.window[i+1]);e.insert&&(e.ins_h=Fi(e,e.ins_h,e.window[i+lt-1]),e.prev[i&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=i,i++,e.insert--,!(e.lookahead+e.insert{let n=e.pending_buf_size-5>e.w_size?e.w_size:e.pending_buf_size-5,r,i,l,g=0,o=e.strm.avail_in;do{if(r=65535,l=e.bi_valid+42>>3,e.strm.avail_outi+e.strm.avail_in&&(r=i+e.strm.avail_in),r>l&&(r=l),r>8,e.pending_buf[e.pending-2]=~r,e.pending_buf[e.pending-1]=~r>>8,$n(e.strm),i&&(i>r&&(i=r),e.strm.output.set(e.window.subarray(e.block_start,e.block_start+i),e.strm.next_out),e.strm.next_out+=i,e.strm.avail_out-=i,e.strm.total_out+=i,e.block_start+=i,r-=i),r&&(Dh(e.strm,e.strm.output,e.strm.next_out,r),e.strm.next_out+=r,e.strm.avail_out-=r,e.strm.total_out+=r)}while(g===0);return o-=e.strm.avail_in,o&&(o>=e.w_size?(e.matches=2,e.window.set(e.strm.input.subarray(e.strm.next_in-e.w_size,e.strm.next_in),0),e.strstart=e.w_size,e.insert=e.strstart):(e.window_size-e.strstart<=o&&(e.strstart-=e.w_size,e.window.set(e.window.subarray(e.w_size,e.w_size+e.strstart),0),e.matches<2&&e.matches++,e.insert>e.strstart&&(e.insert=e.strstart)),e.window.set(e.strm.input.subarray(e.strm.next_in-o,e.strm.next_in),e.strstart),e.strstart+=o,e.insert+=o>e.w_size-e.insert?e.w_size-e.insert:o),e.block_start=e.strstart),e.high_waterl&&e.block_start>=e.w_size&&(e.block_start-=e.w_size,e.strstart-=e.w_size,e.window.set(e.window.subarray(e.w_size,e.w_size+e.strstart),0),e.matches<2&&e.matches++,l+=e.w_size,e.insert>e.strstart&&(e.insert=e.strstart)),l>e.strm.avail_in&&(l=e.strm.avail_in),l&&(Dh(e.strm,e.window,e.strstart,l),e.strstart+=l,e.insert+=l>e.w_size-e.insert?e.w_size-e.insert:l),e.high_water>3,l=e.pending_buf_size-l>65535?65535:e.pending_buf_size-l,n=l>e.w_size?e.w_size:l,i=e.strstart-e.block_start,(i>=n||(i||t===Gn)&&t!==Bi&&e.strm.avail_in===0&&i<=l)&&(r=i>l?l:i,g=t===Gn&&e.strm.avail_in===0&&r===i?1:0,Ah(e,e.block_start,r,g),e.block_start+=r,$n(e.strm)),g?Ba:wn)},rd=(e,t)=>{let n,r;for(;;){if(e.lookahead=lt&&(e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart),n!==0&&e.strstart-n<=e.w_size-Ir&&(e.match_length=j2(e,n)),e.match_length>=lt)if(r=$i(e,e.strstart-e.match_start,e.match_length-lt),e.lookahead-=e.match_length,e.match_length<=e.max_lazy_match&&e.lookahead>=lt){e.match_length--;do e.strstart++,e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart;while(--e.match_length!==0);e.strstart++}else e.strstart+=e.match_length,e.match_length=0,e.ins_h=e.window[e.strstart],e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+1]);else r=$i(e,0,e.window[e.strstart]),e.lookahead--,e.strstart++;if(r&&(jn(e,!1),e.strm.avail_out===0))return wn}return e.insert=e.strstart{let n,r,i;for(;;){if(e.lookahead=lt&&(e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart),e.prev_length=e.match_length,e.prev_match=e.match_start,e.match_length=lt-1,n!==0&&e.prev_length4096)&&(e.match_length=lt-1)),e.prev_length>=lt&&e.match_length<=e.prev_length){i=e.strstart+e.lookahead-lt,r=$i(e,e.strstart-1-e.prev_match,e.prev_length-lt),e.lookahead-=e.prev_length-1,e.prev_length-=2;do++e.strstart<=i&&(e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart);while(--e.prev_length!==0);if(e.match_available=0,e.match_length=lt-1,e.strstart++,r&&(jn(e,!1),e.strm.avail_out===0))return wn}else if(e.match_available){if(r=$i(e,0,e.window[e.strstart-1]),r&&jn(e,!1),e.strstart++,e.lookahead--,e.strm.avail_out===0)return wn}else e.match_available=1,e.strstart++,e.lookahead--}return e.match_available&&(r=$i(e,0,e.window[e.strstart-1]),e.match_available=0),e.insert=e.strstart{let n,r,i,l;const g=e.window;for(;;){if(e.lookahead<=Mi){if(Bo(e),e.lookahead<=Mi&&t===Bi)return wn;if(e.lookahead===0)break}if(e.match_length=0,e.lookahead>=lt&&e.strstart>0&&(i=e.strstart-1,r=g[i],r===g[++i]&&r===g[++i]&&r===g[++i])){l=e.strstart+Mi;do;while(r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&ie.lookahead&&(e.match_length=e.lookahead)}if(e.match_length>=lt?(n=$i(e,1,e.match_length-lt),e.lookahead-=e.match_length,e.strstart+=e.match_length,e.match_length=0):(n=$i(e,0,e.window[e.strstart]),e.lookahead--,e.strstart++),n&&(jn(e,!1),e.strm.avail_out===0))return wn}return e.insert=0,t===Gn?(jn(e,!0),e.strm.avail_out===0?Ba:Jo):e.sym_next&&(jn(e,!1),e.strm.avail_out===0)?wn:Zo},mY=(e,t)=>{let n;for(;;){if(e.lookahead===0&&(Bo(e),e.lookahead===0)){if(t===Bi)return wn;break}if(e.match_length=0,n=$i(e,0,e.window[e.strstart]),e.lookahead--,e.strstart++,n&&(jn(e,!1),e.strm.avail_out===0))return wn}return e.insert=0,t===Gn?(jn(e,!0),e.strm.avail_out===0?Ba:Jo):e.sym_next&&(jn(e,!1),e.strm.avail_out===0)?wn:Zo};function wr(e,t,n,r,i){this.good_length=e,this.max_lazy=t,this.nice_length=n,this.max_chain=r,this.func=i}const gs=[new wr(0,0,0,0,V2),new wr(4,4,8,4,rd),new wr(4,5,16,8,rd),new wr(4,6,32,32,rd),new wr(4,4,16,16,ao),new wr(8,16,32,32,ao),new wr(8,16,128,128,ao),new wr(8,32,128,256,ao),new wr(32,128,258,1024,ao),new wr(32,258,258,4096,ao)],gY=e=>{e.window_size=2*e.w_size,Ai(e.head),e.max_lazy_match=gs[e.level].max_lazy,e.good_match=gs[e.level].good_length,e.nice_match=gs[e.level].nice_length,e.max_chain_length=gs[e.level].max_chain,e.strstart=0,e.block_start=0,e.lookahead=0,e.insert=0,e.match_length=e.prev_length=lt-1,e.match_available=0,e.ins_h=0};function vY(){this.strm=null,this.status=0,this.pending_buf=null,this.pending_buf_size=0,this.pending_out=0,this.pending=0,this.wrap=0,this.gzhead=null,this.gzindex=0,this.method=lf,this.last_flush=-1,this.w_size=0,this.w_bits=0,this.w_mask=0,this.window=null,this.window_size=0,this.prev=null,this.head=null,this.ins_h=0,this.hash_size=0,this.hash_bits=0,this.hash_mask=0,this.hash_shift=0,this.block_start=0,this.match_length=0,this.prev_match=0,this.match_available=0,this.strstart=0,this.match_start=0,this.lookahead=0,this.prev_length=0,this.max_chain_length=0,this.max_lazy_match=0,this.level=0,this.strategy=0,this.good_match=0,this.nice_match=0,this.dyn_ltree=new Uint16Array(lY*2),this.dyn_dtree=new Uint16Array((2*oY+1)*2),this.bl_tree=new Uint16Array((2*sY+1)*2),Ai(this.dyn_ltree),Ai(this.dyn_dtree),Ai(this.bl_tree),this.l_desc=null,this.d_desc=null,this.bl_desc=null,this.bl_count=new Uint16Array(uY+1),this.heap=new Uint16Array(2*Ph+1),Ai(this.heap),this.heap_len=0,this.heap_max=0,this.depth=new Uint16Array(2*Ph+1),Ai(this.depth),this.sym_buf=0,this.lit_bufsize=0,this.sym_next=0,this.sym_end=0,this.opt_len=0,this.static_len=0,this.matches=0,this.insert=0,this.bi_buf=0,this.bi_valid=0}const Ul=e=>{if(!e)return 1;const t=e.state;return!t||t.strm!==e||t.status!==$o&&t.status!==wm&&t.status!==Ih&&t.status!==Mh&&t.status!==Lh&&t.status!==Rh&&t.status!==wa&&t.status!==ms?1:0},U2=e=>{if(Ul(e))return Ca(e,Pr);e.total_in=e.total_out=0,e.data_type=eY;const t=e.state;return t.pending=0,t.pending_out=0,t.wrap<0&&(t.wrap=-t.wrap),t.status=t.wrap===2?wm:t.wrap?$o:wa,e.adler=t.wrap===2?0:1,t.last_flush=-2,UW(t),rn},H2=e=>{const t=U2(e);return t===rn&&gY(e.state),t},yY=(e,t)=>Ul(e)||e.state.wrap!==2?Pr:(e.state.gzhead=t,rn),W2=(e,t,n,r,i,l)=>{if(!e)return Pr;let g=1;if(t===GW&&(t=6),r<0?(g=0,r=-r):r>15&&(g=2,r-=16),i<1||i>tY||n!==lf||r<8||r>15||t<0||t>9||l<0||l>JW||r===8&&g!==1)return Ca(e,Pr);r===8&&(r=9);const o=new vY;return e.state=o,o.strm=e,o.status=$o,o.wrap=g,o.gzhead=null,o.w_bits=r,o.w_size=1<W2(e,t,lf,nY,rY,QW),_Y=(e,t)=>{if(Ul(e)||t>ny||t<0)return e?Ca(e,Pr):Pr;const n=e.state;if(!e.output||e.avail_in!==0&&!e.input||n.status===ms&&t!==Gn)return Ca(e,e.avail_out===0?nd:Pr);const r=n.last_flush;if(n.last_flush=t,n.pending!==0){if($n(e),e.avail_out===0)return n.last_flush=-1,rn}else if(e.avail_in===0&&iy(t)<=iy(r)&&t!==Gn)return Ca(e,nd);if(n.status===ms&&e.avail_in!==0)return Ca(e,nd);if(n.status===$o&&n.wrap===0&&(n.status=wa),n.status===$o){let i=lf+(n.w_bits-8<<4)<<8,l=-1;if(n.strategy>=_u||n.level<2?l=0:n.level<6?l=1:n.level===6?l=2:l=3,i|=l<<6,n.strstart!==0&&(i|=cY),i+=31-i%31,us(n,i),n.strstart!==0&&(us(n,e.adler>>>16),us(n,e.adler&65535)),e.adler=1,n.status=wa,$n(e),n.pending!==0)return n.last_flush=-1,rn}if(n.status===wm){if(e.adler=0,mt(n,31),mt(n,139),mt(n,8),n.gzhead)mt(n,(n.gzhead.text?1:0)+(n.gzhead.hcrc?2:0)+(n.gzhead.extra?4:0)+(n.gzhead.name?8:0)+(n.gzhead.comment?16:0)),mt(n,n.gzhead.time&255),mt(n,n.gzhead.time>>8&255),mt(n,n.gzhead.time>>16&255),mt(n,n.gzhead.time>>24&255),mt(n,n.level===9?2:n.strategy>=_u||n.level<2?4:0),mt(n,n.gzhead.os&255),n.gzhead.extra&&n.gzhead.extra.length&&(mt(n,n.gzhead.extra.length&255),mt(n,n.gzhead.extra.length>>8&255)),n.gzhead.hcrc&&(e.adler=Zt(e.adler,n.pending_buf,n.pending,0)),n.gzindex=0,n.status=Ih;else if(mt(n,0),mt(n,0),mt(n,0),mt(n,0),mt(n,0),mt(n,n.level===9?2:n.strategy>=_u||n.level<2?4:0),mt(n,fY),n.status=wa,$n(e),n.pending!==0)return n.last_flush=-1,rn}if(n.status===Ih){if(n.gzhead.extra){let i=n.pending,l=(n.gzhead.extra.length&65535)-n.gzindex;for(;n.pending+l>n.pending_buf_size;){let o=n.pending_buf_size-n.pending;if(n.pending_buf.set(n.gzhead.extra.subarray(n.gzindex,n.gzindex+o),n.pending),n.pending=n.pending_buf_size,n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),n.gzindex+=o,$n(e),n.pending!==0)return n.last_flush=-1,rn;i=0,l-=o}let g=new Uint8Array(n.gzhead.extra);n.pending_buf.set(g.subarray(n.gzindex,n.gzindex+l),n.pending),n.pending+=l,n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),n.gzindex=0}n.status=Mh}if(n.status===Mh){if(n.gzhead.name){let i=n.pending,l;do{if(n.pending===n.pending_buf_size){if(n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),$n(e),n.pending!==0)return n.last_flush=-1,rn;i=0}n.gzindexi&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),n.gzindex=0}n.status=Lh}if(n.status===Lh){if(n.gzhead.comment){let i=n.pending,l;do{if(n.pending===n.pending_buf_size){if(n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),$n(e),n.pending!==0)return n.last_flush=-1,rn;i=0}n.gzindexi&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i))}n.status=Rh}if(n.status===Rh){if(n.gzhead.hcrc){if(n.pending+2>n.pending_buf_size&&($n(e),n.pending!==0))return n.last_flush=-1,rn;mt(n,e.adler&255),mt(n,e.adler>>8&255),e.adler=0}if(n.status=wa,$n(e),n.pending!==0)return n.last_flush=-1,rn}if(e.avail_in!==0||n.lookahead!==0||t!==Bi&&n.status!==ms){let i=n.level===0?V2(n,t):n.strategy===_u?mY(n,t):n.strategy===ZW?pY(n,t):gs[n.level].func(n,t);if((i===Ba||i===Jo)&&(n.status=ms),i===wn||i===Ba)return e.avail_out===0&&(n.last_flush=-1),rn;if(i===Zo&&(t===YW?WW(n):t!==ny&&(Ah(n,0,0,!1),t===XW&&(Ai(n.head),n.lookahead===0&&(n.strstart=0,n.block_start=0,n.insert=0))),$n(e),e.avail_out===0))return n.last_flush=-1,rn}return t!==Gn?rn:n.wrap<=0?ry:(n.wrap===2?(mt(n,e.adler&255),mt(n,e.adler>>8&255),mt(n,e.adler>>16&255),mt(n,e.adler>>24&255),mt(n,e.total_in&255),mt(n,e.total_in>>8&255),mt(n,e.total_in>>16&255),mt(n,e.total_in>>24&255)):(us(n,e.adler>>>16),us(n,e.adler&65535)),$n(e),n.wrap>0&&(n.wrap=-n.wrap),n.pending!==0?rn:ry)},wY=e=>{if(Ul(e))return Pr;const t=e.state.status;return e.state=null,t===wa?Ca(e,KW):rn},CY=(e,t)=>{let n=t.length;if(Ul(e))return Pr;const r=e.state,i=r.wrap;if(i===2||i===1&&r.status!==$o||r.lookahead)return Pr;if(i===1&&(e.adler=yl(e.adler,t,n,0)),r.wrap=0,n>=r.w_size){i===0&&(Ai(r.head),r.strstart=0,r.block_start=0,r.insert=0);let a=new Uint8Array(r.w_size);a.set(t.subarray(n-r.w_size,n),0),t=a,n=r.w_size}const l=e.avail_in,g=e.next_in,o=e.input;for(e.avail_in=n,e.next_in=0,e.input=t,Bo(r);r.lookahead>=lt;){let a=r.strstart,u=r.lookahead-(lt-1);do r.ins_h=Fi(r,r.ins_h,r.window[a+lt-1]),r.prev[a&r.w_mask]=r.head[r.ins_h],r.head[r.ins_h]=a,a++;while(--u);r.strstart=a,r.lookahead=lt-1,Bo(r)}return r.strstart+=r.lookahead,r.block_start=r.strstart,r.insert=r.lookahead,r.lookahead=0,r.match_length=r.prev_length=lt-1,r.match_available=0,e.next_in=g,e.input=o,e.avail_in=l,r.wrap=i,rn};var SY=bY,xY=W2,TY=H2,kY=U2,EY=yY,OY=_Y,AY=wY,PY=CY,IY="pako deflate (from Nodeca project)",Ds={deflateInit:SY,deflateInit2:xY,deflateReset:TY,deflateResetKeep:kY,deflateSetHeader:EY,deflate:OY,deflateEnd:AY,deflateSetDictionary:PY,deflateInfo:IY};const MY=(e,t)=>Object.prototype.hasOwnProperty.call(e,t);var LY=function(e){const t=Array.prototype.slice.call(arguments,1);for(;t.length;){const n=t.shift();if(!!n){if(typeof n!="object")throw new TypeError(n+"must be non-object");for(const r in n)MY(n,r)&&(e[r]=n[r])}}return e},RY=e=>{let t=0;for(let r=0,i=e.length;r=252?6:e>=248?5:e>=240?4:e>=224?3:e>=192?2:1;bl[254]=bl[254]=1;var DY=e=>{if(typeof TextEncoder=="function"&&TextEncoder.prototype.encode)return new TextEncoder().encode(e);let t,n,r,i,l,g=e.length,o=0;for(i=0;i>>6,t[l++]=128|n&63):n<65536?(t[l++]=224|n>>>12,t[l++]=128|n>>>6&63,t[l++]=128|n&63):(t[l++]=240|n>>>18,t[l++]=128|n>>>12&63,t[l++]=128|n>>>6&63,t[l++]=128|n&63);return t};const $Y=(e,t)=>{if(t<65534&&e.subarray&&Y2)return String.fromCharCode.apply(null,e.length===t?e:e.subarray(0,t));let n="";for(let r=0;r{const n=t||e.length;if(typeof TextDecoder=="function"&&TextDecoder.prototype.decode)return new TextDecoder().decode(e.subarray(0,t));let r,i;const l=new Array(n*2);for(i=0,r=0;r4){l[i++]=65533,r+=o-1;continue}for(g&=o===2?31:o===3?15:7;o>1&&r1){l[i++]=65533;continue}g<65536?l[i++]=g:(g-=65536,l[i++]=55296|g>>10&1023,l[i++]=56320|g&1023)}return $Y(l,i)},FY=(e,t)=>{t=t||e.length,t>e.length&&(t=e.length);let n=t-1;for(;n>=0&&(e[n]&192)===128;)n--;return n<0||n===0?t:n+bl[e[n]]>t?n:t},_l={string2buf:DY,buf2string:BY,utf8border:FY};function zY(){this.input=null,this.next_in=0,this.avail_in=0,this.total_in=0,this.output=null,this.next_out=0,this.avail_out=0,this.total_out=0,this.msg="",this.state=null,this.data_type=2,this.adler=0}var X2=zY;const K2=Object.prototype.toString,{Z_NO_FLUSH:NY,Z_SYNC_FLUSH:jY,Z_FULL_FLUSH:VY,Z_FINISH:UY,Z_OK:Ec,Z_STREAM_END:HY,Z_DEFAULT_COMPRESSION:WY,Z_DEFAULT_STRATEGY:YY,Z_DEFLATED:XY}=qo;function Hl(e){this.options=uf.assign({level:WY,method:XY,chunkSize:16384,windowBits:15,memLevel:8,strategy:YY},e||{});let t=this.options;t.raw&&t.windowBits>0?t.windowBits=-t.windowBits:t.gzip&&t.windowBits>0&&t.windowBits<16&&(t.windowBits+=16),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new X2,this.strm.avail_out=0;let n=Ds.deflateInit2(this.strm,t.level,t.method,t.windowBits,t.memLevel,t.strategy);if(n!==Ec)throw new Error($a[n]);if(t.header&&Ds.deflateSetHeader(this.strm,t.header),t.dictionary){let r;if(typeof t.dictionary=="string"?r=_l.string2buf(t.dictionary):K2.call(t.dictionary)==="[object ArrayBuffer]"?r=new Uint8Array(t.dictionary):r=t.dictionary,n=Ds.deflateSetDictionary(this.strm,r),n!==Ec)throw new Error($a[n]);this._dict_set=!0}}Hl.prototype.push=function(e,t){const n=this.strm,r=this.options.chunkSize;let i,l;if(this.ended)return!1;for(t===~~t?l=t:l=t===!0?UY:NY,typeof e=="string"?n.input=_l.string2buf(e):K2.call(e)==="[object ArrayBuffer]"?n.input=new Uint8Array(e):n.input=e,n.next_in=0,n.avail_in=n.input.length;;){if(n.avail_out===0&&(n.output=new Uint8Array(r),n.next_out=0,n.avail_out=r),(l===jY||l===VY)&&n.avail_out<=6){this.onData(n.output.subarray(0,n.next_out)),n.avail_out=0;continue}if(i=Ds.deflate(n,l),i===HY)return n.next_out>0&&this.onData(n.output.subarray(0,n.next_out)),i=Ds.deflateEnd(this.strm),this.onEnd(i),this.ended=!0,i===Ec;if(n.avail_out===0){this.onData(n.output);continue}if(l>0&&n.next_out>0){this.onData(n.output.subarray(0,n.next_out)),n.avail_out=0;continue}if(n.avail_in===0)break}return!0};Hl.prototype.onData=function(e){this.chunks.push(e)};Hl.prototype.onEnd=function(e){e===Ec&&(this.result=uf.flattenChunks(this.chunks)),this.chunks=[],this.err=e,this.msg=this.strm.msg};function Cm(e,t){const n=new Hl(t);if(n.push(e,!0),n.err)throw n.msg||$a[n.err];return n.result}function KY(e,t){return t=t||{},t.raw=!0,Cm(e,t)}function GY(e,t){return t=t||{},t.gzip=!0,Cm(e,t)}var qY=Hl,ZY=Cm,JY=KY,QY=GY,eX=qo,tX={Deflate:qY,deflate:ZY,deflateRaw:JY,gzip:QY,constants:eX};const wu=16209,nX=16191;var rX=function(t,n){let r,i,l,g,o,a,u,d,c,f,s,h,v,m,y,b,_,w,S,x,T,E,I,D;const N=t.state;r=t.next_in,I=t.input,i=r+(t.avail_in-5),l=t.next_out,D=t.output,g=l-(n-t.avail_out),o=l+(t.avail_out-257),a=N.dmax,u=N.wsize,d=N.whave,c=N.wnext,f=N.window,s=N.hold,h=N.bits,v=N.lencode,m=N.distcode,y=(1<>>24,s>>>=w,h-=w,w=_>>>16&255,w===0)D[l++]=_&65535;else if(w&16){S=_&65535,w&=15,w&&(h>>=w,h-=w),h<15&&(s+=I[r++]<>>24,s>>>=w,h-=w,w=_>>>16&255,w&16){if(x=_&65535,w&=15,ha){t.msg="invalid distance too far back",N.mode=wu;break e}if(s>>>=w,h-=w,w=l-g,x>w){if(w=x-w,w>d&&N.sane){t.msg="invalid distance too far back",N.mode=wu;break e}if(T=0,E=f,c===0){if(T+=u-w,w2;)D[l++]=E[T++],D[l++]=E[T++],D[l++]=E[T++],S-=3;S&&(D[l++]=E[T++],S>1&&(D[l++]=E[T++]))}else{T=l-x;do D[l++]=D[T++],D[l++]=D[T++],D[l++]=D[T++],S-=3;while(S>2);S&&(D[l++]=D[T++],S>1&&(D[l++]=D[T++]))}}else if((w&64)===0){_=m[(_&65535)+(s&(1<>3,r-=S,h-=S<<3,s&=(1<{const a=o.bits;let u=0,d=0,c=0,f=0,s=0,h=0,v=0,m=0,y=0,b=0,_,w,S,x,T,E=null,I;const D=new Uint16Array(oo+1),N=new Uint16Array(oo+1);let L=null,F,O,V;for(u=0;u<=oo;u++)D[u]=0;for(d=0;d=1&&D[f]===0;f--);if(s>f&&(s=f),f===0)return i[l++]=1<<24|64<<16|0,i[l++]=1<<24|64<<16|0,o.bits=1,0;for(c=1;c0&&(e===sy||f!==1))return-1;for(N[1]=0,u=1;uay||e===ly&&y>oy)return 1;for(;;){F=u-v,g[d]+1=I?(O=L[g[d]-I],V=E[g[d]-I]):(O=32+64,V=0),_=1<>v)+w]=F<<24|O<<16|V|0;while(w!==0);for(_=1<>=1;if(_!==0?(b&=_-1,b+=_):b=0,d++,--D[u]===0){if(u===f)break;u=t[n+g[d]]}if(u>s&&(b&x)!==S){for(v===0&&(v=s),T+=c,h=u-v,m=1<ay||e===ly&&y>oy)return 1;S=b&x,i[S]=s<<24|h<<16|T-l|0}}return b!==0&&(i[T+b]=u-v<<24|64<<16|0),o.bits=s,0};var $s=lX;const uX=0,G2=1,q2=2,{Z_FINISH:uy,Z_BLOCK:cX,Z_TREES:Cu,Z_OK:Fa,Z_STREAM_END:fX,Z_NEED_DICT:dX,Z_STREAM_ERROR:tr,Z_DATA_ERROR:Z2,Z_MEM_ERROR:J2,Z_BUF_ERROR:hX,Z_DEFLATED:cy}=qo,cf=16180,fy=16181,dy=16182,hy=16183,py=16184,my=16185,gy=16186,vy=16187,yy=16188,by=16189,Oc=16190,Ur=16191,ad=16192,_y=16193,od=16194,wy=16195,Cy=16196,Sy=16197,xy=16198,Su=16199,xu=16200,Ty=16201,ky=16202,Ey=16203,Oy=16204,Ay=16205,sd=16206,Py=16207,Iy=16208,Bt=16209,Q2=16210,eC=16211,pX=852,mX=592,gX=15,vX=gX,My=e=>(e>>>24&255)+(e>>>8&65280)+((e&65280)<<8)+((e&255)<<24);function yX(){this.strm=null,this.mode=0,this.last=!1,this.wrap=0,this.havedict=!1,this.flags=0,this.dmax=0,this.check=0,this.total=0,this.head=null,this.wbits=0,this.wsize=0,this.whave=0,this.wnext=0,this.window=null,this.hold=0,this.bits=0,this.length=0,this.offset=0,this.extra=0,this.lencode=null,this.distcode=null,this.lenbits=0,this.distbits=0,this.ncode=0,this.nlen=0,this.ndist=0,this.have=0,this.next=null,this.lens=new Uint16Array(320),this.work=new Uint16Array(288),this.lendyn=null,this.distdyn=null,this.sane=0,this.back=0,this.was=0}const Ga=e=>{if(!e)return 1;const t=e.state;return!t||t.strm!==e||t.modeeC?1:0},tC=e=>{if(Ga(e))return tr;const t=e.state;return e.total_in=e.total_out=t.total=0,e.msg="",t.wrap&&(e.adler=t.wrap&1),t.mode=cf,t.last=0,t.havedict=0,t.flags=-1,t.dmax=32768,t.head=null,t.hold=0,t.bits=0,t.lencode=t.lendyn=new Int32Array(pX),t.distcode=t.distdyn=new Int32Array(mX),t.sane=1,t.back=-1,Fa},nC=e=>{if(Ga(e))return tr;const t=e.state;return t.wsize=0,t.whave=0,t.wnext=0,tC(e)},rC=(e,t)=>{let n;if(Ga(e))return tr;const r=e.state;return t<0?(n=0,t=-t):(n=(t>>4)+5,t<48&&(t&=15)),t&&(t<8||t>15)?tr:(r.window!==null&&r.wbits!==t&&(r.window=null),r.wrap=n,r.wbits=t,nC(e))},iC=(e,t)=>{if(!e)return tr;const n=new yX;e.state=n,n.strm=e,n.window=null,n.mode=cf;const r=rC(e,t);return r!==Fa&&(e.state=null),r},bX=e=>iC(e,vX);let Ly=!0,ld,ud;const _X=e=>{if(Ly){ld=new Int32Array(512),ud=new Int32Array(32);let t=0;for(;t<144;)e.lens[t++]=8;for(;t<256;)e.lens[t++]=9;for(;t<280;)e.lens[t++]=7;for(;t<288;)e.lens[t++]=8;for($s(G2,e.lens,0,288,ld,0,e.work,{bits:9}),t=0;t<32;)e.lens[t++]=5;$s(q2,e.lens,0,32,ud,0,e.work,{bits:5}),Ly=!1}e.lencode=ld,e.lenbits=9,e.distcode=ud,e.distbits=5},aC=(e,t,n,r)=>{let i;const l=e.state;return l.window===null&&(l.wsize=1<=l.wsize?(l.window.set(t.subarray(n-l.wsize,n),0),l.wnext=0,l.whave=l.wsize):(i=l.wsize-l.wnext,i>r&&(i=r),l.window.set(t.subarray(n-r,n-r+i),l.wnext),r-=i,r?(l.window.set(t.subarray(n-r,n),0),l.wnext=r,l.whave=l.wsize):(l.wnext+=i,l.wnext===l.wsize&&(l.wnext=0),l.whave{let n,r,i,l,g,o,a,u,d,c,f,s,h,v,m=0,y,b,_,w,S,x,T,E;const I=new Uint8Array(4);let D,N;const L=new Uint8Array([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]);if(Ga(e)||!e.output||!e.input&&e.avail_in!==0)return tr;n=e.state,n.mode===Ur&&(n.mode=ad),g=e.next_out,i=e.output,a=e.avail_out,l=e.next_in,r=e.input,o=e.avail_in,u=n.hold,d=n.bits,c=o,f=a,E=Fa;e:for(;;)switch(n.mode){case cf:if(n.wrap===0){n.mode=ad;break}for(;d<16;){if(o===0)break e;o--,u+=r[l++]<>>8&255,n.check=Zt(n.check,I,2,0),u=0,d=0,n.mode=fy;break}if(n.head&&(n.head.done=!1),!(n.wrap&1)||(((u&255)<<8)+(u>>8))%31){e.msg="incorrect header check",n.mode=Bt;break}if((u&15)!==cy){e.msg="unknown compression method",n.mode=Bt;break}if(u>>>=4,d-=4,T=(u&15)+8,n.wbits===0&&(n.wbits=T),T>15||T>n.wbits){e.msg="invalid window size",n.mode=Bt;break}n.dmax=1<>8&1),n.flags&512&&n.wrap&4&&(I[0]=u&255,I[1]=u>>>8&255,n.check=Zt(n.check,I,2,0)),u=0,d=0,n.mode=dy;case dy:for(;d<32;){if(o===0)break e;o--,u+=r[l++]<>>8&255,I[2]=u>>>16&255,I[3]=u>>>24&255,n.check=Zt(n.check,I,4,0)),u=0,d=0,n.mode=hy;case hy:for(;d<16;){if(o===0)break e;o--,u+=r[l++]<>8),n.flags&512&&n.wrap&4&&(I[0]=u&255,I[1]=u>>>8&255,n.check=Zt(n.check,I,2,0)),u=0,d=0,n.mode=py;case py:if(n.flags&1024){for(;d<16;){if(o===0)break e;o--,u+=r[l++]<>>8&255,n.check=Zt(n.check,I,2,0)),u=0,d=0}else n.head&&(n.head.extra=null);n.mode=my;case my:if(n.flags&1024&&(s=n.length,s>o&&(s=o),s&&(n.head&&(T=n.head.extra_len-n.length,n.head.extra||(n.head.extra=new Uint8Array(n.head.extra_len)),n.head.extra.set(r.subarray(l,l+s),T)),n.flags&512&&n.wrap&4&&(n.check=Zt(n.check,r,s,l)),o-=s,l+=s,n.length-=s),n.length))break e;n.length=0,n.mode=gy;case gy:if(n.flags&2048){if(o===0)break e;s=0;do T=r[l+s++],n.head&&T&&n.length<65536&&(n.head.name+=String.fromCharCode(T));while(T&&s>9&1,n.head.done=!0),e.adler=n.check=0,n.mode=Ur;break;case by:for(;d<32;){if(o===0)break e;o--,u+=r[l++]<>>=d&7,d-=d&7,n.mode=sd;break}for(;d<3;){if(o===0)break e;o--,u+=r[l++]<>>=1,d-=1,u&3){case 0:n.mode=_y;break;case 1:if(_X(n),n.mode=Su,t===Cu){u>>>=2,d-=2;break e}break;case 2:n.mode=Cy;break;case 3:e.msg="invalid block type",n.mode=Bt}u>>>=2,d-=2;break;case _y:for(u>>>=d&7,d-=d&7;d<32;){if(o===0)break e;o--,u+=r[l++]<>>16^65535)){e.msg="invalid stored block lengths",n.mode=Bt;break}if(n.length=u&65535,u=0,d=0,n.mode=od,t===Cu)break e;case od:n.mode=wy;case wy:if(s=n.length,s){if(s>o&&(s=o),s>a&&(s=a),s===0)break e;i.set(r.subarray(l,l+s),g),o-=s,l+=s,a-=s,g+=s,n.length-=s;break}n.mode=Ur;break;case Cy:for(;d<14;){if(o===0)break e;o--,u+=r[l++]<>>=5,d-=5,n.ndist=(u&31)+1,u>>>=5,d-=5,n.ncode=(u&15)+4,u>>>=4,d-=4,n.nlen>286||n.ndist>30){e.msg="too many length or distance symbols",n.mode=Bt;break}n.have=0,n.mode=Sy;case Sy:for(;n.have>>=3,d-=3}for(;n.have<19;)n.lens[L[n.have++]]=0;if(n.lencode=n.lendyn,n.lenbits=7,D={bits:n.lenbits},E=$s(uX,n.lens,0,19,n.lencode,0,n.work,D),n.lenbits=D.bits,E){e.msg="invalid code lengths set",n.mode=Bt;break}n.have=0,n.mode=xy;case xy:for(;n.have>>24,b=m>>>16&255,_=m&65535,!(y<=d);){if(o===0)break e;o--,u+=r[l++]<>>=y,d-=y,n.lens[n.have++]=_;else{if(_===16){for(N=y+2;d>>=y,d-=y,n.have===0){e.msg="invalid bit length repeat",n.mode=Bt;break}T=n.lens[n.have-1],s=3+(u&3),u>>>=2,d-=2}else if(_===17){for(N=y+3;d>>=y,d-=y,T=0,s=3+(u&7),u>>>=3,d-=3}else{for(N=y+7;d>>=y,d-=y,T=0,s=11+(u&127),u>>>=7,d-=7}if(n.have+s>n.nlen+n.ndist){e.msg="invalid bit length repeat",n.mode=Bt;break}for(;s--;)n.lens[n.have++]=T}}if(n.mode===Bt)break;if(n.lens[256]===0){e.msg="invalid code -- missing end-of-block",n.mode=Bt;break}if(n.lenbits=9,D={bits:n.lenbits},E=$s(G2,n.lens,0,n.nlen,n.lencode,0,n.work,D),n.lenbits=D.bits,E){e.msg="invalid literal/lengths set",n.mode=Bt;break}if(n.distbits=6,n.distcode=n.distdyn,D={bits:n.distbits},E=$s(q2,n.lens,n.nlen,n.ndist,n.distcode,0,n.work,D),n.distbits=D.bits,E){e.msg="invalid distances set",n.mode=Bt;break}if(n.mode=Su,t===Cu)break e;case Su:n.mode=xu;case xu:if(o>=6&&a>=258){e.next_out=g,e.avail_out=a,e.next_in=l,e.avail_in=o,n.hold=u,n.bits=d,rX(e,f),g=e.next_out,i=e.output,a=e.avail_out,l=e.next_in,r=e.input,o=e.avail_in,u=n.hold,d=n.bits,n.mode===Ur&&(n.back=-1);break}for(n.back=0;m=n.lencode[u&(1<>>24,b=m>>>16&255,_=m&65535,!(y<=d);){if(o===0)break e;o--,u+=r[l++]<>w)],y=m>>>24,b=m>>>16&255,_=m&65535,!(w+y<=d);){if(o===0)break e;o--,u+=r[l++]<>>=w,d-=w,n.back+=w}if(u>>>=y,d-=y,n.back+=y,n.length=_,b===0){n.mode=Ay;break}if(b&32){n.back=-1,n.mode=Ur;break}if(b&64){e.msg="invalid literal/length code",n.mode=Bt;break}n.extra=b&15,n.mode=Ty;case Ty:if(n.extra){for(N=n.extra;d>>=n.extra,d-=n.extra,n.back+=n.extra}n.was=n.length,n.mode=ky;case ky:for(;m=n.distcode[u&(1<>>24,b=m>>>16&255,_=m&65535,!(y<=d);){if(o===0)break e;o--,u+=r[l++]<>w)],y=m>>>24,b=m>>>16&255,_=m&65535,!(w+y<=d);){if(o===0)break e;o--,u+=r[l++]<>>=w,d-=w,n.back+=w}if(u>>>=y,d-=y,n.back+=y,b&64){e.msg="invalid distance code",n.mode=Bt;break}n.offset=_,n.extra=b&15,n.mode=Ey;case Ey:if(n.extra){for(N=n.extra;d>>=n.extra,d-=n.extra,n.back+=n.extra}if(n.offset>n.dmax){e.msg="invalid distance too far back",n.mode=Bt;break}n.mode=Oy;case Oy:if(a===0)break e;if(s=f-a,n.offset>s){if(s=n.offset-s,s>n.whave&&n.sane){e.msg="invalid distance too far back",n.mode=Bt;break}s>n.wnext?(s-=n.wnext,h=n.wsize-s):h=n.wnext-s,s>n.length&&(s=n.length),v=n.window}else v=i,h=g-n.offset,s=n.length;s>a&&(s=a),a-=s,n.length-=s;do i[g++]=v[h++];while(--s);n.length===0&&(n.mode=xu);break;case Ay:if(a===0)break e;i[g++]=n.length,a--,n.mode=xu;break;case sd:if(n.wrap){for(;d<32;){if(o===0)break e;o--,u|=r[l++]<{if(Ga(e))return tr;let t=e.state;return t.window&&(t.window=null),e.state=null,Fa},SX=(e,t)=>{if(Ga(e))return tr;const n=e.state;return(n.wrap&2)===0?tr:(n.head=t,t.done=!1,Fa)},xX=(e,t)=>{const n=t.length;let r,i,l;return Ga(e)||(r=e.state,r.wrap!==0&&r.mode!==Oc)?tr:r.mode===Oc&&(i=1,i=yl(i,t,n,0),i!==r.check)?Z2:(l=aC(e,t,n,n),l?(r.mode=Q2,J2):(r.havedict=1,Fa))};var TX=nC,kX=rC,EX=tC,OX=bX,AX=iC,PX=wX,IX=CX,MX=SX,LX=xX,RX="pako inflate (from Nodeca project)",qr={inflateReset:TX,inflateReset2:kX,inflateResetKeep:EX,inflateInit:OX,inflateInit2:AX,inflate:PX,inflateEnd:IX,inflateGetHeader:MX,inflateSetDictionary:LX,inflateInfo:RX};function DX(){this.text=0,this.time=0,this.xflags=0,this.os=0,this.extra=null,this.extra_len=0,this.name="",this.comment="",this.hcrc=0,this.done=!1}var $X=DX;const oC=Object.prototype.toString,{Z_NO_FLUSH:BX,Z_FINISH:FX,Z_OK:wl,Z_STREAM_END:cd,Z_NEED_DICT:fd,Z_STREAM_ERROR:zX,Z_DATA_ERROR:Ry,Z_MEM_ERROR:NX}=qo;function Wl(e){this.options=uf.assign({chunkSize:1024*64,windowBits:15,to:""},e||{});const t=this.options;t.raw&&t.windowBits>=0&&t.windowBits<16&&(t.windowBits=-t.windowBits,t.windowBits===0&&(t.windowBits=-15)),t.windowBits>=0&&t.windowBits<16&&!(e&&e.windowBits)&&(t.windowBits+=32),t.windowBits>15&&t.windowBits<48&&(t.windowBits&15)===0&&(t.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new X2,this.strm.avail_out=0;let n=qr.inflateInit2(this.strm,t.windowBits);if(n!==wl)throw new Error($a[n]);if(this.header=new $X,qr.inflateGetHeader(this.strm,this.header),t.dictionary&&(typeof t.dictionary=="string"?t.dictionary=_l.string2buf(t.dictionary):oC.call(t.dictionary)==="[object ArrayBuffer]"&&(t.dictionary=new Uint8Array(t.dictionary)),t.raw&&(n=qr.inflateSetDictionary(this.strm,t.dictionary),n!==wl)))throw new Error($a[n])}Wl.prototype.push=function(e,t){const n=this.strm,r=this.options.chunkSize,i=this.options.dictionary;let l,g,o;if(this.ended)return!1;for(t===~~t?g=t:g=t===!0?FX:BX,oC.call(e)==="[object ArrayBuffer]"?n.input=new Uint8Array(e):n.input=e,n.next_in=0,n.avail_in=n.input.length;;){for(n.avail_out===0&&(n.output=new Uint8Array(r),n.next_out=0,n.avail_out=r),l=qr.inflate(n,g),l===fd&&i&&(l=qr.inflateSetDictionary(n,i),l===wl?l=qr.inflate(n,g):l===Ry&&(l=fd));n.avail_in>0&&l===cd&&n.state.wrap>0&&e[n.next_in]!==0;)qr.inflateReset(n),l=qr.inflate(n,g);switch(l){case zX:case Ry:case fd:case NX:return this.onEnd(l),this.ended=!0,!1}if(o=n.avail_out,n.next_out&&(n.avail_out===0||l===cd))if(this.options.to==="string"){let a=_l.utf8border(n.output,n.next_out),u=n.next_out-a,d=_l.buf2string(n.output,a);n.next_out=u,n.avail_out=r-u,u&&n.output.set(n.output.subarray(a,a+u),0),this.onData(d)}else this.onData(n.output.length===n.next_out?n.output:n.output.subarray(0,n.next_out));if(!(l===wl&&o===0)){if(l===cd)return l=qr.inflateEnd(this.strm),this.onEnd(l),this.ended=!0,!0;if(n.avail_in===0)break}}return!0};Wl.prototype.onData=function(e){this.chunks.push(e)};Wl.prototype.onEnd=function(e){e===wl&&(this.options.to==="string"?this.result=this.chunks.join(""):this.result=uf.flattenChunks(this.chunks)),this.chunks=[],this.err=e,this.msg=this.strm.msg};function Sm(e,t){const n=new Wl(t);if(n.push(e),n.err)throw n.msg||$a[n.err];return n.result}function jX(e,t){return t=t||{},t.raw=!0,Sm(e,t)}var VX=Wl,UX=Sm,HX=jX,WX=Sm,YX=qo,XX={Inflate:VX,inflate:UX,inflateRaw:HX,ungzip:WX,constants:YX};const{Deflate:oq,deflate:sq,deflateRaw:KX,gzip:lq}=tX,{Inflate:uq,inflate:cq,inflateRaw:GX,ungzip:fq}=XX;var qX=KX,ZX=GX;function Dy(e){const t=new Map;for(const n of e){const[r,i]=n.split("="),l=decodeURIComponent(i);t.set(r,l)}return t}const JX=function(){if(!window.location.search.includes("?"))return;const t=window.location.search.replace("?","").split("&");let n=Dy(t);if(console.log("URL params:",n),n.get("share")){const l=ZX(new Uint8Array(atob(n.get("share")).split("").map(g=>g.charCodeAt(0))),{to:"string"});if(!l){Ut().raiseError("Error when trying to decode share parameter!",!1);return}n=Dy(l.split("&")),console.log("Share URL params:",n)}const r={id:-1,image:"",prompt:n.get("prompt")||"",sampler_name:n.get("sampler_name")||"k_euler",seed:Number(n.get("seed"))||-1,steps:Number(n.get("steps")||20),cfg_scale:Number(n.get("cfg_scale")||5),height:Number(n.get("height")||512),width:Number(n.get("width")||512),clip_skip:Number(n.get("clip_skip")||0),frames:Number(n.get("frames")||1),fps:Number(n.get("fps")||16),scheduler:n.get("scheduler")||"default"};Jt().generateText2Img(r,!1)},QX=10;function Ku(e,t,n,r,i=l=>Ut().raiseError(l,!1)){if(e.status===n&&t)return!0;if(!t.message)return i(`${r}: Got response code ${e.status}`);if(!t.errors)return i(`${r}: ${t.message}`);const l=Object.entries(t.errors).map(g=>`${g[0]} - ${g[1]}`).join(" | ");return i(`${r}: ${t.message} (${l})`)}const eK=zo("interrogate",()=>{const e=oe({}),t=oe(!1);async function n(g){Ut().raiseError(g,!1),t.value=!1,e.value={}}async function r(){const g=zt(),{source_image:o}=e.value;if(!o)return n("Failed to get interrogation ID: No image supplied.");t.value=!0;const a=await fetch(`${g.baseURL.length===0?".":g.baseURL}/sdapi/v1/interrogate`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({image:o.split(",")[1],model:"clip"})}),u=await a.json();!Ku(a,u,200,"Failed to get interrogation",n)||(e.value.id=u.id,e.value.status=u.caption)}function i(){e.value={},t.value=!1}function l(){return e.value.status||!1}return{currentInterrogation:e,interrogating:t,interrogateImage:r,getFormStatus:l,resetInterrogation:i}}),xm=e=>(ui("data-v-db184ac6"),e=e(),ci(),e),tK={key:0,style:{"margin-top":"16px"}},nK=xm(()=>ne("div",null,[He("Drop file here OR "),ne("em",null,"click to upload")],-1)),rK={key:1,style:{"margin-top":"16px"}},iK={key:2},aK={style:{"margin-top":"8px"}},oK=xm(()=>ne("h2",{style:{margin:"16px 0 8px 0"}},"Interrogation Results",-1)),sK={key:0},lK=xm(()=>ne("h3",null,"Caption",-1)),uK={key:0},cK={key:1},fK=Ee({__name:"InterrogationView",setup(e){const t=eK(),n=Jt(),r=Ut(),i=oe();async function l(u){if(i.value.clearFiles(),!u.raw.type.includes("image")){r.raiseError("Uploaded file needs to be a image!",!1);return}const d=await Tc(u.raw);t.currentInterrogation.source_image=d,t.interrogateImage()}function g(){n.generateText2Img({prompt:o.value})}const o=ee(()=>t.getFormStatus()),{ellipsis:a}=x2();return(u,d)=>C(t).currentInterrogation.source_image?C(t).currentInterrogation.status?(z(),se("div",iK,[ne("div",aK,[le(C(at),{icon:C(Mg),onClick:C(t).resetInterrogation},{default:he(()=>[He("New Interrogation")]),_:1},8,["icon","onClick"]),C(o)?(z(),be(C(at),{key:0,icon:C(Mg),onClick:g,disabled:!C(o)},{default:he(()=>[He("Text2Img (Caption)")]),_:1},8,["icon","disabled"])):ye("",!0)]),oK,le(C(ef),{src:C(t).currentInterrogation.source_image,alt:"Uploaded Image"},null,8,["src"]),C(o)?(z(),se("div",sK,[lK,C(o)?(z(),se("div",cK,[ne("strong",null,Ae(C(o)),1)])):(z(),se("div",uK,"Processing"+Ae(C(a)),1))])):ye("",!0)])):(z(),se("div",rK,[ne("strong",null,"Uploading image"+Ae(C(a)),1)])):(z(),se("div",tK,[ne("div",null,[le(C(Yp),{onChange:l,"auto-upload":!1,limit:1,class:"interrogation-upload",ref_key:"upload",ref:i,multiple:"",drag:""},{default:he(()=>[le(C(Le),{size:100},{default:he(()=>[le(C(dp))]),_:1}),nK]),_:1},512)])]))}});const dK=qt(fK,[["__scopeId","data-v-db184ac6"]]);function hK(e,t,n){if(e===0)return"0"+(t?"s":"seconds");if(e==null)return"?";const r=Math.floor(e/86400),i=Math.floor(e%86400/3600),l=Math.floor(e%86400%3600/60),g=Math.floor(e%86400%3600%60),o=r>0?r+(t?"d":"days"):"",a=i>0?i+(t?"h":"hours"):"",u=l>0?l+(t?"m":"minutes"):"",d=g>0?g+(t?"s":"seconds"):"",c=[];return n!=null&&n.days&&c.push(o),n!=null&&n.hours&&c.push(a),n!=null&&n.minutes&&c.push(u),n!=null&&n.seconds&&c.push(d),c.join(" ")}const pK={class:"form"},mK={key:0,style:{"padding-bottom":"50px"}},gK=ne("h1",{style:{margin:"0"}},"Interrogation",-1),vK=ne("div",null,"Interrogate images to get their predicted descriptions.",-1),yK={class:"sidebar"},bK={class:"reference-images-header"},_K=ne("span",{class:"reference-images-label"},"Reference Images",-1),wK={class:"reference-images-actions"},CK={key:0,class:"reference-image-list"},SK={class:"reference-image-index"},xK=["title"],TK={class:"reference-image-controls"},kK={key:1,class:"reference-image-empty"},EK={class:"main"},OK={class:"image center-horizontal"},AK={key:0},PK=Ee({__name:"GenerateView",setup(e){const n=Ep(kp).smallerOrEqual("md"),r=Jt(),i=Ut(),l=Ls();zt();const g=gv(async()=>{const m=r.cacheVersion,b=(await r.getAvailableSamplers()).map(_=>_.name);return b.length===0?[]:u(b)},[]),o=gv(async()=>{const m=r.cacheVersion,b=(await r.getAvailableSchedulers()).map(_=>_.name);return b.length===0?[]:d(b)},[]),a=yt({prompt:[{required:!0,message:"Please input prompt",trigger:"change"}]});function u(m){return!r.params||!r.params.sampler_name||m.indexOf(r.params.sampler_name)===-1&&(r.params.sampler_name=m[0]),m}function d(m){return!r.params||!r.params.scheduler||m.indexOf(r.params.scheduler)===-1&&(r.params.scheduler=m[0]),m}function c(m){return"Elapsed: "+hK(m,!0,{days:!0,hours:!0,minutes:!0,seconds:!0})}function f(){r.validGeneratorTypes.includes(r.generatorType)||(i.showGeneratorBadge=!1)}function s(m){r.generatorType=m,f(),console.log(m)}function h(){l.showCropPreview=!0,l.updateCropPreview()}function v(){var m;(m=document.getElementById("extra_image_input"))==null||m.click()}return f(),JX(),(m,y)=>(z(),se(je,null,[le(C(mw),{"default-active":C(r).generatorType,collapse:!0,onSelect:s,mode:C(n)?"horizontal":"vertical",class:de(C(n)?"mobile-generator-types":"generator-types"),style:ze(C(n)?"overflow-x: auto":"")},{default:he(()=>[le(bu,{index:"Text2Img","icon-one":C(EE),"icon-two":C(tu),isMobile:C(n)},null,8,["icon-one","icon-two","isMobile"]),le(bu,{index:"Img2Img","icon-one":C(tu),"icon-two":C(tu),isMobile:C(n)},null,8,["icon-one","icon-two","isMobile"]),le(bu,{index:"Inpainting","icon-one":S2,"icon-two":C(tu),isMobile:C(n)},null,8,["icon-two","isMobile"]),le(bu,{index:"Interrogation","icon-one":gH,isMobile:C(n)},null,8,["isMobile"])]),_:1},8,["default-active","mode","class","style"]),ne("div",pK,[C(r).generatorType==="Interrogation"?(z(),se("div",mK,[gK,vK,le(dK)])):(z(),be(C(jp),{key:1,"label-position":"left","label-width":"140px",model:C(r),class:"container",rules:a,onSubmit:y[31]||(y[31]=et(()=>{},["prevent"]))},{default:he(()=>[ne("div",yK,[le(RH),le(kh,{label:"Negative Prompt",prop:"negativePrompt",modelValue:C(r).negativePrompt,"onUpdate:modelValue":y[0]||(y[0]=b=>C(r).negativePrompt=b),autosize:{maxRows:15},resize:"vertical",type:"textarea",placeholder:"Enter negative prompt here",info:"What to exclude from the image. Not working? Try increasing the guidance.","label-position":"top"},null,8,["modelValue"]),le(kh,{label:"Seed",prop:"seed",modelValue:C(r).params.seed,"onUpdate:modelValue":y[2]||(y[2]=b=>C(r).params.seed=b),placeholder:"Enter seed here",clearable:"","clear-icon":C(gE)},{append:he(()=>[le(C(zn),{content:"Randomize!",placement:"top"},{default:he(()=>[le(C(at),{icon:C(X3),onClick:y[1]||(y[1]=()=>C(r).params.seed=C(sC)())},null,8,["icon"])]),_:1})]),_:1},8,["modelValue","clear-icon"]),(z(),be(C(b0),{key:0,gutter:10},{default:he(()=>[C(r).multiSelect.sampler.state==="Multiple"?(z(),be(C(to),{key:0,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Sampler(s)",prop:"samplers",modelValue:C(r).multiSelect.sampler.selected,"onUpdate:modelValue":y[3]||(y[3]=b=>C(r).multiSelect.sampler.selected=b),options:C(g),info:"Multi-select enabled. Heun and DPM2 double generation time per step, but converge twice as fast.",multiple:""},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0),C(r).multiSelect.sampler.state==="Enabled"?(z(),be(C(to),{key:1,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Sampler",prop:"sampler",modelValue:C(r).params.sampler_name,"onUpdate:modelValue":y[4]||(y[4]=b=>C(r).params.sampler_name=b),options:C(g),info:"Heun and DPM2 double generation time per step, but converge twice as fast."},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0),C(r).multiSelect.scheduler.state==="Multiple"?(z(),be(C(to),{key:2,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Scheduler(s)",prop:"schedulers",modelValue:C(r).multiSelect.scheduler.selected,"onUpdate:modelValue":y[5]||(y[5]=b=>C(r).multiSelect.scheduler.selected=b),options:C(o),info:"Multi-select enabled. Experimental! KoboldCpp only, allows you to use a different scheduler. Leave as default otherwise.",multiple:""},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0),C(r).multiSelect.scheduler.state==="Enabled"?(z(),be(C(to),{key:3,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Scheduler",prop:"scheduler",modelValue:C(r).params.scheduler,"onUpdate:modelValue":y[6]||(y[6]=b=>C(r).params.scheduler=b),options:C(o),info:"Experimental! KoboldCpp only, allows you to use a different scheduler. Leave as default otherwise."},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0)]),_:1})),le(bn,{label:"Batch Size",prop:"batchSize",modelValue:C(r).params.n,"onUpdate:modelValue":y[7]||(y[7]=b=>C(r).params.n=b),min:C(r).minImages,max:C(r).maxImages},null,8,["modelValue","min","max"]),C(r).multiSelect.steps.state==="Multiple"?(z(),be(bn,{key:1,label:"Steps(s)",prop:"multiSteps",modelValue:C(r).multiSelect.steps.selected,"onUpdate:modelValue":y[8]||(y[8]=b=>C(r).multiSelect.steps.selected=b),min:C(r).minSteps,max:C(r).maxSteps,info:"Multi-select enabled. Keep step count between 30 to 50 for optimal generation times. Coherence typically peaks between 60 and 90 steps, with a trade-off in speed.",multiple:""},null,8,["modelValue","min","max"])):C(r).multiSelect.steps.state==="Enabled"?(z(),be(bn,{key:2,label:"Steps",prop:"steps",modelValue:C(r).params.steps,"onUpdate:modelValue":y[9]||(y[9]=b=>C(r).params.steps=b),min:C(r).minSteps,max:C(r).maxSteps,info:"Keep step count between 30 to 50 for optimal generation times. Coherence typically peaks between 60 and 90 steps, with a trade-off in speed."},null,8,["modelValue","min","max"])):ye("",!0),le(bn,{label:"Width",prop:"width",modelValue:C(r).params.width,"onUpdate:modelValue":y[10]||(y[10]=b=>C(r).params.width=b),min:C(r).minDimensions,max:C(r).maxDimensions,step:64,change:h},null,8,["modelValue","min","max"]),le(bn,{label:"Height",prop:"height",modelValue:C(r).params.height,"onUpdate:modelValue":y[11]||(y[11]=b=>C(r).params.height=b),min:C(r).minDimensions,max:C(r).maxDimensions,step:64,change:h},null,8,["modelValue","min","max"]),C(r).multiSelect.guidance.state==="Multiple"?(z(),be(bn,{key:3,label:"Guidance(s)",prop:"cfgScales",modelValue:C(r).multiSelect.guidance.selected,"onUpdate:modelValue":y[12]||(y[12]=b=>C(r).multiSelect.guidance.selected=b),min:C(r).minCfgScale,max:C(r).maxCfgScale,info:"Multi-select enabled. Higher values will make the AI respect your prompt more. Lower values allow the AI to be more creative.",multiple:""},null,8,["modelValue","min","max"])):C(r).multiSelect.guidance.state==="Enabled"?(z(),be(bn,{key:4,label:"Guidance",prop:"cfgScale",modelValue:C(r).params.cfg_scale,"onUpdate:modelValue":y[13]||(y[13]=b=>C(r).params.cfg_scale=b),min:C(r).minCfgScale,max:C(r).maxCfgScale,step:.5,info:"Higher values will make the AI respect your prompt more. Lower values allow the AI to be more creative."},null,8,["modelValue","min","max","step"])):ye("",!0),C(r).multiSelect.eta.state==="Enabled"?(z(),be(bn,{key:5,label:"Eta",prop:"eta",modelValue:C(r).params.eta,"onUpdate:modelValue":y[14]||(y[14]=b=>C(r).params.eta=b),min:C(r).minEta,max:C(r).maxEta,step:.1,info:"Noise multiplier for ancestral samplers. 0 disables noise injection."},null,8,["modelValue","min","max","step"])):ye("",!0),C(r).multiSelect.clipSkip.state==="Multiple"?(z(),be(bn,{key:6,label:"CLIP Skip(s)",prop:"clipSkips",modelValue:C(r).multiSelect.clipSkip.selected,"onUpdate:modelValue":y[15]||(y[15]=b=>C(r).multiSelect.clipSkip.selected=b),min:C(r).minClipSkip,max:C(r).maxClipSkip,info:"Multi-select enabled. Last layers of CLIP to ignore. For most situations this can be left alone.",multiple:""},null,8,["modelValue","min","max"])):C(r).multiSelect.clipSkip.state==="Enabled"?(z(),be(bn,{key:7,label:"CLIP Skip",prop:"clipSkip",modelValue:C(r).params.clip_skip,"onUpdate:modelValue":y[16]||(y[16]=b=>C(r).params.clip_skip=b),min:C(r).minClipSkip,max:C(r).maxClipSkip,info:"Last layers of CLIP to ignore. For most situations this can be left alone."},null,8,["modelValue","min","max"])):ye("",!0),C(r).sourceGeneratorTypes.includes(C(r).generatorType)?(z(),be(bn,{key:8,label:"Init Strength",prop:"denoise",modelValue:C(r).params.denoising_strength,"onUpdate:modelValue":y[17]||(y[17]=b=>C(r).params.denoising_strength=b),min:C(r).minDenoise,max:C(r).maxDenoise,step:.01,info:"The final image will diverge from the starting image at higher values. 0=unchanged, 1=fullychanged"},null,8,["modelValue","min","max","step"])):ye("",!0),le(bn,{label:"Video Frames",prop:"frames",modelValue:C(r).params.frames,"onUpdate:modelValue":y[18]||(y[18]=b=>C(r).params.frames=b),min:C(r).minFrames,max:C(r).maxFrames,info:"Number of consecutive video frames to generate (Video models only). More frames increases memory usage."},null,8,["modelValue","min","max"]),C(r).params.frames>1?(z(),be(bn,{key:9,label:"FPS",prop:"fps",modelValue:C(r).params.fps,"onUpdate:modelValue":y[19]||(y[19]=b=>C(r).params.fps=b),min:C(r).minFps,max:C(r).maxFps,disabled:C(r).params.frames<=1,info:"Frames per second for video generation."},null,8,["modelValue","min","max","disabled"])):ye("",!0),ne("div",{class:"reference-images",onDragover:y[22]||(y[22]=et(()=>{},["prevent"])),onDrop:y[23]||(y[23]=et(b=>C(r).setExtraImage(b),["prevent"]))},[ne("div",bK,[_K,ne("div",wK,[ne("input",{class:"reference-images-input",type:"file",id:"extra_image_input",onChange:y[20]||(y[20]=b=>C(r).setExtraImage(b)),accept:"image/*",multiple:""},null,32),le(C(at),{onClick:v},{default:he(()=>[He(" Select or Drag Files ")]),_:1}),le(C(at),{onClick:y[21]||(y[21]=b=>C(r).clearExtraImage()),disabled:C(r).referenceImages.length===0},{default:he(()=>[He(" Clear Images ")]),_:1},8,["disabled"])])]),C(r).referenceImages.length>0?(z(),se("div",CK,[(z(!0),se(je,null,Nt(C(r).referenceImages,(b,_)=>(z(),se("div",{class:"reference-image-item",key:b.id},[ne("span",SK,Ae(_+1),1),le(C(ef),{class:"reference-image-thumb",src:b.dataUrl,fit:"cover","preview-src-list":C(r).referenceImages.map(w=>w.dataUrl),"initial-index":_,"preview-teleported":""},null,8,["src","preview-src-list","initial-index"]),ne("span",{class:"reference-image-name",title:b.name},Ae(b.name),9,xK),ne("div",TK,[le(C(zn),{content:"Move up",placement:"top"},{default:he(()=>[le(C(at),{class:"small-btn",icon:C(f_),disabled:_===0,onClick:w=>C(r).moveExtraImage(_,-1)},null,8,["icon","disabled","onClick"])]),_:2},1024),le(C(zn),{content:"Move down",placement:"top"},{default:he(()=>[le(C(at),{class:"small-btn",icon:C(Tl),disabled:_===C(r).referenceImages.length-1,onClick:w=>C(r).moveExtraImage(_,1)},null,8,["icon","disabled","onClick"])]),_:2},1024),le(C(zn),{content:"Remove",placement:"top"},{default:he(()=>[le(C(at),{class:"small-btn",type:"danger",icon:C(Ol),plain:"",onClick:w=>C(r).removeExtraImage(_)},null,8,["icon","onClick"])]),_:2},1024)])]))),128))])):(z(),se("div",kK," No reference images selected. "))],32),le(C(b0),null,{default:he(()=>[le(C(to),{span:C(n)?24:12},{default:he(()=>[le(qf,{label:"ESRGAN Upscale",prop:"enable_hr",modelValue:C(r).params.enable_hr,"onUpdate:modelValue":y[24]||(y[24]=b=>C(r).params.enable_hr=b),info:"Enable upscale with ESRGAN."},null,8,["modelValue"])]),_:1},8,["span"]),le(C(to),{span:C(n)?24:12},{default:he(()=>[C(r).generatorType==="Img2Img"?(z(),be(qf,{key:0,label:"Send as RefImg",prop:"send_as_refimg",modelValue:C(r).params.send_as_refimg,"onUpdate:modelValue":y[25]||(y[25]=b=>C(r).params.send_as_refimg=b),info:"Instead of regular Img2Img, send the image as a reference image for edit models."},null,8,["modelValue"])):ye("",!0),C(r).generatorType==="Img2Img"&&C(r).params.send_as_refimg&&C(r).params.frames>1?(z(),be(qf,{key:1,label:"Reverse RefImg",prop:"reverse_refimg",modelValue:C(r).params.reverse_refimg,"onUpdate:modelValue":y[26]||(y[26]=b=>C(r).params.reverse_refimg=b),info:"Use the reference image as the final frame instead of the first frame."},null,8,["modelValue"])):ye("",!0)]),_:1},8,["span"])]),_:1})]),ne("div",EK,[le(C(at),{onClick:y[27]||(y[27]=()=>{C(r).cancelled=!0,C(r).generating=!1,C(r).resetStore()}),class:"reset-btn"},{default:he(()=>[He("Reset")]),_:1}),le(C(at),{type:"primary",class:"generate-cancel-btn",style:ze(C(r).generating?"width: 55%;":""),onClick:y[28]||(y[28]=()=>C(r).generateImage(C(r).generatorType))},{default:he(()=>[ne("span",null," Generate "+Ae(C(r).totalImageCount)+" image"+Ae(C(r).totalImageCount===1?"":"s"),1)]),_:1},8,["style"]),C(r).generating?(z(),be(C(at),{key:0,type:"danger",class:"generate-cancel-btn",style:{width:"25%"},disabled:C(r).cancelled,onClick:y[29]||(y[29]=()=>{C(r).cancelled=!0,C(r).generating=!1,C(r).clearQueue()})},{default:he(()=>[He("Cancel all")]),_:1},8,["disabled"])):ye("",!0)]),ne("div",OK,[le(C(g$),{class:"center-both generated-image"},{default:he(()=>[!C(r).generating&&C(r).outputs.length==0?(z(),se("div",AK,[/Inpainting/.test(C(r).generatorType)?(z(),be(K0,{key:0})):ye("",!0),/Img2Img/.test(C(r).generatorType)?(z(),be(K0,{key:1})):ye("",!0)])):ye("",!0),!C(i).showGeneratedImages&&C(r).generating?(z(),be(xH,{key:1,generated:C(r).outputs.length,total:C(r).queue.length,elapsed:c(C(r).timer.seconds),onShowGenerated:y[30]||(y[30]=b=>C(i).showGeneratedImages=!0)},null,8,["generated","total","elapsed"])):ye("",!0),C(i).showGeneratedImages&&C(r).outputs.length!==0?(z(),be(HH,{key:2})):ye("",!0)]),_:1})])]),_:1},8,["model","rules"]))])],64))}});const Bs=uk({history:kT("./"),routes:[{path:"/",name:"generate",component:PK},{path:"/images",name:"images",component:()=>Gf(()=>Promise.resolve().then(()=>RG),void 0,import.meta.url)},{path:"/about",name:"about",component:()=>Gf(()=>Promise.resolve().then(()=>XG),void 0,import.meta.url)},{path:"/options",name:"options",component:()=>Gf(()=>Promise.resolve().then(()=>nq),void 0,import.meta.url)},{path:"/return",name:"return",redirect:e=>(window.location.href=window.location.pathname.endsWith("/")?"..":".","/")}]});function IK(e){const t=[],n=/]+):([^>]+)>/g;return[e.replace(n,(i,l,g)=>{if(g.trim()==="")return"";const o=Number(g);if(isNaN(o))return"";let a=l,u=!1;const d="|high_noise|";return a.startsWith(d)&&(a=a.substring(d.length),u=!0),t.push({name:a,multiplier:o,...u?{is_high_noise:!0}:{}}),""}),t]}function Tu(){return{steps:20,n:1,sampler_name:"Euler",width:512,height:512,cfg_scale:5,eta:1,clip_skip:0,seed:-1,denoising_strength:.6,frames:1,fps:16,enable_hr:!1,send_as_refimg:!0,reverse_refimg:!1,scheduler:"default"}}function sC(){return Math.floor(Math.random()*9999999)+1}const Jt=zo("generator",()=>{const e=["Text2Img","Img2Img","Inpainting"],t=["Img2Img","Inpainting"],n=oe("Text2Img"),r=oe(""),i=kn("promptHistory",[]),l=oe(""),g=kn("negativeLibrary",[]),o=oe(Tu()),a=oe({interval:0,seconds:0}),u=oe({sampler:{name:"Sampler",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.sampler_name],mapToParam:Ce=>Ce.sampler_name},scheduler:{name:"Scheduler",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.scheduler],mapToParam:Ce=>Ce.scheduler},steps:{name:"Steps",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.steps],mapToParam:Ce=>Ce.steps},guidance:{name:"CFG Scale",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.cfg_scale],mapToParam:Ce=>Ce.cfg_scale},clipSkip:{name:"Clip Skip",state:"Disabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.clip_skip],mapToParam:Ce=>Ce.clip_skip},eta:{name:"Eta",state:"Disabled",allowedStates:["Disabled","Enabled"],selected:[o.value.eta],mapToParam:Ce=>Ce.eta}}),d=()=>({sourceProcessing:void 0,sourceImage:void 0,maskImage:void 0}),c=oe({...d(),sourceProcessing:"inpainting"}),f=oe({...d(),sourceProcessing:"img2img"}),s=Ce=>Ce==="Inpainting"?c.value:Ce==="Img2Img"?f.value:d(),h=ee(()=>s(n.value)),v=oe(""),m=oe(!1),y=oe(!1),b=oe([]),_=oe([]),w=oe(64),S=ee(()=>zt().allowLargerParams==="Enabled"?3072:1024),x=oe(1),T=oe(20),E=oe(1),I=ee(()=>zt().allowLargerParams==="Enabled"?150:50),D=oe(1),N=oe(24),L=oe(0),F=oe(1),O=oe(0),V=oe(1),re=oe(0),J=oe(10),ue=oe(1),M=ee(()=>zt().allowLargerParams==="Enabled"?400:200),U=oe(16),R=ee(()=>zt().allowLargerParams==="Enabled"?32:24),P=(Ce,Ve,Ne)=>Array.from({length:(Ve-Ce+1)/Ne},(Ue,pt)=>(pt+Ce)*Ne),Q=oe(P(re.value,J.value,1)),j=oe(P(D.value,N.value,.5)),A=ee(()=>{const Ce=(yr,Ln,es=1)=>yr*(Ln.state==="Multiple"&&Ln.selected.length>0?Ln.selected.length:es),Ne=o.value.n*B().length,Ue=Ce(Ne,u.value.sampler),pt=Ce(Ue,u.value.scheduler),Ze=Ce(pt,u.value.steps),ln=Ce(Ze,u.value.guidance);return Ce(ln,u.value.clipSkip)});function q(){return o.value=Tu(),c.value=d(),f.value=d(),b.value=[],Ut().showGeneratedImages=!1,G(),!0}function G(){_.value=[]}function te(){b.value=[]}async function fe(){const Ce=zt(),Ve=Ce.baseURL.length===0?".":Ce.baseURL,Ne=await fetch(`${Ve}/sdapi/v1/loras`),Ue=await Ne.json();return Ku(Ne,Ue,200,"Failed to get available LoRAs")?Ue:[]}async function pe(Ce){if(!e.includes(Ce))return[];if(r.value==="")return H("Failed to generate: No prompt submitted.");const Ve=Ls(),Ne=Ut();Ve.saveImages();const{sourceImage:Ue,maskImage:pt,sourceProcessing:Ze}=s(Ce);_e(r.value);const ln=[],yr=B().map(rt=>{const Vt=rt.split(" ### ");return{full_prompt:rt,prompt:Vt[0],negative_prompt:Vt[1]||""}}).map(rt=>{const[Vt,zr]=IK(rt.prompt);return{...rt,prompt:Vt,extractedLoras:zr}}),Ln=yr.some(rt=>rt.extractedLoras.length>0)?await fe():[],es=yr.map(({extractedLoras:rt,...Vt})=>{const zr=rt.length>0&&Ln.length>0?rt.map(rr=>{const Ji=Ln.find(Qi=>Qi.name===rr.name||Qi.path===rr.name);return{path:Ji?Ji.path:rr.name,multiplier:rr.multiplier,...rr.is_high_noise?{is_high_noise:!0}:{}}}):[];return{...Vt,...zr.length>0?{lora:zr}:{}}}),{seed:Zi,cfg_scale:br,eta:km,steps:Fr,clip_skip:pC,sampler_name:mC,scheduler:gC,n:vC,...Xl}=o.value,df=parseInt(Zi.toString()),yC=isNaN(df)||df<0?sC():df,bC=Array.from({length:vC},(rt,Vt)=>yC+Vt),Za=(rt,Vt)=>rt.state==="Disabled"?[]:rt.state==="Enabled"?[Vt]:rt.state==="Multiple"&&rt.selected.length===0?[]:rt.selected,_C={promptVariant:es,seed:bC,cfg_scale:Za(u.value.guidance,br),eta:Za(u.value.eta,km),steps:Za(u.value.steps,Fr),clip_skip:Za(u.value.clipSkip,pC),sampler_name:Za(u.value.sampler,mC),scheduler:Za(u.value.scheduler,gC)},wC=(rt=>Object.entries(rt).filter(([zr,rr])=>rr.length>0).reduce((zr,[rr,Ji])=>{const Qi=[];for(const nn of zr)for(const Kl of Ji)Qi.push({...nn,[rr]:Kl});return Qi},[{}]))(_C),CC=[await ie()];for(const rt of wC){const{promptVariant:{full_prompt:Vt,...zr},...rr}=rt,Ji=Ce==="Img2Img"?Xl.send_as_refimg:!1,Qi=Ji&&Xl.frames>1?Xl.reverse_refimg:!1;let nn={prompt:Vt,params:{...Xl,send_as_refimg:Ji,reverse_refimg:Qi,...rr,...zr,init_images:Ue?[Ue.split(",")[1]]:[],mask:pt,inpainting_mask_invert:pt?0:null,inpainting_fill:pt?1:null},source_image:Ue==null?void 0:Ue.split(",")[1],source_mask:pt,source_processing:Ze,models:CC};nn.params.sampler_name=="default"&&delete nn.params.sampler_name,nn.params.scheduler=="default"&&delete nn.params.scheduler,!nn.params.frames||nn.params.frames<=1?(delete nn.params.frames,delete nn.params.fps):nn.params.fps=Math.min(R.value,Math.max(U.value,Math.round(Number(nn.params.fps)||Tu().fps)));const Kl=Me.value.map(SC=>SC.base64);Kl.length>0&&(nn.params.extra_images=Kl),zt().alsoRequestAvi==="Enabled"&&nn.params.frames&&nn.params.frames>1&&(nn.params.video_output_type=2),ln.push(nn)}let Em=!1;m.value||(Em=!0,b.value=[]),m.value=!0,Ne.showGeneratedImages=!1;let Om=_.value.filter(rt=>!rt.gathered&&!rt.failed).length;for(let rt=0;rt{a.value.seconds++},1e3);!_.value.every(rt=>rt.gathered||rt.failed)&&!y.value;){const rt=_.value.find(Vt=>!Vt.gathered&&!Vt.failed);if(!rt)break;rt.gathered=!0;try{const Vt=await K(rt.params);if(!Vt){rt.failed=!0;continue}ve([{...Vt,...rt}])}catch(Vt){rt.failed=!0,console.error("Error fetching image:",Vt)}}}async function ve(Ce){const Ve=Da();console.log(Ce);const Ne=await Promise.all(Ce.map(async Ze=>{const ln=Ze.images[0],yr=!!Ze.animated?"gif":"png",Ln=Ze.extra_data?`data:video/avi;base64,${Ze.extra_data}`:"",es=Ze.final_frame?`data:image/jpeg;base64,${Ze.final_frame}`:"";let Zi={id:-1,image:`data:image/${yr};base64,${ln}`,prompt:Ze.prompt,modelName:Ze.models[0],frames:Ze.params.frames,fps:Ze.params.fps,extra_avi:Ln,final_frame:es,enable_hr:Ze.params.enable_hr,send_as_refimg:Ze.params.send_as_refimg,reverse_refimg:Ze.params.reverse_refimg};if(Ze.info&&typeof Ze.info=="string"&&Ze.info.trim()!=="")try{const br=JSON.parse(Ze.info);["seed","steps","sampler_name","cfg_scale","eta","width","height","clip_skip","fps"].forEach(Fr=>{br[Fr]!=null&&br[Fr]!=null?Zi[Fr]=br[Fr]:Ze.params[Fr]!=null&&(Zi[Fr]=Ze.params[Fr])}),br.extra_generation_params&&br.extra_generation_params["Schedule type"]?Zi.scheduler=br.extra_generation_params["Schedule type"]:Zi.scheduler=Ze.params.scheduler}catch(br){console.warn("Failed to parse info JSON:",br)}return Zi})),Ue=await Ve.pushOutputs(Ne),pt=0;return b.value=[...Ue.map(Ze=>({type:"image",index:pt,output:Ze})),...b.value].sort((Ze,ln)=>Ze.index-ln.index),b.value.length===_.value.length&&(_.value=[],m.value=!1,Ut().showGeneratedImages=!0,clearInterval(a.value.interval),a.value.interval=0,a.value.seconds=0),Ne}async function H(Ce){const Ve=Ut();return Ce&&Ve.raiseError(Ce,!1),[]}function W(Ce,Ve,Ne,Ue){return Ve<=Ne?Ve:(Ut().raiseWarning(`This image was generated using the 'Larger Values' option. Setting '${Ce}' to its default value instead of ${Ve}.`,!0),Ue)}async function k(Ce,Ve=!0){const Ne=Tu();if(n.value="Text2Img",u.value.guidance.state="Enabled",u.value.sampler.state="Enabled",u.value.steps.state="Enabled",u.value.clipSkip.state="Disabled",u.value.scheduler.state="Enabled",Bs.push("/"),Ve&&(Ce.width=Ce.width||Ne.width,Ce.height=Ce.height||Ne.height),Ce.prompt){const Ue=Ce.prompt.split(" ### ");r.value=Ue[0],l.value=Ue[1]||""}if(Ce.sampler_name){o.value.sampler_name=Ce.sampler_name;const pt=(await we()).find(Ze=>Ze.aliases&&Ze.aliases.includes(Ce.sampler_name));pt&&(o.value.sampler_name=pt.name)}Ce.steps&&(o.value.steps=W("steps",Ce.steps,I.value,Ne.steps)),Ce.cfg_scale&&(o.value.cfg_scale=Ce.cfg_scale),(Ce.eta||Ce.eta===0)&&(o.value.eta=Ce.eta),Ce.width&&(o.value.width=W("width",Ce.width,S.value,Ne.width)),Ce.height&&(o.value.height=W("height",Ce.height,S.value,Ne.height)),Ce.seed&&(o.value.seed=Ce.seed),Ce.clip_skip&&(o.value.clip_skip=W("clip_skip",Ce.clip_skip,J.value,Ne.clip_skip)),Ce.scheduler&&(o.value.scheduler=Ce.scheduler),Ce.frames&&(o.value.frames=W("frames",Ce.frames,M.value,Ne.frames)),Ce.fps&&(o.value.fps=Math.min(R.value,Math.max(U.value,Math.round(W("fps",Ce.fps,R.value,Ne.fps)))))}function X(Ce){const Ve=Ls();n.value="Img2Img",f.value.sourceImage=Ce,Ve.drawing=!1,b.value=[],Bs.push("/"),Kn.fabric.Image.fromURL(Ce,Ve.newImage)}function Y(Ce){const Ve=Ls();b.value=[],c.value.sourceImage=Ce,n.value="Inpainting",Bs.push("/"),Kn.fabric.Image.fromURL(Ce,Ve.newImage)}function $(){return l.value===""?r.value:`${r.value} ### ${l.value}`}function B(){const Ce=$(),Ve=Ce.match(/\{(.*?)\}/g)||[];if(Ve.length===0)return[Ce];let Ne=[];return Ve.forEach(Ue=>{const pt=[],Ze=Ue.replace("{","").replace("}","").split("|");Ne.length===0?Ze.forEach(ln=>{const gn=Ce.replace(Ue,ln);pt.push(gn)}):Ne.forEach(ln=>{Ze.forEach(gn=>{const yr=ln.replace(Ue,gn);pt.push(yr)})}),Ne=[...pt]}),Ne}async function K(Ce){const Ve=zt();try{const Ne=await fetch(`${Ve.baseURL.length===0?".":Ve.baseURL}/sdapi/v1/${Ce.init_images.length>0?"img":"txt"}2img`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(Ce)}),Ue=await Ne.json();return Ku(Ne,Ue,200,"Failed to fetch",ce)?Ue:!1}catch{return!1}}function ce(Ce){return Ut().raiseError(Ce,!1),y.value=!1,b.value=[],!1}async function ie(){const Ce=zt(),Ve=await fetch(`${Ce.baseURL.length===0?".":Ce.baseURL}/sdapi/v1/sd-models`),Ne=await Ve.json();if(!!Ku(Ve,Ne,200,"Failed to get available models"))return Ne.length===0?"(No model loaded)":Ne[0].model_name}const Z=oe(0),me=new Map;function ae(){me.clear(),Z.value++}xe(()=>zt().baseURL,()=>{ae()});async function ge(Ce){const Ve=zt(),Ue=((Ve.baseURL.length===0?".":Ve.baseURL).replace(/\/+$/,"")||".")+"/"+Ce.replace(/^\/+/,"");if(me.has(Ue))return me.get(Ue);const pt=(async()=>{try{const Ze=await fetch(Ue);if(Ze.ok)return await Ze.json();console.error(`API Error: ${Ze.status} ${Ze.statusText} at ${Ue}`)}catch(Ze){console.error(`Fetch error for ${Ue}:`,Ze)}return me.delete(Ue),null})();return me.set(Ue,pt),pt}async function we(){const Ce=await ge("/sdapi/v1/samplers");return Array.isArray(Ce)?Ce:[]}async function ke(){const Ce=await ge("/sdapi/v1/schedulers");return Array.isArray(Ce)?Ce:[]}function Oe(Ce){g.value.indexOf(Ce)===-1&&(g.value=[...g.value,Ce])}function Fe(Ce){g.value=g.value.filter(Ve=>Ve!=Ce)}function _e(Ce){if(i.value.findIndex(Ve=>Ve.prompt===Ce)===-1){if(i.value.length>=10+i.value.filter(Ve=>Ve.starred).length){const Ve=i.value.filter(Ue=>!Ue.starred),Ne=i.value.findIndex(Ue=>Ue===Ve[Ve.length-1]);i.value.splice(Ne,1)}i.value=[...i.value,{starred:!1,timestamp:Date.now(),prompt:Ce}]}}function Se(Ce){i.value=i.value.filter(Ve=>Ve.prompt!=Ce&&Ve!=Ce)}function De(){return!1}const Me=oe([]);async function bt(Ce){var Ze,ln;const Ve=Ce.target,Ne=(ln=Ve.files)!=null?ln:(Ze=Ce.dataTransfer)==null?void 0:Ze.files,Ue=Array.from(Ne!=null?Ne:[]);if(Ue.length===0)return;const pt=await Promise.all(Ue.map(async(gn,yr)=>{const Ln=await Tc(gn);return{id:`${Date.now()}-${yr}-${gn.name}`,name:gn.name,size:gn.size,dataUrl:Ln,base64:Ln.includes("data:image")?Ln.split(",")[1]:Ln}}));Me.value||(Me.value=[]);for(let gn=0;gn=Me.value.length)return;const Ue=[...Me.value],[pt]=Ue.splice(Ce,1);Ue.splice(Ne,0,pt),Me.value=Ue}function tn(){Me.value=[];const Ce=document.getElementById("extra_image_input");Ce&&(Ce.value="")}return{generatorType:n,prompt:r,params:o,outputs:b,inpainting:c,img2img:f,uploadDimensions:v,cancelled:y,multiSelect:u,referenceImages:Me,negativePrompt:l,generating:m,negativePromptLibrary:g,minDimensions:w,maxDimensions:S,minImages:x,maxImages:T,minSteps:E,maxSteps:I,minCfgScale:D,maxCfgScale:N,minDenoise:O,maxDenoise:V,minClipSkip:re,maxClipSkip:J,minFrames:ue,maxFrames:M,minFps:U,maxFps:R,minEta:L,maxEta:F,clipSkipList:Q,cfgList:j,queue:_,promptHistory:i,timer:a,validGeneratorTypes:e,sourceGeneratorTypes:t,currentImageProps:h,totalImageCount:A,generateImage:pe,generateText2Img:k,generateImg2Img:X,generateInpainting:Y,getPrompt:De,resetStore:q,clearQueue:G,clearOutputs:te,pushToNegativeLibrary:Oe,removeFromNegativeLibrary:Fe,pushToPromptHistory:_e,removeFromPromptHistory:Se,setExtraImage:bt,removeExtraImage:_t,moveExtraImage:ht,clearExtraImage:tn,getAvailableSamplers:we,getAvailableSchedulers:ke,cacheVersion:Z,invalidateApiCaches:ae,getCachedEndpoint:ge}});"stream"in Blob.prototype||Object.defineProperty(Blob.prototype,"stream",{value(){return new Response(this).body}});var ff=e=>new DataView(new ArrayBuffer(e)),qa=e=>new Uint8Array(e.buffer||e),mo=e=>new TextEncoder().encode(String(e));function MK(e,t){if(t===void 0||t instanceof Date||(t=new Date(t)),e instanceof File)return{t:t||new Date(e.lastModified),o:e.stream()};if(e instanceof Response)return{t:t||new Date(e.headers.get("Last-Modified")||Date.now()),o:e.body};if(t===void 0)t=new Date;else if(isNaN(t))throw new Error("Invalid modification date.");if(typeof e=="string")return{t,o:mo(e)};if(e instanceof Blob)return{t,o:e.stream()};if(e instanceof Uint8Array||e instanceof ReadableStream)return{t,o:e};if(e instanceof ArrayBuffer||ArrayBuffer.isView(e))return{t,o:qa(e)};if(Symbol.asyncIterator in e)return{t,o:lC(e)};throw new TypeError("Unsupported input format.")}function lC(e){const t="next"in e?e:e[Symbol.asyncIterator]();return new ReadableStream({async pull(n){let r=0;for(;n.desiredSize>r;){const i=await t.next();if(!i.value){n.close();break}{const l=LK(i.value);n.enqueue(l),r+=l.byteLength}}}})}function LK(e){return typeof e=="string"?mo(e):e instanceof Uint8Array?e:qa(e)}function RK(e,t,n){if(t===void 0||t instanceof Uint8Array||(t=mo(t)),e instanceof File)return{i:t||mo(e.name),A:e.size};if(e instanceof Response){const r=e.headers.get("content-disposition"),i=r&&r.match(/;\s*filename\*?=["']?(.*?)["']?$/i),l=i&&i[1]||new URL(e.url).pathname.split("/").pop(),g=l&&decodeURIComponent(l),o=n||+e.headers.get("content-length");return{i:t||mo(g),A:o}}if(!t||t.length===0)throw new Error("The file must have a name.");return typeof e=="string"?{i:t,A:mo(e).length}:e instanceof Blob?{i:t,A:e.size}:e instanceof ArrayBuffer||ArrayBuffer.isView(e)?{i:t,A:e.byteLength}:{i:t,A:n}}var DK=new WebAssembly.Instance(new WebAssembly.Module(Uint8Array.from(atob("AGFzbQEAAAABCgJgAABgAn9/AXwDAwIAAQUDAQACBwkCAW0CAAFjAAEIAQAKlQECSQEDfwNAIAEhAEEAIQIDQCAAQQF2IABBAXFBoIbi7X5scyEAIAJBAWoiAkEIRw0ACyABQQJ0IAA2AgAgAUEBaiIBQYACRw0ACwtJAQF/IAFBf3MhAUGAgAQhAkGAgAQgAGohAANAIAFB/wFxIAItAABzQQJ0KAIAIAFBCHZzIQEgAkEBaiICIABJDQALIAFBf3O4Cw"),e=>e.charCodeAt(0)))),{c:$K,m:BK}=DK.exports,FK=qa(BK).subarray(65536);function $y(e,t=0){for(const n of function*(r){for(;r.length>65536;)yield r.subarray(0,65536),r=r.subarray(65536);r.length&&(yield r)}(e))FK.set(n),t=$K(n.length,t);return t}function uC(e,t,n=0){const r=e.getSeconds()>>1|e.getMinutes()<<5|e.getHours()<<11,i=e.getDate()|e.getMonth()+1<<5|e.getFullYear()-1980<<9;t.setUint16(n,r,1),t.setUint16(n+2,i,1)}function zK(e){const t=ff(30);return t.setUint32(0,1347093252),t.setUint32(4,335546368),uC(e.t,t,10),t.setUint16(26,e.i.length,1),qa(t)}async function*NK(e){let{o:t}=e;if("then"in t&&(t=await t),t instanceof Uint8Array)yield t,e.u=$y(t,0),e.A=t.length;else{e.A=0;const n=t.getReader();for(;;){const{value:r,done:i}=await n.read();if(i)break;e.u=$y(r,e.u),e.A+=r.length,yield r}}}function jK(e){const t=ff(16);return t.setUint32(0,1347094280),t.setUint32(4,e.u,1),t.setUint32(8,e.A,1),t.setUint32(12,e.A,1),qa(t)}function VK(e,t){const n=ff(46);return n.setUint32(0,1347092738),n.setUint32(4,352523264),n.setUint16(8,2048),uC(e.t,n,12),n.setUint32(16,e.u,1),n.setUint32(20,e.A,1),n.setUint32(24,e.A,1),n.setUint16(28,e.i.length,1),n.setUint16(40,33204,1),n.setUint32(42,t,1),qa(n)}function UK(e){return e instanceof File||e instanceof Response?[[e],[e]]:[[e.input,e.name,e.size],[e.input,e.lastModified]]}function HK(e,t={}){const n={"Content-Type":"application/zip","Content-Disposition":"attachment"};return Number.isInteger(t.length)&&t.length>0&&(n["Content-Length"]=t.length),t.metadata&&(n["Content-Length"]=p(t.metadata)),new Response(lC(async function*(r){const i=[];let l=0,g=0;for await(const u of r)yield zK(u),yield u.i,yield*NK(u),yield jK(u),i.push(VK(u,l)),i.push(u.i),g++,l+=46+u.i.length+u.A;let o=0;for(const u of i)yield u,o+=u.length;const a=ff(22);a.setUint32(0,1347093766),a.setUint16(8,g,1),a.setUint16(10,g,1),a.setUint32(12,o,1),a.setUint32(16,l,1),yield qa(a)}(async function*(r){for await(const i of r){const[l,g]=UK(i);yield Object.assign(MK(...g),RK(...l))}}(e))),{headers:n})}async function cC(e,t=!0,n){const r=zt();t&&Wi({message:`Downloading ${e.length} image(s)...`,type:"info"});const i=[];for(let o=0;o]/g,"").substring(0,128).trimEnd();let s=r.imageDownloadType;u.startsWith("data:image/gif")&&(s="GIF"),s==="PNG"?i.push({name:f+".png",input:await ba(u,"image/png")}):s==="JPG"?i.push({name:f+".jpg",input:await ba(u,"image/jpeg")}):s==="GIF"?i.push({name:f+".gif",input:await ba(u,"image/gif")}):i.push({name:f+".webp",input:await ba(u,"image/webp")}),i.push({name:f+".json",input:JSON.stringify(c,void 0,4)}),n&&n()}const l=await HK(i).blob(),g=document.createElement("a");g.href=URL.createObjectURL(l),g.download="sdui_images.zip",g.click()}async function fC(e,t){const n=zt(),r=document.createElement("a");let i,l=n.imageDownloadType;e.startsWith("data:image/gif")&&(l="GIF"),l==="PNG"?(i=await ba(e,"image/png"),r.href=URL.createObjectURL(i),r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".png"):l==="JPG"?(i=await ba(e,"image/jpeg"),r.href=URL.createObjectURL(i),r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".jpg"):l==="GIF"?(i=await ba(e,"image/gif"),r.href=URL.createObjectURL(i),r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".gif"):(r.href=e,r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".webp"),r.click(),i&&URL.revokeObjectURL(r.href)}function dC(e,t){const n=e;if(!n)return;const r=atob(n),i=r.length,l=new Uint8Array(i);for(let u=0;u{Ew.confirm("This action will permanently delete this image. Continue?","Warning",{confirmButtonText:"OK",cancelButtonText:"Cancel",type:"warning"}).then(()=>{r.deleteOutput(t.imageData.id),t.onDelete!==void 0&&t.onDelete(t.imageData.id),Wi({type:"success",message:"Deleted Image"})})},l=a=>{if(a.extra_avi){const u=a.extra_avi.split(",")[1];if(!u)return;const d=`${a.seed}-${a.prompt}.avi`;dC(u,d);return}else fC(a.image,`${a.seed}-${a.prompt}`)},g=()=>{Jt().clearOutputs(),Ut().showGeneratedImages=!1,Jt().clearQueue()};async function o(a){const u=window.location.origin,d={prompt:a.prompt,width:a.width?a.width:void 0,height:a.height?a.height:void 0,steps:a.steps,cfg_scale:a.cfg_scale,sampler_name:a.sampler_name,model_name:a.modelName,seed:a.seed,clip_skip:a.clip_skip,frames:a.frames,fps:a.fps,scheduler:a.scheduler,extra_avi:a.extra_avi,final_frame:a.final_frame,enable_hr:a.enable_hr,send_as_refimg:a.send_as_refimg,reverse_refimg:a.reverse_refimg},c=window.location.pathname.replace("images","");let f=`${u}${c}?share=`,s="",h="";for(const[m,y]of Object.entries(d)){if(!y)continue;let b=y;typeof y=="string"?b=encodeURIComponent(y):Array.isArray(y)&&(b=JSON.stringify(y)),s+=`${h}${m}=${b}`,h="&"}f+=btoa(String.fromCharCode.apply(null,Array.from(qX(s)))),await navigator.clipboard.writeText(f),Wi({type:"success",message:"Copied shareable link to clipboard"})}return(a,u)=>(z(),se(je,null,[le(C(at),{class:"compact-button",onClick:i,type:"danger",size:"small",icon:C(Ol),plain:""},{default:he(()=>[He("Delete")]),_:1},8,["icon"]),le(C(at),{class:"compact-button",onClick:u[0]||(u[0]=d=>l(e.imageData)),type:"success",size:"small",icon:C(Ks),plain:""},{default:he(()=>[He("Download")]),_:1},8,["icon"]),e.imageData.starred?ye("",!0):(z(),be(C(at),{key:0,class:"compact-button",onClick:u[1]||(u[1]=d=>C(r).toggleStarred(e.imageData.id)),type:"warning",size:"small",icon:C(w4),plain:""},{default:he(()=>[He("Star")]),_:1},8,["icon"])),e.imageData.starred?(z(),be(C(at),{key:1,class:"compact-button",onClick:u[2]||(u[2]=d=>C(r).toggleStarred(e.imageData.id)),type:"warning",size:"small",icon:C(__),plain:""},{default:he(()=>[He("Unstar")]),_:1},8,["icon"])):ye("",!0),le(C(at),{class:"compact-button",onClick:u[3]||(u[3]=d=>C(n).generateText2Img(e.imageData)),type:"success",size:"small",plain:""},{default:he(()=>[He("Txt2img")]),_:1}),le(C(at),{class:"compact-button",onClick:u[4]||(u[4]=d=>C(n).generateImg2Img(e.imageData.image)),type:"success",size:"small",plain:""},{default:he(()=>[He("Img2img")]),_:1}),le(C(at),{class:"compact-button",onClick:u[5]||(u[5]=d=>C(n).generateInpainting(e.imageData.image)),type:"success",size:"small",plain:""},{default:he(()=>[He("Inpaint")]),_:1}),e.showDismiss?(z(),be(C(at),{key:2,class:"compact-button",onClick:u[6]||(u[6]=d=>g()),type:"success",size:"small",plain:""},{default:he(()=>[He("Dismiss")]),_:1})):ye("",!0),le(C(at),{class:"compact-button",onClick:u[7]||(u[7]=d=>o(e.imageData)),type:"success",icon:C($3),size:"small",plain:""},{default:he(()=>[He("Share")]),_:1},8,["icon"])],64))}});const hC=qt(WK,[["__scopeId","data-v-4d472d34"]]),YK={class:"main-output",style:{position:"relative",display:"flex","align-items":"center","justify-content":"center"}},XK=["src"],KK={style:{"font-size":"16px","font-weight":"500"}},GK={style:{"font-family":"'Segoe UI', Tahoma, Geneva, Verdana, sans-serif","letter-spacing":"0.025em"}},qK={key:0},ZK=ne("br",null,null,-1),JK={key:0},QK=["onClick"],eG={key:1},tG=["onClick"],nG={key:2},rG=["onClick"],iG=Ee({__name:"ImageDialog",setup(e){const t=Da(),n=Ut(),r=oe();fM(r,{onSwipeEnd(d,c){c==="RIGHT"&&n.openModalToLeft(),c==="LEFT"&&n.openModalToRight()}});const i=ee({get(){return n.activeModal!==-1},set(){n.activeModal=-1}}),l=oe(t.currentOutputs[0]);xe(()=>n.activeModal,async()=>{const d=t.currentOutputs.find(c=>c.id===n.activeModal);if(d)return l.value=d;l.value=await jt.outputs.get(n.activeModal)||t.currentOutputs[0]});function g(){i.value=!1}function o(){var f;if(!((f=l.value)!=null&&f.final_frame))return;const d=l.value.final_frame;Jt().generateImg2Img(d)}function a(){var d;!((d=l.value)!=null&&d.image)||fC(l.value.image,`${l.value.seed}-${l.value.prompt}`)}function u(){var f,s;if(!((f=l.value)!=null&&f.extra_avi))return;const d=l.value.extra_avi.split(",")[1];if(!d)return;const c=`output-${(s=l.value.id)!=null?s:"video"}.avi`;dC(d,c)}return(d,c)=>{var f;return z(),be(C(WB),{"model-value":C(i),width:(f=l.value)==null?void 0:f.width,class:"image-viewer",onClosed:g,"align-center":""},{footer:he(()=>[le(hC,{"image-data":l.value,"on-delete":g},null,8,["image-data"])]),default:he(()=>{var s,h,v,m;return[ne("div",{class:"main-output-container",ref_key:"target",ref:r},[ne("div",YK,[(s=l.value)!=null&&s.image?(z(),se("img",{key:0,src:l.value.image,alt:"Output image",style:{"max-width":"100%","max-height":"100%","object-fit":"contain"}},null,8,XK)):ye("",!0)])],512),ne("div",KK,Ae(((h=l.value.prompt)==null?void 0:h.split("###")[0])||"Unknown Creation"),1),ne("div",GK,[ne("div",null,"Negative Prompt: "+Ae(((v=l.value.prompt)==null?void 0:v.split("###")[1])||"None"),1),ne("span",null,"Model: "+Ae(l.value.modelName||"Unknown")+" - ",1),ne("span",null,"Sampler: "+Ae(l.value.sampler_name||"Unknown")+" - ",1),ne("span",null,"Scheduler: "+Ae(l.value.scheduler||"Unknown")+" - ",1),ne("span",null,"Seed: "+Ae(l.value.seed||"Unknown")+" - ",1),ne("span",null,"Steps: "+Ae(l.value.steps||"Unknown")+" - ",1),ne("span",null,"CFG Scale: "+Ae(l.value.cfg_scale||"Unknown")+" - ",1),ne("span",null,"Clip Skip: "+Ae((m=l.value.clip_skip)!=null?m:"Unknown")+" - ",1),ne("span",null,"Dimensions: "+Ae(l.value.width||"???")+"x"+Ae(l.value.height||"???")+" - ",1),ne("span",null,"Frames: "+Ae(l.value.frames||"1"),1),l.value.frames&&l.value.frames>1?(z(),se("span",qK," - FPS: "+Ae(l.value.fps||"Unknown"),1)):ye("",!0),ZK,ne("b",null,[l.value.frames&&l.value.frames>1?(z(),se("span",JK,[ne("a",{href:"#",onClick:et(a,["prevent"]),style:{cursor:"pointer",color:"var(--el-color-primary)"}},"[Download GIF]",8,QK)])):ye("",!0),l.value.extra_avi?(z(),se("span",eG,[He(" - "),ne("a",{href:"#",onClick:et(u,["prevent"]),style:{cursor:"pointer",color:"var(--el-color-primary)"}},"[Download AVI]",8,tG)])):ye("",!0),l.value.final_frame?(z(),se("span",nG,[He(" - "),ne("a",{href:"#",onClick:et(o,["prevent"]),style:{cursor:"pointer",color:"var(--el-color-primary)"}},"[Extend Video]",8,rG)])):ye("",!0)])])]}),_:1},8,["model-value","width"])}}});const aG=e=>(ui("data-v-8f4d2380"),e=e(),ci(),e),oG=aG(()=>ne("div",{style:{"font-size":"20px"}},"Stable UI",-1)),sG={class:"generator-icons"},lG=Ee({__name:"App",setup(e){const n=Ep(kp).smallerOrEqual("md"),r=Ut();zt();const i=fk(),l=oe();return xe(()=>i.path,g=>{l.value&&l.value.open(g)}),(g,o)=>(z(),se(je,null,[ne("div",{class:de({"menu-container":!C(n)})},[le(C(mw),{"default-active":C(i).path,mode:"horizontal",router:!0,ellipsis:!C(n),class:de(C(n)?"mobile-menu":"menu"),ref_key:"menuRef",ref:l},{default:he(()=>[C(n)?ye("",!0):(z(),be(C(Up),{key:0,class:"remove-item-styling center-vertical"},{title:he(()=>[oG]),_:1})),le(ls,{isMobile:C(n),index:"/"},{icon:he(()=>[ne("div",sG,[le(C(Le),null,{default:he(()=>[le(C(wO))]),_:1}),C(r).showGeneratorBadge?(z(),be(C(Le),{key:0,class:"generator-badge",size:10},{default:he(()=>[le(tH)]),_:1})):ye("",!0)])]),title:he(()=>[He("Generate")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/images"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(Q3))]),_:1})]),title:he(()=>[He("Images")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/about"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(g_))]),_:1})]),title:he(()=>[He("About")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/options"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(M4))]),_:1})]),title:he(()=>[He("Options")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/return"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(d_))]),_:1})]),title:he(()=>[He("Return to Lite")]),_:1},8,["isMobile"])]),_:1},8,["default-active","ellipsis","class"])],2),ne("div",{class:de({view:!C(n)})},[le(C(c_))],2),le(iG)],64))}});const uG=qt(lG,[["__scopeId","data-v-8f4d2380"]]);const Tm=tT(uG);Tm.use(iT());Tm.use(Bs);Tm.mount("#app");Bs.replace("/");window.addEventListener("beforeunload",e=>{Jt().generating&&(e.preventDefault(),e.returnValue="")});const cG={key:1,class:"image-action"},fG=Ee({__name:"CustomImage",props:{imageData:null},setup(e){const t=e,n=Ut(),r=oe(null);VI(r,n.toggleMultiSelect,{modifiers:{prevent:!0}});const i=oe(!1);cM(r,([{isIntersecting:g}])=>{g&&(i.value=g)},{rootMargin:"500px"});const l=ee(()=>n.selected.includes(t.imageData.id));return(g,o)=>(z(),se("div",{class:"relative",ref_key:"containerRef",ref:r},[i.value?(z(),be(C(ef),{key:0,class:"thumbnail",src:e.imageData.image,onClick:o[0]||(o[0]=a=>C(n).activeModal=e.imageData.id),fit:"cover",loading:"lazy",style:ze(`${C(l)&&"opacity: 0.5"}`)},null,8,["src","style"])):ye("",!0),i.value?(z(),se("div",cG,[e.imageData.starred?(z(),be(C(Le),{key:0,class:"starred-icon",size:35,color:"var(--el-color-warning)"},{default:he(()=>[le(C(__))]),_:1})):ye("",!0),C(n).multiSelect?(z(),se("div",{key:1,class:"select-container",onClick:o[1]||(o[1]=a=>C(n).toggleSelection(e.imageData.id))},[le(C(Le),{class:"select-icon",size:35,color:`rgba(255, 255, 255, ${C(l)?"1":"0.5"})`},{default:he(()=>[C(l)?ye("",!0):(z(),be(C(El),{key:0})),C(l)?(z(),be(C(p_),{key:1})):ye("",!0)]),_:1},8,["color"])])):ye("",!0)])):ye("",!0)],512))}});const By=qt(fG,[["__scopeId","data-v-b9569bbd"]]);function dG({mobileWidth:e=768,hideAfterDistanceFromTop:t=100,hideAfterScroll:n=100}={}){const{width:r}=a1(),i=ee(()=>r.value<=e),l=oe(!0),{y:g}=uM(window);let o=g.value,a=0;return xe(g,u=>{if(!i.value)return;const d=u-o;if(o=u,d>0&&u>t){a+=d,a>=n&&(l.value=!1,a=0);return}d<0&&(a=0,l.value=!0)}),{isVisible:l,isMobile:i}}const hG={},pG={xmlns:"http://www.w3.org/2000/svg","xmlns:xlink":"http://www.w3.org/1999/xlink",viewBox:"0 0 1024 1024"},mG=ne("path",{d:"M880 112H144c-17.7 0-32 14.3-32 32v736c0 17.7 14.3 32 32 32h736c17.7 0 32-14.3 32-32V144c0-17.7-14.3-32-32-32zm-696 72h136v656H184V184zm656 656H384V384h456v456zM384 320V184h456v136H384z",fill:"currentColor"},null,-1),gG=[mG];function vG(e,t){return z(),se("svg",pG,gG)}const yG=qt(hG,[["render",vG]]),bG=e=>(ui("data-v-b39b47c5"),e=e(),ci(),e),_G={class:"options"},wG=["onClick"],CG=["onClick"],SG=["onClick"],xG={key:1,class:"center-both",style:{gap:"12px"}},TG={key:2},kG=bG(()=>ne("em",{style:{"font-size":"14px"}},"(long press to select multiple images)",-1)),EG=[kG],OG={key:0},AG={key:0,style:{display:"flex",gap:"8px"}},PG={key:1,class:"images"},IG={key:1},MG=Ee({__name:"ImagesView",setup(e){const{width:t}=a1(),{isVisible:n,isMobile:r}=dG(),i=Da(),l=zt(),g=Ut();function o(){g.selected=g.selected.filter(h=>!i.currentOutputs.map(v=>v.id).includes(h)),g.selected=[...g.selected,...i.currentOutputs.map(h=>h.id)],g.multiSelect=!0}async function a(){const h=await jt.outputs.toCollection().primaryKeys();g.selected=h,g.multiSelect=!0}function u(){g.selected=g.selected.filter(h=>!i.currentOutputs.map(v=>v.id).includes(h)),g.selected.length===0&&(g.multiSelect=!1)}function d(){g.selected=[],g.multiSelect=!1}const c=()=>{Ew.confirm(`This action will permanently delete ${g.selected.length} images. Continue?`,"Warning",{confirmButtonText:"OK",cancelButtonText:"Cancel",type:"warning"}).then(()=>{i.deleteMultipleOutputs(g.selected)})};vv(["a","A","ArrowLeft"],g.openModalToLeft),vv(["d","D","ArrowRight"],g.openModalToRight);async function f(){cC(g.selected)}const s=ee(()=>{let h=2;t.value>1440?h=6:t.value>1280?h=5:t.value>768?h=4:t.value>480&&(h=3);const v=[];for(let m=0;m(z(),se(je,null,[ne("div",{class:de(["images-top-bar",{"mobile-hidden":C(r)&&!C(n)}])},[ne("div",_G,[le(C(cu),{placement:"bottom",title:"Sort By",trigger:"click",width:200,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[le(C(c4))]),_:1})]),_:1})]),default:he(()=>[(z(),se(je,null,Nt(["Newest","Oldest"],m=>ne("div",{key:m,onClick:()=>C(i).sortBy=m,class:de(`el-select-dropdown__item ${C(i).sortBy===m?"selected":""}`)},Ae(m),11,wG)),64))]),_:1}),le(C(cu),{placement:"bottom",title:"Filter By",trigger:"click",width:240,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[le(C(d3))]),_:1})]),_:1})]),default:he(()=>[(z(),se(je,null,Nt(["all","favourited","unfavourited","unrated"],m=>ne("div",{key:m,onClick:()=>C(i).filterBy=m,class:de(`el-select-dropdown__item ${C(i).filterBy===m?"selected":""}`)},Ae(C(i).filterBy===m?"Showing":"Show")+" "+Ae(m),11,CG)),64))]),_:1}),le(C(cu),{placement:"bottom",title:"Image Layout",trigger:"click",width:240,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[le(yG)]),_:1})]),_:1})]),default:he(()=>[(z(),se(je,null,Nt([{label:"Square Grid",value:"grid"},{label:"Dynamic Layout",value:"dynamic"}],m=>ne("div",{key:m.value,onClick:()=>C(i).currentLayout=m.value,class:de(`el-select-dropdown__item ${C(i).currentLayout===m.value?"selected":""}`)},Ae(m.label),11,SG)),64))]),_:1}),le(C(cu),{placement:"bottom",title:"Selection",trigger:"click",width:240,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[C(g).multiSelect?(z(),be(C(p_),{key:0})):(z(),be(C(El),{key:1}))]),_:1})]),_:1})]),default:he(()=>[C(g).multiSelect?(z(),se("div",{key:0,class:"el-select-dropdown__item selected",onClick:v[0]||(v[0]=(...m)=>C(g).toggleMultiSelect&&C(g).toggleMultiSelect(...m))},"Disable multi-select")):(z(),se("div",{key:1,class:"el-select-dropdown__item",onClick:v[1]||(v[1]=(...m)=>C(g).toggleMultiSelect&&C(g).toggleMultiSelect(...m))},"Enable multi-select")),C(g).selected.length>0?(z(),se("div",{key:2,class:"el-select-dropdown__item selected",onClick:d},"Deselect All")):(z(),se("div",{key:3,class:"el-select-dropdown__item",onClick:a},"Select All")),C(g).selected.every(m=>!C(i).currentOutputs.map(y=>y.id).includes(m))?(z(),se("div",{key:5,class:"el-select-dropdown__item",onClick:o},"Select Page")):(z(),se("div",{key:4,class:"el-select-dropdown__item selected",onClick:u},"Deselect Page"))]),_:1})]),C(l).pageless==="Disabled"?(z(),be(C(m9),{key:0,layout:"prev, pager, next",total:C(i).outputsLength,"page-size":C(l).pageSize,"current-page":C(i).currentPage,"onUpdate:currentPage":v[2]||(v[2]=m=>C(i).currentPage=m),"hide-on-single-page":""},null,8,["total","page-size","current-page"])):ye("",!0),C(g).multiSelect?(z(),se("div",xG,[ne("div",null,Ae(C(g).selected.length)+" selected",1),le(C(at),{type:"danger",onClick:c,icon:C(Ol),plain:""},{default:he(()=>[He("Delete")]),_:1},8,["icon"]),le(C(at),{type:"success",onClick:f,icon:C(Ks),plain:"",style:{margin:"0"}},{default:he(()=>[He("Download")]),_:1},8,["icon"])])):(z(),se("div",TG,EG))],2),C(i).outputsLength!=0?(z(),se("div",OG,[C(i).currentLayout==="dynamic"?(z(),se("div",AG,[(z(!0),se(je,null,Nt(C(s),(m,y)=>(z(),se("div",{key:y,style:{flex:"1 1 0%"}},[(z(!0),se(je,null,Nt(m,b=>(z(),be(By,{key:b.id,"image-data":b,style:{"margin-bottom":"8px"}},null,8,["image-data"]))),128))]))),128))])):ye("",!0),C(i).currentLayout==="grid"?(z(),se("div",PG,[(z(!0),se(je,null,Nt(C(i).currentOutputs,m=>(z(),be(By,{key:m.id,"image-data":m,style:{width:"200px",height:"200px"}},null,8,["image-data"]))),128))])):ye("",!0)])):ye("",!0),C(i).outputsLength==0?(z(),se("div",IG,[le(C(FF),{description:"No Images Found"})])):ye("",!0)],64))}});const LG=qt(MG,[["__scopeId","data-v-b39b47c5"]]),RG=Object.freeze(Object.defineProperty({__proto__:null,default:LG},Symbol.toStringTag,{value:"Module"})),DG=["href"],$G=Ee({__name:"BaseLink",props:{href:null,router:{type:Boolean}},setup(e){return(t,n)=>{const r=gt("router-link");return z(),se(je,null,[e.router?ye("",!0):(z(),se("a",{key:0,target:"_blank",rel:"noreferrer noopener",href:e.href},[Te(t.$slots,"default",{},void 0,!0)],8,DG)),e.router?(z(),be(r,{key:1,to:e.href},{default:he(()=>[Te(t.$slots,"default",{},void 0,!0)]),_:3},8,["to"])):ye("",!0)],64)}}});const BG=qt($G,[["__scopeId","data-v-17b53b7d"]]),Yl=e=>(ui("data-v-ecc278c5"),e=e(),ci(),e),FG={class:"about"},zG={class:"about-content"},NG=Yl(()=>ne("h1",{style:{"margin-top":"0"}},"Stable UI",-1)),jG=Yl(()=>ne("div",null,[He("This tool was originally a front-end for the AI Horde and has since been converted for local generations with the A1111 API, such as in "),ne("a",{href:"https://github.com/LostRuins/koboldcpp"},"KoboldCpp"),He(".")],-1)),VG=Yl(()=>ne("br",null,null,-1)),UG=Yl(()=>ne("div",null,"If you want to help improve this tool, you can find the currently maintained source code from this modified version on https://github.com/LostRuins/stable-ui and https://github.com/henk717/stable-ui, which is based off https://github.com/ayunami2000/stable-ui, which derives from the original AI Horde version on https://github.com/aqualxx/stable-ui (Original author aqualxx#5004). Feel free to contribute!",-1)),HG=Yl(()=>ne("br",null,null,-1)),WG=Ee({__name:"AboutView",setup(e){return(t,n)=>(z(),se("div",FG,[ne("div",zG,[NG,jG,VG,UG,HG,ne("div",null,[He("You can find the KoboldAI community and authors of this fork on the "),le(BG,{href:"https://koboldai.org/discord"},{default:he(()=>[He("KoboldAI Discord")]),_:1})])])]))}});const YG=qt(WG,[["__scopeId","data-v-ecc278c5"]]),XG=Object.freeze(Object.defineProperty({__proto__:null,default:YG},Symbol.toStringTag,{value:"Module"}));const so=Ee({__name:"FormRadio",props:{label:null,modelValue:null,prop:null,useBoolean:{type:Boolean},options:null,disabled:{type:Boolean},info:null,labelStyle:null,change:null},emits:["update:modelValue"],setup(e,{emit:t}){const n=e;function r(l){if(n.useBoolean&&l==="Enabled"?t("update:modelValue",!0):n.useBoolean&&l==="Disabled"?t("update:modelValue",!1):t("update:modelValue",l),!!n.change)return n.useBoolean&&l==="Enabled"?n.change(!0):n.useBoolean&&l==="Disabled"?n.change(!1):n.change(l)}const i=ee(()=>n.useBoolean?n.modelValue===!0?"Enabled":n.modelValue===!1?"Disabled":n.modelValue:n.modelValue);return(l,g)=>(z(),be(C(Di),{prop:e.prop},{label:he(()=>[le(jl,{info:e.info,"label-style":e.labelStyle},{default:he(()=>[Te(l.$slots,"label",{},()=>[He(Ae(e.label),1)])]),_:3},8,["info","label-style"])]),default:he(()=>[le(C(H$),{disabled:e.disabled,"model-value":C(i),onChange:r},{default:he(()=>[(z(!0),se(je,null,Nt(e.options,o=>(z(),be(C(W$),{key:o,label:o},null,8,["label"]))),128))]),_:1},8,["disabled","model-value"]),Te(l.$slots,"inline")]),_:3},8,["prop"]))}}),Qo=e=>(ui("data-v-0345ee1f"),e=e(),ci(),e),KG=Qo(()=>ne("h1",null,"Options",-1)),GG=Qo(()=>ne("h2",null,"Generation Options",-1)),qG=Qo(()=>ne("h3",null,"Parameter Controls",-1)),ZG=Qo(()=>ne("h2",null,"Image Options",-1)),JG=Qo(()=>ne("div",null,[He("Drop file here OR "),ne("em",null,"click to upload")],-1)),QG=Qo(()=>ne("h2",null,"General Options",-1)),eq=Ee({__name:"OptionsView",setup(e){const t=zt(),n=Da(),r=Jt(),i=[{value:"dark",label:"Dark"},{value:"light",label:"Light"},{value:"auto",label:"Auto"}],l=oe([]),g=oe(),o=oe(!1),a=oe(0);async function u(c){n.importFromZip(c),g.value.clearFiles()}async function d(){Wi({message:`Downloading ${n.outputsLength} image(s)... (this may take a while)`,type:"info"}),o.value=!0,a.value=0;const c=await jt.outputs.toCollection().primaryKeys();await cC(c,!1,()=>{a.value++}),o.value=!1,a.value=0}return(c,f)=>(z(),se(je,null,[KG,le(C(jp),{"label-position":"top",model:C(t).options,onSubmit:f[10]||(f[10]=et(()=>{},["prevent"]))},{default:he(()=>[le(C(zj),{type:"border-card",style:{"min-height":"50vh"}},{default:he(()=>[le(C(zf),{label:"\u{1F5A8}\uFE0F Generation"},{default:he(()=>[GG,le(C(Di),{label:"Base URL"},{default:he(()=>[le(C(Xa),{class:"apikey",prop:"baseURL",modelValue:C(t).baseURL,"onUpdate:modelValue":f[0]||(f[0]=s=>C(t).baseURL=s)},null,8,["modelValue"])]),_:1}),qG,(z(!0),se(je,null,Nt(C(r).multiSelect,(s,h)=>{var v;return z(),se("div",{key:h},[le(so,{label:s.name,prop:"pageless",modelValue:s.state,"onUpdate:modelValue":m=>s.state=m,options:(v=s.allowedStates)!=null?v:[]},null,8,["label","modelValue","onUpdate:modelValue","options"])])}),128)),le(so,{label:"Allow Larger Params",prop:"pageless",modelValue:C(t).allowLargerParams,"onUpdate:modelValue":f[1]||(f[1]=s=>C(t).allowLargerParams=s),options:["Enabled","Disabled"]},null,8,["modelValue"]),le(po,{label:"Image Resize Mode",prop:"imageResizeMode",modelValue:C(t).imageResizeMode,"onUpdate:modelValue":f[2]||(f[2]=s=>C(t).imageResizeMode=s),options:[{label:"No Scale",value:"NoScale"},{label:"Scale and Crop",value:"ScaleAndCrop"},{label:"Scale and Pad",value:"ScaleAndPad"},{label:"Stretch",value:"Stretch"},{label:"Original",value:"Original"}],info:"How to adapt input image dimensions to the requested image size. No Scale: do not scale, just crop or pad each dimension to fit (default behavior). Scale and Crop: scale to match the smaller dimension, preserving aspect ratio, then crop to fit. Scale and Pad: scale to match the larger dimension, preserving aspect ratio, then pad to fit. Stretch: stretch both dimensions to fit, possibly not preserving aspect ratio. Original: send the input image as-is to the server, with no scaling, cropping or padding."},null,8,["modelValue"]),le(so,{label:"Video Gen: Request AVI download",prop:"pageless",modelValue:C(t).alsoRequestAvi,"onUpdate:modelValue":f[3]||(f[3]=s=>C(t).alsoRequestAvi=s),options:["Enabled","Disabled"]},null,8,["modelValue"])]),_:1}),le(C(zf),{label:"\u{1F4F7} Images"},{default:he(()=>[ZG,le(bn,{label:"Images Per Page",prop:"pageSize",modelValue:C(t).pageSize,"onUpdate:modelValue":f[4]||(f[4]=s=>C(t).pageSize=s),min:10,max:50,step:5,disabled:C(t).pageless==="Enabled"},null,8,["modelValue","disabled"]),le(so,{label:"Pageless Format",prop:"pageless",modelValue:C(t).pageless,"onUpdate:modelValue":f[5]||(f[5]=s=>C(t).pageless=s),options:["Enabled","Disabled"]},null,8,["modelValue"]),le(so,{label:"Carousel Auto Cycle",prop:"autoCarousel",modelValue:C(t).autoCarousel,"onUpdate:modelValue":f[6]||(f[6]=s=>C(t).autoCarousel=s),options:["Enabled","Disabled"]},null,8,["modelValue"]),le(so,{label:"Image Download Format",prop:"downloadType",modelValue:C(t).imageDownloadType,"onUpdate:modelValue":f[7]||(f[7]=s=>C(t).imageDownloadType=s),options:["PNG","JPG","WEBP","GIF"]},null,8,["modelValue"]),le(C(Di),{label:"Export Images (ZIP File)"},{default:he(()=>[o.value?(z(),be(C(at),{key:1,icon:C(Ks),disabled:""},{default:he(()=>[He("Downloading... ("+Ae(a.value)+" / "+Ae(C(n).outputsLength)+" image(s))",1)]),_:1},8,["icon"])):(z(),be(C(at),{key:0,icon:C(Ks),onClick:f[8]||(f[8]=s=>d())},{default:he(()=>[He("Download "+Ae(C(n).outputsLength)+" image(s)",1)]),_:1},8,["icon"]))]),_:1}),le(C(Di),{label:"Import Images (ZIP File)"},{default:he(()=>[le(C(Yp),{drag:"",ref_key:"upload",ref:g,"auto-upload":!1,onChange:u,"file-list":l.value,limit:1,multiple:""},{default:he(()=>[le(C(Le),{size:100},{default:he(()=>[le(C(dp))]),_:1}),JG]),_:1},8,["file-list"])]),_:1})]),_:1}),le(C(zf),{label:"\u2699\uFE0F General"},{default:he(()=>[QG,le(po,{label:"Color Scheme",prop:"colorScheme",modelValue:C(t).options.colorMode,"onUpdate:modelValue":f[9]||(f[9]=s=>C(t).options.colorMode=s),options:i},null,8,["modelValue"])]),_:1})]),_:1})]),_:1},8,["model"])],64))}});const tq=qt(eq,[["__scopeId","data-v-0345ee1f"]]),nq=Object.freeze(Object.defineProperty({__proto__:null,default:tq},Symbol.toStringTag,{value:"Module"})); +`&&h>0?(c=0,f++,d++):!this.splitByGrapheme&&this._reSpaceAndTab.test(u.graphemeText[f])&&h>0&&(c++,f++),s[h]={line:d,offset:c},f+=u.graphemeLines[h].length,c+=u.graphemeLines[h].length;return s},styleHas:function(u,d){if(this._styleMap&&!this.isWrapping){var c=this._styleMap[d];c&&(d=c.line)}return a.Text.prototype.styleHas.call(this,u,d)},isEmptyStyles:function(u){if(!this.styles)return!0;var d=0,c=u+1,f,s,h=!1,v=this._styleMap[u],m=this._styleMap[u+1];v&&(u=v.line,d=v.offset),m&&(c=m.line,h=c===u,f=m.offset),s=typeof u>"u"?this.styles:{line:this.styles[u]};for(var y in s)for(var b in s[y])if(b>=d&&(!h||bc&&!E?(v.push(m),m=[],s=S,E=!0):s+=I,!E&&!h&&m.push(w),m=m.concat(b),x=h?0:this._measureWord([w],d,_),_++,E=!1,S>T&&(T=S);return N&&v.push(m),T+D>this.dynamicMinWidth&&(this.dynamicMinWidth=T-I+D),v},isEndOfWrapping:function(u){return!this._styleMap[u+1]||this._styleMap[u+1].line!==this._styleMap[u].line},missingNewlineOffset:function(u){return this.splitByGrapheme?this.isEndOfWrapping(u)?1:0:1},_splitTextIntoLines:function(u){for(var d=a.Text.prototype._splitTextIntoLines.call(this,u),c=this._wrapText(d.lines,this.width),f=new Array(c.length),s=0;s{const e=()=>({canvas:void 0,brush:void 0,visibleImageLayer:void 0,imageLayer:void 0,visibleDrawLayer:void 0,drawLayer:void 0,cropPreviewLayer:void 0,maskPathColor:"",maskBackgroundColor:"",imageScale:1,undoHistory:[],redoHistory:[],drawing:!1}),t=oe({...e(),maskPathColor:"white",maskBackgroundColor:"black"}),n=oe({...e(),maskPathColor:"black",maskBackgroundColor:"white"}),r=ee(()=>Jt().generatorType==="Inpainting"),i=ee(()=>r.value?t.value:n.value),l=ee(()=>Jt().currentImageProps),g=ee({get:()=>i.value.drawing&&!r.value,set:P=>i.value.drawing=P}),o=oe(512),a=oe(512),u=oe(!1),d=oe(30),c=oe(!1),f=new Kn.fabric.Circle({radius:d.value,left:0,originX:"center",originY:"center",angle:0,fill:"",stroke:"red",strokeWidth:3,opacity:0}),s=oe("Erase"),h=oe("rgb(0, 0, 0, 1)");function v(){!i.value.canvas||i.value.canvas.renderAll()}function m(){u.value=!u.value,s.value=u.value?"Draw":"Erase"}function y(P=null){!i.value.canvas||(i.value.brush=i.value.canvas.freeDrawingBrush,i.value.brush.color=P||i.value.brush.color,i.value.brush.width=d.value)}async function b({history:P,erase:Q=!1,draw:j=!1}={}){if(!P||!i.value.drawLayer||!i.value.visibleDrawLayer||!i.value.imageLayer||!i.value.visibleImageLayer||!i.value.canvas)return;P.path.selectable=!1,P.path.opacity=1,P.drawPath=await M(P.path),P.visibleDrawPath=await M(P.path),Q?(P.visibleDrawPath.globalCompositeOperation="destination-out",P.drawPath.stroke=i.value.maskBackgroundColor):(P.visibleDrawPath.globalCompositeOperation="source-over",P.drawPath.stroke=j?h.value:i.value.maskPathColor);let A=await M(P.drawPath);A=A.scale(i.value.imageScale),A.left=A.left+P.drawPath.left*(i.value.imageScale-1),A.top=A.top+P.drawPath.top*(i.value.imageScale-1),j?(i.value.imageLayer.add(A),i.value.visibleImageLayer.addWithUpdate(P.visibleDrawPath)):(i.value.drawLayer.add(A),i.value.visibleDrawLayer.addWithUpdate(P.visibleDrawPath)),i.value.canvas.remove(P.path),v()}function _(){if(i.value.undoHistory.length===0)return;const P=i.value.undoHistory.pop();b({history:P,erase:!1,draw:g.value}),i.value.redoHistory.push(P)}function w(){if(i.value.redoHistory.length===0||!i.value.drawLayer||!i.value.visibleDrawLayer||!i.value.imageLayer||!i.value.visibleImageLayer||!i.value.canvas)return;const P=i.value.redoHistory.pop();i.value.undoHistory.push(P),g.value?(i.value.imageLayer.remove(P.drawPath),i.value.visibleImageLayer.remove(P.visibleDrawPath)):(i.value.drawLayer.remove(P.drawPath),i.value.visibleDrawLayer.remove(P.visibleDrawPath)),delete P.drawPath,delete P.visibleDrawPath,v()}function S(P){i.value.canvas=new Kn.fabric.Canvas(P,{isDrawingMode:!1,width:o.value,height:a.value,backgroundColor:"white"}),i.value.canvas.selection=!1,i.value.canvas.freeDrawingCursor="crosshair",y(i.value.maskPathColor),i.value.canvas.on("mouse:move",R),i.value.canvas.on("path:created",U),v()}function x(P,Q,j,A){let q=A,G=A;return Q>j?(P.scaleToWidth(A),q=A*(a.value/o.value)):(P.scaleToHeight(A),G=A*(o.value/a.value)),{newHeight:q,newWidth:G}}function T(P){const Q=Jt();if(re(),P.selectable=!1,o.value=P.width,a.value=P.height,o.value>Q.maxDimensions||a.value>Q.maxDimensions){const{newHeight:G,newWidth:te}=x(P,o.value,a.value,Q.maxDimensions);o.value=te,a.value=G}if(o.value{i.value.imageLayer=F({image:G,layerHeight:q,layerWidth:A})}),P.cloneAsImage(G=>{if(!i.value.canvas)return;if(o.value!==j||a.value!==j){const{newHeight:pe,newWidth:ve}=x(G,o.value,a.value,j);o.value=ve,a.value=pe}i.value.canvas.setWidth(o.value),i.value.canvas.setHeight(a.value),i.value.canvas.isDrawingMode=!0,i.value.visibleDrawLayer=V(),i.value.visibleImageLayer=V({image:G}),i.value.drawLayer=F({layerHeight:q,layerWidth:A});const te=o.value*i.value.imageScale,fe=a.value*i.value.imageScale;Q.params.width=te-te%64,Q.params.height=fe-fe%64,i.value.visibleDrawLayer.set("opacity",.8),i.value.canvas.add(i.value.visibleImageLayer),i.value.canvas.add(i.value.visibleDrawLayer),i.value.canvas.add(f),c.value=!0,N(),I()})}function E(P,Q,j,A,q,G){const te=G?G.width:P.width,fe=G?G.height:P.height;if(q==="Original")return P;const pe=document.createElement("canvas");pe.width=Q,pe.height=j;const ve=pe.getContext("2d");ve.fillStyle=A,ve.fillRect(0,0,pe.width,pe.height);let H=0,W=0,k=P.width,X=P.height,Y=0,$=0,B=pe.width,K=pe.height;switch(q){case"Stretch":break;case"ScaleAndCrop":{const ce=pe.width/te,ie=pe.height/fe;ce>ie?(X=pe.height/ce,W=(P.height-X)/2):(k=pe.width/ie,H=(P.width-k)/2);break}case"ScaleAndPad":{const ce=pe.width/te,ie=pe.height/fe;ceQ?(k=Q,H=ce-Q/2):(B=te,Y=Q/2-ce),fe>j?(X=j,W=ie-j/2):(K=fe,$=j/2-ie);break}}return ve.drawImage(P,H,W,k,X,Y,$,B,K),pe}function I(){const P=Jt(),Q=zt();if(!i.value.imageLayer||!i.value.drawLayer)return;const j=P.params.width,A=P.params.height,q=i.value.imageLayer.backgroundColor||"#FFFFFF",G=Q.imageResizeMode,te=i.value.imageLayer.toCanvasElement(),fe=E(te,j,A,q,G,te);if(l.value.sourceImage=fe.toDataURL("image/jpeg",1),l.value.maskImage=void 0,i.value.redoHistory.length>0&&!g.value){const ve=i.value.drawLayer.toCanvasElement(),H=E(ve,j,A,q,G,te);l.value.maskImage=H.toDataURL("image/jpeg",1).split(",")[1]}}let D;function N(){if(!i.value.canvas)return;const P=Jt();i.value.cropPreviewLayer&&(i.value.canvas.remove(i.value.cropPreviewLayer),i.value.cropPreviewLayer=void 0),c.value&&(i.value.cropPreviewLayer=V({layerWidth:P.params.width/i.value.imageScale,layerHeight:P.params.height/i.value.imageScale,fill:"rgba(100, 0, 0, 0.5)"}),i.value.canvas.centerObject(i.value.cropPreviewLayer),i.value.canvas.add(i.value.cropPreviewLayer),D&&clearTimeout(D),D=setTimeout(()=>{c.value=!1,N(),D=void 0},5e3))}function L(P,Q){const j="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAA1JREFUGFdj+P///38ACfsD/QVDRcoAAAAASUVORK5CYII=";Kn.fabric.Image.fromURL(j,A=>{A.set({height:P,width:Q});const q=A.toDataURL({format:"png"});l.value.sourceImage=q,g.value=!0,T(A)})}function F({image:P,layerWidth:Q,layerHeight:j}={}){const A=new Kn.fabric.Canvas(null);return A.selection=!1,A.backgroundColor=i.value.maskBackgroundColor,A.setHeight(j||a.value),A.setWidth(Q||o.value),P&&A.add(P),A}function O(){return i.value.imageLayer?{layerHeight:i.value.imageLayer.getHeight(),layerWidth:i.value.imageLayer.getWidth()}:{}}function V({image:P,layerWidth:Q,layerHeight:j,fill:A,abosolute:q}={}){const G=P||new Kn.fabric.Rect({width:Q||o.value,height:j||a.value,left:0,top:0,fill:A||"transparent",absolutePositioned:q||!0,selectable:!1});return new Kn.fabric.Group([G],{selectable:!1,absolutePositioned:q||!0})}function re(){!i.value.canvas||(i.value.visibleImageLayer&&(i.value.canvas.remove(i.value.visibleImageLayer),i.value.visibleImageLayer=void 0),i.value.visibleDrawLayer&&(i.value.canvas.remove(i.value.visibleDrawLayer),i.value.visibleDrawLayer=void 0),i.value.imageLayer=void 0,i.value.drawLayer=void 0,i.value.redoHistory=[],i.value.undoHistory=[],i.value.canvas.isDrawingMode=!1)}function J(){if(!!i.value.canvas){if(i.value.visibleDrawLayer&&(i.value.canvas.remove(i.value.visibleDrawLayer),i.value.visibleDrawLayer=void 0),g.value){const P=Jt();L(P.params.height||512,P.params.width||512)}i.value.drawLayer=void 0,i.value.redoHistory=[],i.value.undoHistory=[],i.value.visibleDrawLayer=V(),i.value.drawLayer=F(O()),i.value.visibleDrawLayer.set("opacity",.8),i.value.canvas.add(i.value.visibleDrawLayer)}}function ue(){var Q;I();const P=document.createElement("a");if(g.value){P.href="data:image/png;base64,"+((Q=l.value.sourceImage)==null?void 0:Q.split(",")[1]),P.download="image_drawing.png",P.click();return}P.href="data:image/png;base64,"+l.value.maskImage,P.download="image_mask.png",P.click()}async function M(P){return new Promise((Q,j)=>{try{P.clone(Q)}catch(A){j(A)}})}async function U(P){const Q={path:P.path};b({history:Q,erase:u.value,draw:g.value}),i.value.redoHistory.push(Q)}function R(P){if(!i.value.canvas)return;const Q=i.value.canvas.getPointer(P.e);f.left=Q.x,f.top=Q.y,f.opacity=.8,u.value?(f.set("strokeWidth",3),f.set("fill",""),y("red")):(f.set("strokeWidth",0),g.value?(f.set("fill",h.value),y(h.value)):(f.set("fill","white"),y("white"))),f.set("radius",d.value/2),v()}return{showCropPreview:c,erasing:u,switchToolText:s,brushSize:d,drawColor:h,drawing:g,imageProps:i,updateCropPreview:N,createNewCanvas:S,downloadMask:ue,resetCanvas:re,resetDrawing:J,flipErase:m,undoAction:w,redoAction:_,newImage:T,newBlankImage:L,setBrush:y,saveImages:I}});const XH={},KH={xmlns:"http://www.w3.org/2000/svg","xmlns:xlink":"http://www.w3.org/1999/xlink",viewBox:"0 0 20 20"},GH=ne("g",{fill:"none"},[ne("path",{d:"M11.197 2.44a1.5 1.5 0 0 1 2.121 0l4.243 4.242a1.5 1.5 0 0 1 0 2.121L9.364 17H14.5a.5.5 0 0 1 0 1H7.82a1.496 1.496 0 0 1-1.14-.437L2.437 13.32a1.5 1.5 0 0 1 0-2.121l8.76-8.76zm1.414.706a.5.5 0 0 0-.707 0L5.538 9.512l4.95 4.95l6.366-6.366a.5.5 0 0 0 0-.707L12.61 3.146zM9.781 15.17l-4.95-4.95l-1.687 1.687a.5.5 0 0 0 0 .707l4.243 4.243a.5.5 0 0 0 .707 0l1.687-1.687z",fill:"currentColor"})],-1),qH=[GH];function ZH(e,t){return z(),se("svg",KH,qH)}const JH=qt(XH,[["render",ZH]]);async function QH(e,t){const n=document.createElement("canvas"),r=n.getContext("2d"),i=new Image;return i.src=e,await new Promise(g=>i.onload=g),n.width=i.width,n.height=i.height,r==null||r.drawImage(i,0,0),n.toDataURL(t)}async function eW(e,t){const n=e.split(";base64,"),r=t!=null?t:n[0].split(":")[1],i=window.atob(r===n[0].split(":")[1]?n[1]:(await QH(e,r)).split(",")[1]),l=new Uint8Array(i.length);for(let g=0;g{const r=new FileReader;r.onload=()=>t(r.result),r.onerror=i=>n(i),r.readAsDataURL(e)})}const gm=e=>(ui("data-v-7dc3b5bb"),e=e(),ci(),e),tW=gm(()=>ne("div",null,[He("Drop file here, paste an image OR "),ne("em",null,"click to upload")],-1)),nW={key:0},rW=gm(()=>ne("div",{class:"center-horizontal",style:{"margin-top":"5px"}},"OR",-1)),iW={class:"canvas-container"},aW=gm(()=>ne("canvas",{id:"canvas"},null,-1)),oW={class:"action-buttons",style:{left:"10px",right:"unset"}},sW={class:"action-buttons"},lW=Ee({__name:"CustomCanvas",setup(e){const t=Jt(),n=Ut(),r=Ls(),i=oe();async function l(a){if(!a.raw.type.includes("image")){n.raiseError("Uploaded file needs to be a image!",!1),i.value.clearFiles();return}const u=await Tc(a.raw);t.currentImageProps.sourceImage=u,r.drawing=!1,Kn.fabric.Image.fromURL(u,r.newImage)}function g(){t.currentImageProps.sourceImage="",r.resetCanvas()}nt(()=>{r.createNewCanvas("canvas"),t.currentImageProps.sourceImage&&Kn.fabric.Image.fromURL(t.currentImageProps.sourceImage,r.newImage),window.addEventListener("paste",o)}),Yt(()=>{window.removeEventListener("paste",o)});async function o(a){var d;const u=(d=a.clipboardData)==null?void 0:d.items;if(!!u)for(let c=0;c(z(),se(je,null,[C(t).currentImageProps.sourceImage?ye("",!0):(z(),be(C(Yp),{key:0,drag:"",ref_key:"upload",ref:i,"auto-upload":!1,onChange:l,limit:1,multiple:""},{tip:he(()=>[C(t).generatorType==="Img2Img"?(z(),se("div",nW,[rW,ne("div",{class:"center-both",style:{cursor:"pointer","text-decoration":"underline","font-size":"1rem"},onClick:u[0]||(u[0]=d=>C(r).newBlankImage(C(t).params.height||512,C(t).params.width||512))},[le(C(Le),{size:20,style:{"margin-right":"2px"}},{default:he(()=>[le(S2)]),_:1}),He("draw something")])])):ye("",!0)]),default:he(()=>[le(C(Le),{size:100},{default:he(()=>[le(C(dp))]),_:1}),tW]),_:1},512)),Et(ne("div",null,[ne("div",iW,[aW,ne("div",oW,[le(C(at),{onClick:u[1]||(u[1]=d=>C(r).undoAction()),icon:C(y_),plain:"",disabled:C(r).imageProps.redoHistory.length===0},null,8,["icon","disabled"]),le(C(at),{onClick:u[2]||(u[2]=d=>C(r).redoAction()),icon:C(b_),plain:"",disabled:C(r).imageProps.undoHistory.length===0},null,8,["icon","disabled"])]),ne("div",sW,[le(C(at),{onClick:u[3]||(u[3]=d=>C(r).resetDrawing()),icon:C(Mr),plain:""},null,8,["icon"]),le(C(at),{onClick:g,icon:C(Ol),plain:""},null,8,["icon"]),le(C(at),{onClick:u[4]||(u[4]=d=>C(r).downloadMask()),icon:C(Ks),plain:""},null,8,["icon"]),le(C(at),{onClick:u[5]||(u[5]=d=>C(r).flipErase()),icon:C(r).erasing?C(o3):JH,plain:""},null,8,["icon"]),C(r).drawing?(z(),be(C(EB),{key:0,modelValue:C(r).drawColor,"onUpdate:modelValue":u[6]||(u[6]=d=>C(r).drawColor=d),"show-alpha":""},null,8,["modelValue"])):ye("",!0)]),le(C(jp),{"label-width":"110px",style:{"margin-top":"10px"}},{default:he(()=>[le(bn,{style:{"margin-bottom":"5px"},label:"Brush Size",prop:"brushSize",modelValue:C(r).brushSize,"onUpdate:modelValue":u[7]||(u[7]=d=>C(r).brushSize=d),min:10,max:100,step:10,change:C(r).setBrush},null,8,["modelValue","change"])]),_:1})])],512),[[Ht,C(t).currentImageProps.sourceImage]])],64))}});const K0=qt(lW,[["__scopeId","data-v-7dc3b5bb"]]),uW={class:"centerIcons"},cW={class:"stackedIcons"},fW=Ee({__name:"StackedIcon",props:{iconOne:null,iconTwo:null,size:null},setup(e){const t=e;return Fx(n=>({"2ad037ca":e.size+"px"})),(n,r)=>(z(),se("div",uW,[ne("div",cW,[le(C(Le),{class:"firstIcon",size:e.size},{default:he(()=>[(z(),be(kt(t.iconOne)))]),_:1},8,["size"]),le(C(Le),{class:"secondIcon",size:e.size},{default:he(()=>[(z(),be(kt(t.iconTwo)))]),_:1},8,["size"])])]))}});const dW=qt(fW,[["__scopeId","data-v-74586a39"]]),hW={key:1,style:{width:"40px"}},bu=Ee({__name:"GeneratorMenuItem",props:{index:null,iconOne:null,iconTwo:null,isMobile:{type:Boolean}},setup(e){const t=e;return(n,r)=>(z(),be(C(zn),{content:e.index,placement:e.isMobile?"bottom":"right",enterable:!1,"hide-after":100},{default:he(()=>[le(C(Up),{index:e.index,style:{height:"60px",display:"flex","justify-content":"center"}},{default:he(()=>[e.iconTwo?(z(),be(dW,{key:0,iconOne:e.iconOne,iconTwo:e.iconTwo,size:40},null,8,["iconOne","iconTwo"])):(z(),se("div",hW,[le(C(Le),{style:{width:"35px"},size:40},{default:he(()=>[(z(),be(kt(t.iconOne)))]),_:1})]))]),_:1},8,["index"])]),_:1},8,["content","placement"]))}});/*! pako 2.1.0 https://github.com/nodeca/pako @license (MIT AND Zlib) */const pW=4,G0=0,q0=1,mW=2;function Go(e){let t=e.length;for(;--t>=0;)e[t]=0}const gW=0,k2=1,vW=2,yW=3,bW=258,vm=29,Vl=256,pl=Vl+1+vm,Co=30,ym=19,E2=2*pl+1,_a=15,Jf=16,_W=7,bm=256,O2=16,A2=17,P2=18,Eh=new Uint8Array([0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0]),Xu=new Uint8Array([0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13]),wW=new Uint8Array([0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7]),I2=new Uint8Array([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]),CW=512,Gr=new Array((pl+2)*2);Go(Gr);const Rs=new Array(Co*2);Go(Rs);const ml=new Array(CW);Go(ml);const gl=new Array(bW-yW+1);Go(gl);const _m=new Array(vm);Go(_m);const kc=new Array(Co);Go(kc);function Qf(e,t,n,r,i){this.static_tree=e,this.extra_bits=t,this.extra_base=n,this.elems=r,this.max_length=i,this.has_stree=e&&e.length}let M2,L2,R2;function ed(e,t){this.dyn_tree=e,this.max_code=0,this.stat_desc=t}const D2=e=>e<256?ml[e]:ml[256+(e>>>7)],vl=(e,t)=>{e.pending_buf[e.pending++]=t&255,e.pending_buf[e.pending++]=t>>>8&255},En=(e,t,n)=>{e.bi_valid>Jf-n?(e.bi_buf|=t<>Jf-e.bi_valid,e.bi_valid+=n-Jf):(e.bi_buf|=t<{En(e,n[t*2],n[t*2+1])},$2=(e,t)=>{let n=0;do n|=e&1,e>>>=1,n<<=1;while(--t>0);return n>>>1},SW=e=>{e.bi_valid===16?(vl(e,e.bi_buf),e.bi_buf=0,e.bi_valid=0):e.bi_valid>=8&&(e.pending_buf[e.pending++]=e.bi_buf&255,e.bi_buf>>=8,e.bi_valid-=8)},xW=(e,t)=>{const n=t.dyn_tree,r=t.max_code,i=t.stat_desc.static_tree,l=t.stat_desc.has_stree,g=t.stat_desc.extra_bits,o=t.stat_desc.extra_base,a=t.stat_desc.max_length;let u,d,c,f,s,h,v=0;for(f=0;f<=_a;f++)e.bl_count[f]=0;for(n[e.heap[e.heap_max]*2+1]=0,u=e.heap_max+1;ua&&(f=a,v++),n[d*2+1]=f,!(d>r)&&(e.bl_count[f]++,s=0,d>=o&&(s=g[d-o]),h=n[d*2],e.opt_len+=h*(f+s),l&&(e.static_len+=h*(i[d*2+1]+s)));if(v!==0){do{for(f=a-1;e.bl_count[f]===0;)f--;e.bl_count[f]--,e.bl_count[f+1]+=2,e.bl_count[a]--,v-=2}while(v>0);for(f=a;f!==0;f--)for(d=e.bl_count[f];d!==0;)c=e.heap[--u],!(c>r)&&(n[c*2+1]!==f&&(e.opt_len+=(f-n[c*2+1])*n[c*2],n[c*2+1]=f),d--)}},B2=(e,t,n)=>{const r=new Array(_a+1);let i=0,l,g;for(l=1;l<=_a;l++)i=i+n[l-1]<<1,r[l]=i;for(g=0;g<=t;g++){let o=e[g*2+1];o!==0&&(e[g*2]=$2(r[o]++,o))}},TW=()=>{let e,t,n,r,i;const l=new Array(_a+1);for(n=0,r=0;r>=7;r{let t;for(t=0;t{e.bi_valid>8?vl(e,e.bi_buf):e.bi_valid>0&&(e.pending_buf[e.pending++]=e.bi_buf),e.bi_buf=0,e.bi_valid=0},Z0=(e,t,n,r)=>{const i=t*2,l=n*2;return e[i]{const r=e.heap[n];let i=n<<1;for(;i<=e.heap_len&&(i{let r,i,l=0,g,o;if(e.sym_next!==0)do r=e.pending_buf[e.sym_buf+l++]&255,r+=(e.pending_buf[e.sym_buf+l++]&255)<<8,i=e.pending_buf[e.sym_buf+l++],r===0?Tr(e,i,t):(g=gl[i],Tr(e,g+Vl+1,t),o=Eh[g],o!==0&&(i-=_m[g],En(e,i,o)),r--,g=D2(r),Tr(e,g,n),o=Xu[g],o!==0&&(r-=kc[g],En(e,r,o)));while(l{const n=t.dyn_tree,r=t.stat_desc.static_tree,i=t.stat_desc.has_stree,l=t.stat_desc.elems;let g,o,a=-1,u;for(e.heap_len=0,e.heap_max=E2,g=0;g>1;g>=1;g--)td(e,n,g);u=l;do g=e.heap[1],e.heap[1]=e.heap[e.heap_len--],td(e,n,1),o=e.heap[1],e.heap[--e.heap_max]=g,e.heap[--e.heap_max]=o,n[u*2]=n[g*2]+n[o*2],e.depth[u]=(e.depth[g]>=e.depth[o]?e.depth[g]:e.depth[o])+1,n[g*2+1]=n[o*2+1]=u,e.heap[1]=u++,td(e,n,1);while(e.heap_len>=2);e.heap[--e.heap_max]=e.heap[1],xW(e,t),B2(n,a,e.bl_count)},Q0=(e,t,n)=>{let r,i=-1,l,g=t[0*2+1],o=0,a=7,u=4;for(g===0&&(a=138,u=3),t[(n+1)*2+1]=65535,r=0;r<=n;r++)l=g,g=t[(r+1)*2+1],!(++o{let r,i=-1,l,g=t[0*2+1],o=0,a=7,u=4;for(g===0&&(a=138,u=3),r=0;r<=n;r++)if(l=g,g=t[(r+1)*2+1],!(++o{let t;for(Q0(e,e.dyn_ltree,e.l_desc.max_code),Q0(e,e.dyn_dtree,e.d_desc.max_code),Oh(e,e.bl_desc),t=ym-1;t>=3&&e.bl_tree[I2[t]*2+1]===0;t--);return e.opt_len+=3*(t+1)+5+5+4,t},EW=(e,t,n,r)=>{let i;for(En(e,t-257,5),En(e,n-1,5),En(e,r-4,4),i=0;i{let t=4093624447,n;for(n=0;n<=31;n++,t>>>=1)if(t&1&&e.dyn_ltree[n*2]!==0)return G0;if(e.dyn_ltree[9*2]!==0||e.dyn_ltree[10*2]!==0||e.dyn_ltree[13*2]!==0)return q0;for(n=32;n{ty||(TW(),ty=!0),e.l_desc=new ed(e.dyn_ltree,M2),e.d_desc=new ed(e.dyn_dtree,L2),e.bl_desc=new ed(e.bl_tree,R2),e.bi_buf=0,e.bi_valid=0,F2(e)},N2=(e,t,n,r)=>{En(e,(gW<<1)+(r?1:0),3),z2(e),vl(e,n),vl(e,~n),n&&e.pending_buf.set(e.window.subarray(t,t+n),e.pending),e.pending+=n},PW=e=>{En(e,k2<<1,3),Tr(e,bm,Gr),SW(e)},IW=(e,t,n,r)=>{let i,l,g=0;e.level>0?(e.strm.data_type===mW&&(e.strm.data_type=OW(e)),Oh(e,e.l_desc),Oh(e,e.d_desc),g=kW(e),i=e.opt_len+3+7>>>3,l=e.static_len+3+7>>>3,l<=i&&(i=l)):i=l=n+5,n+4<=i&&t!==-1?N2(e,t,n,r):e.strategy===pW||l===i?(En(e,(k2<<1)+(r?1:0),3),J0(e,Gr,Rs)):(En(e,(vW<<1)+(r?1:0),3),EW(e,e.l_desc.max_code+1,e.d_desc.max_code+1,g+1),J0(e,e.dyn_ltree,e.dyn_dtree)),F2(e),r&&z2(e)},MW=(e,t,n)=>(e.pending_buf[e.sym_buf+e.sym_next++]=t,e.pending_buf[e.sym_buf+e.sym_next++]=t>>8,e.pending_buf[e.sym_buf+e.sym_next++]=n,t===0?e.dyn_ltree[n*2]++:(e.matches++,t--,e.dyn_ltree[(gl[n]+Vl+1)*2]++,e.dyn_dtree[D2(t)*2]++),e.sym_next===e.sym_end);var LW=AW,RW=N2,DW=IW,$W=MW,BW=PW,FW={_tr_init:LW,_tr_stored_block:RW,_tr_flush_block:DW,_tr_tally:$W,_tr_align:BW};const zW=(e,t,n,r)=>{let i=e&65535|0,l=e>>>16&65535|0,g=0;for(;n!==0;){g=n>2e3?2e3:n,n-=g;do i=i+t[r++]|0,l=l+i|0;while(--g);i%=65521,l%=65521}return i|l<<16|0};var yl=zW;const NW=()=>{let e,t=[];for(var n=0;n<256;n++){e=n;for(var r=0;r<8;r++)e=e&1?3988292384^e>>>1:e>>>1;t[n]=e}return t},jW=new Uint32Array(NW()),VW=(e,t,n,r)=>{const i=jW,l=r+n;e^=-1;for(let g=r;g>>8^i[(e^t[g])&255];return e^-1};var Zt=VW,$a={2:"need dictionary",1:"stream end",0:"","-1":"file error","-2":"stream error","-3":"data error","-4":"insufficient memory","-5":"buffer error","-6":"incompatible version"},qo={Z_NO_FLUSH:0,Z_PARTIAL_FLUSH:1,Z_SYNC_FLUSH:2,Z_FULL_FLUSH:3,Z_FINISH:4,Z_BLOCK:5,Z_TREES:6,Z_OK:0,Z_STREAM_END:1,Z_NEED_DICT:2,Z_ERRNO:-1,Z_STREAM_ERROR:-2,Z_DATA_ERROR:-3,Z_MEM_ERROR:-4,Z_BUF_ERROR:-5,Z_NO_COMPRESSION:0,Z_BEST_SPEED:1,Z_BEST_COMPRESSION:9,Z_DEFAULT_COMPRESSION:-1,Z_FILTERED:1,Z_HUFFMAN_ONLY:2,Z_RLE:3,Z_FIXED:4,Z_DEFAULT_STRATEGY:0,Z_BINARY:0,Z_TEXT:1,Z_UNKNOWN:2,Z_DEFLATED:8};const{_tr_init:UW,_tr_stored_block:Ah,_tr_flush_block:HW,_tr_tally:$i,_tr_align:WW}=FW,{Z_NO_FLUSH:Bi,Z_PARTIAL_FLUSH:YW,Z_FULL_FLUSH:XW,Z_FINISH:Gn,Z_BLOCK:ny,Z_OK:rn,Z_STREAM_END:ry,Z_STREAM_ERROR:Pr,Z_DATA_ERROR:KW,Z_BUF_ERROR:nd,Z_DEFAULT_COMPRESSION:GW,Z_FILTERED:qW,Z_HUFFMAN_ONLY:_u,Z_RLE:ZW,Z_FIXED:JW,Z_DEFAULT_STRATEGY:QW,Z_UNKNOWN:eY,Z_DEFLATED:lf}=qo,tY=9,nY=15,rY=8,iY=29,aY=256,Ph=aY+1+iY,oY=30,sY=19,lY=2*Ph+1,uY=15,lt=3,Mi=258,Ir=Mi+lt+1,cY=32,$o=42,wm=57,Ih=69,Mh=73,Lh=91,Rh=103,wa=113,ms=666,wn=1,Zo=2,Ba=3,Jo=4,fY=3,Ca=(e,t)=>(e.msg=$a[t],t),iy=e=>e*2-(e>4?9:0),Ai=e=>{let t=e.length;for(;--t>=0;)e[t]=0},dY=e=>{let t,n,r,i=e.w_size;t=e.hash_size,r=t;do n=e.head[--r],e.head[r]=n>=i?n-i:0;while(--t);t=i,r=t;do n=e.prev[--r],e.prev[r]=n>=i?n-i:0;while(--t)};let hY=(e,t,n)=>(t<{const t=e.state;let n=t.pending;n>e.avail_out&&(n=e.avail_out),n!==0&&(e.output.set(t.pending_buf.subarray(t.pending_out,t.pending_out+n),e.next_out),e.next_out+=n,t.pending_out+=n,e.total_out+=n,e.avail_out-=n,t.pending-=n,t.pending===0&&(t.pending_out=0))},jn=(e,t)=>{HW(e,e.block_start>=0?e.block_start:-1,e.strstart-e.block_start,t),e.block_start=e.strstart,$n(e.strm)},mt=(e,t)=>{e.pending_buf[e.pending++]=t},us=(e,t)=>{e.pending_buf[e.pending++]=t>>>8&255,e.pending_buf[e.pending++]=t&255},Dh=(e,t,n,r)=>{let i=e.avail_in;return i>r&&(i=r),i===0?0:(e.avail_in-=i,t.set(e.input.subarray(e.next_in,e.next_in+i),n),e.state.wrap===1?e.adler=yl(e.adler,t,i,n):e.state.wrap===2&&(e.adler=Zt(e.adler,t,i,n)),e.next_in+=i,e.total_in+=i,i)},j2=(e,t)=>{let n=e.max_chain_length,r=e.strstart,i,l,g=e.prev_length,o=e.nice_match;const a=e.strstart>e.w_size-Ir?e.strstart-(e.w_size-Ir):0,u=e.window,d=e.w_mask,c=e.prev,f=e.strstart+Mi;let s=u[r+g-1],h=u[r+g];e.prev_length>=e.good_match&&(n>>=2),o>e.lookahead&&(o=e.lookahead);do if(i=t,!(u[i+g]!==h||u[i+g-1]!==s||u[i]!==u[r]||u[++i]!==u[r+1])){r+=2,i++;do;while(u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&u[++r]===u[++i]&&rg){if(e.match_start=t,g=l,l>=o)break;s=u[r+g-1],h=u[r+g]}}while((t=c[t&d])>a&&--n!==0);return g<=e.lookahead?g:e.lookahead},Bo=e=>{const t=e.w_size;let n,r,i;do{if(r=e.window_size-e.lookahead-e.strstart,e.strstart>=t+(t-Ir)&&(e.window.set(e.window.subarray(t,t+t-r),0),e.match_start-=t,e.strstart-=t,e.block_start-=t,e.insert>e.strstart&&(e.insert=e.strstart),dY(e),r+=t),e.strm.avail_in===0)break;if(n=Dh(e.strm,e.window,e.strstart+e.lookahead,r),e.lookahead+=n,e.lookahead+e.insert>=lt)for(i=e.strstart-e.insert,e.ins_h=e.window[i],e.ins_h=Fi(e,e.ins_h,e.window[i+1]);e.insert&&(e.ins_h=Fi(e,e.ins_h,e.window[i+lt-1]),e.prev[i&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=i,i++,e.insert--,!(e.lookahead+e.insert{let n=e.pending_buf_size-5>e.w_size?e.w_size:e.pending_buf_size-5,r,i,l,g=0,o=e.strm.avail_in;do{if(r=65535,l=e.bi_valid+42>>3,e.strm.avail_outi+e.strm.avail_in&&(r=i+e.strm.avail_in),r>l&&(r=l),r>8,e.pending_buf[e.pending-2]=~r,e.pending_buf[e.pending-1]=~r>>8,$n(e.strm),i&&(i>r&&(i=r),e.strm.output.set(e.window.subarray(e.block_start,e.block_start+i),e.strm.next_out),e.strm.next_out+=i,e.strm.avail_out-=i,e.strm.total_out+=i,e.block_start+=i,r-=i),r&&(Dh(e.strm,e.strm.output,e.strm.next_out,r),e.strm.next_out+=r,e.strm.avail_out-=r,e.strm.total_out+=r)}while(g===0);return o-=e.strm.avail_in,o&&(o>=e.w_size?(e.matches=2,e.window.set(e.strm.input.subarray(e.strm.next_in-e.w_size,e.strm.next_in),0),e.strstart=e.w_size,e.insert=e.strstart):(e.window_size-e.strstart<=o&&(e.strstart-=e.w_size,e.window.set(e.window.subarray(e.w_size,e.w_size+e.strstart),0),e.matches<2&&e.matches++,e.insert>e.strstart&&(e.insert=e.strstart)),e.window.set(e.strm.input.subarray(e.strm.next_in-o,e.strm.next_in),e.strstart),e.strstart+=o,e.insert+=o>e.w_size-e.insert?e.w_size-e.insert:o),e.block_start=e.strstart),e.high_waterl&&e.block_start>=e.w_size&&(e.block_start-=e.w_size,e.strstart-=e.w_size,e.window.set(e.window.subarray(e.w_size,e.w_size+e.strstart),0),e.matches<2&&e.matches++,l+=e.w_size,e.insert>e.strstart&&(e.insert=e.strstart)),l>e.strm.avail_in&&(l=e.strm.avail_in),l&&(Dh(e.strm,e.window,e.strstart,l),e.strstart+=l,e.insert+=l>e.w_size-e.insert?e.w_size-e.insert:l),e.high_water>3,l=e.pending_buf_size-l>65535?65535:e.pending_buf_size-l,n=l>e.w_size?e.w_size:l,i=e.strstart-e.block_start,(i>=n||(i||t===Gn)&&t!==Bi&&e.strm.avail_in===0&&i<=l)&&(r=i>l?l:i,g=t===Gn&&e.strm.avail_in===0&&r===i?1:0,Ah(e,e.block_start,r,g),e.block_start+=r,$n(e.strm)),g?Ba:wn)},rd=(e,t)=>{let n,r;for(;;){if(e.lookahead=lt&&(e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart),n!==0&&e.strstart-n<=e.w_size-Ir&&(e.match_length=j2(e,n)),e.match_length>=lt)if(r=$i(e,e.strstart-e.match_start,e.match_length-lt),e.lookahead-=e.match_length,e.match_length<=e.max_lazy_match&&e.lookahead>=lt){e.match_length--;do e.strstart++,e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart;while(--e.match_length!==0);e.strstart++}else e.strstart+=e.match_length,e.match_length=0,e.ins_h=e.window[e.strstart],e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+1]);else r=$i(e,0,e.window[e.strstart]),e.lookahead--,e.strstart++;if(r&&(jn(e,!1),e.strm.avail_out===0))return wn}return e.insert=e.strstart{let n,r,i;for(;;){if(e.lookahead=lt&&(e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart),e.prev_length=e.match_length,e.prev_match=e.match_start,e.match_length=lt-1,n!==0&&e.prev_length4096)&&(e.match_length=lt-1)),e.prev_length>=lt&&e.match_length<=e.prev_length){i=e.strstart+e.lookahead-lt,r=$i(e,e.strstart-1-e.prev_match,e.prev_length-lt),e.lookahead-=e.prev_length-1,e.prev_length-=2;do++e.strstart<=i&&(e.ins_h=Fi(e,e.ins_h,e.window[e.strstart+lt-1]),n=e.prev[e.strstart&e.w_mask]=e.head[e.ins_h],e.head[e.ins_h]=e.strstart);while(--e.prev_length!==0);if(e.match_available=0,e.match_length=lt-1,e.strstart++,r&&(jn(e,!1),e.strm.avail_out===0))return wn}else if(e.match_available){if(r=$i(e,0,e.window[e.strstart-1]),r&&jn(e,!1),e.strstart++,e.lookahead--,e.strm.avail_out===0)return wn}else e.match_available=1,e.strstart++,e.lookahead--}return e.match_available&&(r=$i(e,0,e.window[e.strstart-1]),e.match_available=0),e.insert=e.strstart{let n,r,i,l;const g=e.window;for(;;){if(e.lookahead<=Mi){if(Bo(e),e.lookahead<=Mi&&t===Bi)return wn;if(e.lookahead===0)break}if(e.match_length=0,e.lookahead>=lt&&e.strstart>0&&(i=e.strstart-1,r=g[i],r===g[++i]&&r===g[++i]&&r===g[++i])){l=e.strstart+Mi;do;while(r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&r===g[++i]&&ie.lookahead&&(e.match_length=e.lookahead)}if(e.match_length>=lt?(n=$i(e,1,e.match_length-lt),e.lookahead-=e.match_length,e.strstart+=e.match_length,e.match_length=0):(n=$i(e,0,e.window[e.strstart]),e.lookahead--,e.strstart++),n&&(jn(e,!1),e.strm.avail_out===0))return wn}return e.insert=0,t===Gn?(jn(e,!0),e.strm.avail_out===0?Ba:Jo):e.sym_next&&(jn(e,!1),e.strm.avail_out===0)?wn:Zo},mY=(e,t)=>{let n;for(;;){if(e.lookahead===0&&(Bo(e),e.lookahead===0)){if(t===Bi)return wn;break}if(e.match_length=0,n=$i(e,0,e.window[e.strstart]),e.lookahead--,e.strstart++,n&&(jn(e,!1),e.strm.avail_out===0))return wn}return e.insert=0,t===Gn?(jn(e,!0),e.strm.avail_out===0?Ba:Jo):e.sym_next&&(jn(e,!1),e.strm.avail_out===0)?wn:Zo};function wr(e,t,n,r,i){this.good_length=e,this.max_lazy=t,this.nice_length=n,this.max_chain=r,this.func=i}const gs=[new wr(0,0,0,0,V2),new wr(4,4,8,4,rd),new wr(4,5,16,8,rd),new wr(4,6,32,32,rd),new wr(4,4,16,16,ao),new wr(8,16,32,32,ao),new wr(8,16,128,128,ao),new wr(8,32,128,256,ao),new wr(32,128,258,1024,ao),new wr(32,258,258,4096,ao)],gY=e=>{e.window_size=2*e.w_size,Ai(e.head),e.max_lazy_match=gs[e.level].max_lazy,e.good_match=gs[e.level].good_length,e.nice_match=gs[e.level].nice_length,e.max_chain_length=gs[e.level].max_chain,e.strstart=0,e.block_start=0,e.lookahead=0,e.insert=0,e.match_length=e.prev_length=lt-1,e.match_available=0,e.ins_h=0};function vY(){this.strm=null,this.status=0,this.pending_buf=null,this.pending_buf_size=0,this.pending_out=0,this.pending=0,this.wrap=0,this.gzhead=null,this.gzindex=0,this.method=lf,this.last_flush=-1,this.w_size=0,this.w_bits=0,this.w_mask=0,this.window=null,this.window_size=0,this.prev=null,this.head=null,this.ins_h=0,this.hash_size=0,this.hash_bits=0,this.hash_mask=0,this.hash_shift=0,this.block_start=0,this.match_length=0,this.prev_match=0,this.match_available=0,this.strstart=0,this.match_start=0,this.lookahead=0,this.prev_length=0,this.max_chain_length=0,this.max_lazy_match=0,this.level=0,this.strategy=0,this.good_match=0,this.nice_match=0,this.dyn_ltree=new Uint16Array(lY*2),this.dyn_dtree=new Uint16Array((2*oY+1)*2),this.bl_tree=new Uint16Array((2*sY+1)*2),Ai(this.dyn_ltree),Ai(this.dyn_dtree),Ai(this.bl_tree),this.l_desc=null,this.d_desc=null,this.bl_desc=null,this.bl_count=new Uint16Array(uY+1),this.heap=new Uint16Array(2*Ph+1),Ai(this.heap),this.heap_len=0,this.heap_max=0,this.depth=new Uint16Array(2*Ph+1),Ai(this.depth),this.sym_buf=0,this.lit_bufsize=0,this.sym_next=0,this.sym_end=0,this.opt_len=0,this.static_len=0,this.matches=0,this.insert=0,this.bi_buf=0,this.bi_valid=0}const Ul=e=>{if(!e)return 1;const t=e.state;return!t||t.strm!==e||t.status!==$o&&t.status!==wm&&t.status!==Ih&&t.status!==Mh&&t.status!==Lh&&t.status!==Rh&&t.status!==wa&&t.status!==ms?1:0},U2=e=>{if(Ul(e))return Ca(e,Pr);e.total_in=e.total_out=0,e.data_type=eY;const t=e.state;return t.pending=0,t.pending_out=0,t.wrap<0&&(t.wrap=-t.wrap),t.status=t.wrap===2?wm:t.wrap?$o:wa,e.adler=t.wrap===2?0:1,t.last_flush=-2,UW(t),rn},H2=e=>{const t=U2(e);return t===rn&&gY(e.state),t},yY=(e,t)=>Ul(e)||e.state.wrap!==2?Pr:(e.state.gzhead=t,rn),W2=(e,t,n,r,i,l)=>{if(!e)return Pr;let g=1;if(t===GW&&(t=6),r<0?(g=0,r=-r):r>15&&(g=2,r-=16),i<1||i>tY||n!==lf||r<8||r>15||t<0||t>9||l<0||l>JW||r===8&&g!==1)return Ca(e,Pr);r===8&&(r=9);const o=new vY;return e.state=o,o.strm=e,o.status=$o,o.wrap=g,o.gzhead=null,o.w_bits=r,o.w_size=1<W2(e,t,lf,nY,rY,QW),_Y=(e,t)=>{if(Ul(e)||t>ny||t<0)return e?Ca(e,Pr):Pr;const n=e.state;if(!e.output||e.avail_in!==0&&!e.input||n.status===ms&&t!==Gn)return Ca(e,e.avail_out===0?nd:Pr);const r=n.last_flush;if(n.last_flush=t,n.pending!==0){if($n(e),e.avail_out===0)return n.last_flush=-1,rn}else if(e.avail_in===0&&iy(t)<=iy(r)&&t!==Gn)return Ca(e,nd);if(n.status===ms&&e.avail_in!==0)return Ca(e,nd);if(n.status===$o&&n.wrap===0&&(n.status=wa),n.status===$o){let i=lf+(n.w_bits-8<<4)<<8,l=-1;if(n.strategy>=_u||n.level<2?l=0:n.level<6?l=1:n.level===6?l=2:l=3,i|=l<<6,n.strstart!==0&&(i|=cY),i+=31-i%31,us(n,i),n.strstart!==0&&(us(n,e.adler>>>16),us(n,e.adler&65535)),e.adler=1,n.status=wa,$n(e),n.pending!==0)return n.last_flush=-1,rn}if(n.status===wm){if(e.adler=0,mt(n,31),mt(n,139),mt(n,8),n.gzhead)mt(n,(n.gzhead.text?1:0)+(n.gzhead.hcrc?2:0)+(n.gzhead.extra?4:0)+(n.gzhead.name?8:0)+(n.gzhead.comment?16:0)),mt(n,n.gzhead.time&255),mt(n,n.gzhead.time>>8&255),mt(n,n.gzhead.time>>16&255),mt(n,n.gzhead.time>>24&255),mt(n,n.level===9?2:n.strategy>=_u||n.level<2?4:0),mt(n,n.gzhead.os&255),n.gzhead.extra&&n.gzhead.extra.length&&(mt(n,n.gzhead.extra.length&255),mt(n,n.gzhead.extra.length>>8&255)),n.gzhead.hcrc&&(e.adler=Zt(e.adler,n.pending_buf,n.pending,0)),n.gzindex=0,n.status=Ih;else if(mt(n,0),mt(n,0),mt(n,0),mt(n,0),mt(n,0),mt(n,n.level===9?2:n.strategy>=_u||n.level<2?4:0),mt(n,fY),n.status=wa,$n(e),n.pending!==0)return n.last_flush=-1,rn}if(n.status===Ih){if(n.gzhead.extra){let i=n.pending,l=(n.gzhead.extra.length&65535)-n.gzindex;for(;n.pending+l>n.pending_buf_size;){let o=n.pending_buf_size-n.pending;if(n.pending_buf.set(n.gzhead.extra.subarray(n.gzindex,n.gzindex+o),n.pending),n.pending=n.pending_buf_size,n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),n.gzindex+=o,$n(e),n.pending!==0)return n.last_flush=-1,rn;i=0,l-=o}let g=new Uint8Array(n.gzhead.extra);n.pending_buf.set(g.subarray(n.gzindex,n.gzindex+l),n.pending),n.pending+=l,n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),n.gzindex=0}n.status=Mh}if(n.status===Mh){if(n.gzhead.name){let i=n.pending,l;do{if(n.pending===n.pending_buf_size){if(n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),$n(e),n.pending!==0)return n.last_flush=-1,rn;i=0}n.gzindexi&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),n.gzindex=0}n.status=Lh}if(n.status===Lh){if(n.gzhead.comment){let i=n.pending,l;do{if(n.pending===n.pending_buf_size){if(n.gzhead.hcrc&&n.pending>i&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i)),$n(e),n.pending!==0)return n.last_flush=-1,rn;i=0}n.gzindexi&&(e.adler=Zt(e.adler,n.pending_buf,n.pending-i,i))}n.status=Rh}if(n.status===Rh){if(n.gzhead.hcrc){if(n.pending+2>n.pending_buf_size&&($n(e),n.pending!==0))return n.last_flush=-1,rn;mt(n,e.adler&255),mt(n,e.adler>>8&255),e.adler=0}if(n.status=wa,$n(e),n.pending!==0)return n.last_flush=-1,rn}if(e.avail_in!==0||n.lookahead!==0||t!==Bi&&n.status!==ms){let i=n.level===0?V2(n,t):n.strategy===_u?mY(n,t):n.strategy===ZW?pY(n,t):gs[n.level].func(n,t);if((i===Ba||i===Jo)&&(n.status=ms),i===wn||i===Ba)return e.avail_out===0&&(n.last_flush=-1),rn;if(i===Zo&&(t===YW?WW(n):t!==ny&&(Ah(n,0,0,!1),t===XW&&(Ai(n.head),n.lookahead===0&&(n.strstart=0,n.block_start=0,n.insert=0))),$n(e),e.avail_out===0))return n.last_flush=-1,rn}return t!==Gn?rn:n.wrap<=0?ry:(n.wrap===2?(mt(n,e.adler&255),mt(n,e.adler>>8&255),mt(n,e.adler>>16&255),mt(n,e.adler>>24&255),mt(n,e.total_in&255),mt(n,e.total_in>>8&255),mt(n,e.total_in>>16&255),mt(n,e.total_in>>24&255)):(us(n,e.adler>>>16),us(n,e.adler&65535)),$n(e),n.wrap>0&&(n.wrap=-n.wrap),n.pending!==0?rn:ry)},wY=e=>{if(Ul(e))return Pr;const t=e.state.status;return e.state=null,t===wa?Ca(e,KW):rn},CY=(e,t)=>{let n=t.length;if(Ul(e))return Pr;const r=e.state,i=r.wrap;if(i===2||i===1&&r.status!==$o||r.lookahead)return Pr;if(i===1&&(e.adler=yl(e.adler,t,n,0)),r.wrap=0,n>=r.w_size){i===0&&(Ai(r.head),r.strstart=0,r.block_start=0,r.insert=0);let a=new Uint8Array(r.w_size);a.set(t.subarray(n-r.w_size,n),0),t=a,n=r.w_size}const l=e.avail_in,g=e.next_in,o=e.input;for(e.avail_in=n,e.next_in=0,e.input=t,Bo(r);r.lookahead>=lt;){let a=r.strstart,u=r.lookahead-(lt-1);do r.ins_h=Fi(r,r.ins_h,r.window[a+lt-1]),r.prev[a&r.w_mask]=r.head[r.ins_h],r.head[r.ins_h]=a,a++;while(--u);r.strstart=a,r.lookahead=lt-1,Bo(r)}return r.strstart+=r.lookahead,r.block_start=r.strstart,r.insert=r.lookahead,r.lookahead=0,r.match_length=r.prev_length=lt-1,r.match_available=0,e.next_in=g,e.input=o,e.avail_in=l,r.wrap=i,rn};var SY=bY,xY=W2,TY=H2,kY=U2,EY=yY,OY=_Y,AY=wY,PY=CY,IY="pako deflate (from Nodeca project)",Ds={deflateInit:SY,deflateInit2:xY,deflateReset:TY,deflateResetKeep:kY,deflateSetHeader:EY,deflate:OY,deflateEnd:AY,deflateSetDictionary:PY,deflateInfo:IY};const MY=(e,t)=>Object.prototype.hasOwnProperty.call(e,t);var LY=function(e){const t=Array.prototype.slice.call(arguments,1);for(;t.length;){const n=t.shift();if(!!n){if(typeof n!="object")throw new TypeError(n+"must be non-object");for(const r in n)MY(n,r)&&(e[r]=n[r])}}return e},RY=e=>{let t=0;for(let r=0,i=e.length;r=252?6:e>=248?5:e>=240?4:e>=224?3:e>=192?2:1;bl[254]=bl[254]=1;var DY=e=>{if(typeof TextEncoder=="function"&&TextEncoder.prototype.encode)return new TextEncoder().encode(e);let t,n,r,i,l,g=e.length,o=0;for(i=0;i>>6,t[l++]=128|n&63):n<65536?(t[l++]=224|n>>>12,t[l++]=128|n>>>6&63,t[l++]=128|n&63):(t[l++]=240|n>>>18,t[l++]=128|n>>>12&63,t[l++]=128|n>>>6&63,t[l++]=128|n&63);return t};const $Y=(e,t)=>{if(t<65534&&e.subarray&&Y2)return String.fromCharCode.apply(null,e.length===t?e:e.subarray(0,t));let n="";for(let r=0;r{const n=t||e.length;if(typeof TextDecoder=="function"&&TextDecoder.prototype.decode)return new TextDecoder().decode(e.subarray(0,t));let r,i;const l=new Array(n*2);for(i=0,r=0;r4){l[i++]=65533,r+=o-1;continue}for(g&=o===2?31:o===3?15:7;o>1&&r1){l[i++]=65533;continue}g<65536?l[i++]=g:(g-=65536,l[i++]=55296|g>>10&1023,l[i++]=56320|g&1023)}return $Y(l,i)},FY=(e,t)=>{t=t||e.length,t>e.length&&(t=e.length);let n=t-1;for(;n>=0&&(e[n]&192)===128;)n--;return n<0||n===0?t:n+bl[e[n]]>t?n:t},_l={string2buf:DY,buf2string:BY,utf8border:FY};function zY(){this.input=null,this.next_in=0,this.avail_in=0,this.total_in=0,this.output=null,this.next_out=0,this.avail_out=0,this.total_out=0,this.msg="",this.state=null,this.data_type=2,this.adler=0}var X2=zY;const K2=Object.prototype.toString,{Z_NO_FLUSH:NY,Z_SYNC_FLUSH:jY,Z_FULL_FLUSH:VY,Z_FINISH:UY,Z_OK:Ec,Z_STREAM_END:HY,Z_DEFAULT_COMPRESSION:WY,Z_DEFAULT_STRATEGY:YY,Z_DEFLATED:XY}=qo;function Hl(e){this.options=uf.assign({level:WY,method:XY,chunkSize:16384,windowBits:15,memLevel:8,strategy:YY},e||{});let t=this.options;t.raw&&t.windowBits>0?t.windowBits=-t.windowBits:t.gzip&&t.windowBits>0&&t.windowBits<16&&(t.windowBits+=16),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new X2,this.strm.avail_out=0;let n=Ds.deflateInit2(this.strm,t.level,t.method,t.windowBits,t.memLevel,t.strategy);if(n!==Ec)throw new Error($a[n]);if(t.header&&Ds.deflateSetHeader(this.strm,t.header),t.dictionary){let r;if(typeof t.dictionary=="string"?r=_l.string2buf(t.dictionary):K2.call(t.dictionary)==="[object ArrayBuffer]"?r=new Uint8Array(t.dictionary):r=t.dictionary,n=Ds.deflateSetDictionary(this.strm,r),n!==Ec)throw new Error($a[n]);this._dict_set=!0}}Hl.prototype.push=function(e,t){const n=this.strm,r=this.options.chunkSize;let i,l;if(this.ended)return!1;for(t===~~t?l=t:l=t===!0?UY:NY,typeof e=="string"?n.input=_l.string2buf(e):K2.call(e)==="[object ArrayBuffer]"?n.input=new Uint8Array(e):n.input=e,n.next_in=0,n.avail_in=n.input.length;;){if(n.avail_out===0&&(n.output=new Uint8Array(r),n.next_out=0,n.avail_out=r),(l===jY||l===VY)&&n.avail_out<=6){this.onData(n.output.subarray(0,n.next_out)),n.avail_out=0;continue}if(i=Ds.deflate(n,l),i===HY)return n.next_out>0&&this.onData(n.output.subarray(0,n.next_out)),i=Ds.deflateEnd(this.strm),this.onEnd(i),this.ended=!0,i===Ec;if(n.avail_out===0){this.onData(n.output);continue}if(l>0&&n.next_out>0){this.onData(n.output.subarray(0,n.next_out)),n.avail_out=0;continue}if(n.avail_in===0)break}return!0};Hl.prototype.onData=function(e){this.chunks.push(e)};Hl.prototype.onEnd=function(e){e===Ec&&(this.result=uf.flattenChunks(this.chunks)),this.chunks=[],this.err=e,this.msg=this.strm.msg};function Cm(e,t){const n=new Hl(t);if(n.push(e,!0),n.err)throw n.msg||$a[n.err];return n.result}function KY(e,t){return t=t||{},t.raw=!0,Cm(e,t)}function GY(e,t){return t=t||{},t.gzip=!0,Cm(e,t)}var qY=Hl,ZY=Cm,JY=KY,QY=GY,eX=qo,tX={Deflate:qY,deflate:ZY,deflateRaw:JY,gzip:QY,constants:eX};const wu=16209,nX=16191;var rX=function(t,n){let r,i,l,g,o,a,u,d,c,f,s,h,v,m,y,b,_,w,S,x,T,E,I,D;const N=t.state;r=t.next_in,I=t.input,i=r+(t.avail_in-5),l=t.next_out,D=t.output,g=l-(n-t.avail_out),o=l+(t.avail_out-257),a=N.dmax,u=N.wsize,d=N.whave,c=N.wnext,f=N.window,s=N.hold,h=N.bits,v=N.lencode,m=N.distcode,y=(1<>>24,s>>>=w,h-=w,w=_>>>16&255,w===0)D[l++]=_&65535;else if(w&16){S=_&65535,w&=15,w&&(h>>=w,h-=w),h<15&&(s+=I[r++]<>>24,s>>>=w,h-=w,w=_>>>16&255,w&16){if(x=_&65535,w&=15,ha){t.msg="invalid distance too far back",N.mode=wu;break e}if(s>>>=w,h-=w,w=l-g,x>w){if(w=x-w,w>d&&N.sane){t.msg="invalid distance too far back",N.mode=wu;break e}if(T=0,E=f,c===0){if(T+=u-w,w2;)D[l++]=E[T++],D[l++]=E[T++],D[l++]=E[T++],S-=3;S&&(D[l++]=E[T++],S>1&&(D[l++]=E[T++]))}else{T=l-x;do D[l++]=D[T++],D[l++]=D[T++],D[l++]=D[T++],S-=3;while(S>2);S&&(D[l++]=D[T++],S>1&&(D[l++]=D[T++]))}}else if((w&64)===0){_=m[(_&65535)+(s&(1<>3,r-=S,h-=S<<3,s&=(1<{const a=o.bits;let u=0,d=0,c=0,f=0,s=0,h=0,v=0,m=0,y=0,b=0,_,w,S,x,T,E=null,I;const D=new Uint16Array(oo+1),N=new Uint16Array(oo+1);let L=null,F,O,V;for(u=0;u<=oo;u++)D[u]=0;for(d=0;d=1&&D[f]===0;f--);if(s>f&&(s=f),f===0)return i[l++]=1<<24|64<<16|0,i[l++]=1<<24|64<<16|0,o.bits=1,0;for(c=1;c0&&(e===sy||f!==1))return-1;for(N[1]=0,u=1;uay||e===ly&&y>oy)return 1;for(;;){F=u-v,g[d]+1=I?(O=L[g[d]-I],V=E[g[d]-I]):(O=32+64,V=0),_=1<>v)+w]=F<<24|O<<16|V|0;while(w!==0);for(_=1<>=1;if(_!==0?(b&=_-1,b+=_):b=0,d++,--D[u]===0){if(u===f)break;u=t[n+g[d]]}if(u>s&&(b&x)!==S){for(v===0&&(v=s),T+=c,h=u-v,m=1<ay||e===ly&&y>oy)return 1;S=b&x,i[S]=s<<24|h<<16|T-l|0}}return b!==0&&(i[T+b]=u-v<<24|64<<16|0),o.bits=s,0};var $s=lX;const uX=0,G2=1,q2=2,{Z_FINISH:uy,Z_BLOCK:cX,Z_TREES:Cu,Z_OK:Fa,Z_STREAM_END:fX,Z_NEED_DICT:dX,Z_STREAM_ERROR:tr,Z_DATA_ERROR:Z2,Z_MEM_ERROR:J2,Z_BUF_ERROR:hX,Z_DEFLATED:cy}=qo,cf=16180,fy=16181,dy=16182,hy=16183,py=16184,my=16185,gy=16186,vy=16187,yy=16188,by=16189,Oc=16190,Ur=16191,ad=16192,_y=16193,od=16194,wy=16195,Cy=16196,Sy=16197,xy=16198,Su=16199,xu=16200,Ty=16201,ky=16202,Ey=16203,Oy=16204,Ay=16205,sd=16206,Py=16207,Iy=16208,Bt=16209,Q2=16210,eC=16211,pX=852,mX=592,gX=15,vX=gX,My=e=>(e>>>24&255)+(e>>>8&65280)+((e&65280)<<8)+((e&255)<<24);function yX(){this.strm=null,this.mode=0,this.last=!1,this.wrap=0,this.havedict=!1,this.flags=0,this.dmax=0,this.check=0,this.total=0,this.head=null,this.wbits=0,this.wsize=0,this.whave=0,this.wnext=0,this.window=null,this.hold=0,this.bits=0,this.length=0,this.offset=0,this.extra=0,this.lencode=null,this.distcode=null,this.lenbits=0,this.distbits=0,this.ncode=0,this.nlen=0,this.ndist=0,this.have=0,this.next=null,this.lens=new Uint16Array(320),this.work=new Uint16Array(288),this.lendyn=null,this.distdyn=null,this.sane=0,this.back=0,this.was=0}const Ga=e=>{if(!e)return 1;const t=e.state;return!t||t.strm!==e||t.modeeC?1:0},tC=e=>{if(Ga(e))return tr;const t=e.state;return e.total_in=e.total_out=t.total=0,e.msg="",t.wrap&&(e.adler=t.wrap&1),t.mode=cf,t.last=0,t.havedict=0,t.flags=-1,t.dmax=32768,t.head=null,t.hold=0,t.bits=0,t.lencode=t.lendyn=new Int32Array(pX),t.distcode=t.distdyn=new Int32Array(mX),t.sane=1,t.back=-1,Fa},nC=e=>{if(Ga(e))return tr;const t=e.state;return t.wsize=0,t.whave=0,t.wnext=0,tC(e)},rC=(e,t)=>{let n;if(Ga(e))return tr;const r=e.state;return t<0?(n=0,t=-t):(n=(t>>4)+5,t<48&&(t&=15)),t&&(t<8||t>15)?tr:(r.window!==null&&r.wbits!==t&&(r.window=null),r.wrap=n,r.wbits=t,nC(e))},iC=(e,t)=>{if(!e)return tr;const n=new yX;e.state=n,n.strm=e,n.window=null,n.mode=cf;const r=rC(e,t);return r!==Fa&&(e.state=null),r},bX=e=>iC(e,vX);let Ly=!0,ld,ud;const _X=e=>{if(Ly){ld=new Int32Array(512),ud=new Int32Array(32);let t=0;for(;t<144;)e.lens[t++]=8;for(;t<256;)e.lens[t++]=9;for(;t<280;)e.lens[t++]=7;for(;t<288;)e.lens[t++]=8;for($s(G2,e.lens,0,288,ld,0,e.work,{bits:9}),t=0;t<32;)e.lens[t++]=5;$s(q2,e.lens,0,32,ud,0,e.work,{bits:5}),Ly=!1}e.lencode=ld,e.lenbits=9,e.distcode=ud,e.distbits=5},aC=(e,t,n,r)=>{let i;const l=e.state;return l.window===null&&(l.wsize=1<=l.wsize?(l.window.set(t.subarray(n-l.wsize,n),0),l.wnext=0,l.whave=l.wsize):(i=l.wsize-l.wnext,i>r&&(i=r),l.window.set(t.subarray(n-r,n-r+i),l.wnext),r-=i,r?(l.window.set(t.subarray(n-r,n),0),l.wnext=r,l.whave=l.wsize):(l.wnext+=i,l.wnext===l.wsize&&(l.wnext=0),l.whave{let n,r,i,l,g,o,a,u,d,c,f,s,h,v,m=0,y,b,_,w,S,x,T,E;const I=new Uint8Array(4);let D,N;const L=new Uint8Array([16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15]);if(Ga(e)||!e.output||!e.input&&e.avail_in!==0)return tr;n=e.state,n.mode===Ur&&(n.mode=ad),g=e.next_out,i=e.output,a=e.avail_out,l=e.next_in,r=e.input,o=e.avail_in,u=n.hold,d=n.bits,c=o,f=a,E=Fa;e:for(;;)switch(n.mode){case cf:if(n.wrap===0){n.mode=ad;break}for(;d<16;){if(o===0)break e;o--,u+=r[l++]<>>8&255,n.check=Zt(n.check,I,2,0),u=0,d=0,n.mode=fy;break}if(n.head&&(n.head.done=!1),!(n.wrap&1)||(((u&255)<<8)+(u>>8))%31){e.msg="incorrect header check",n.mode=Bt;break}if((u&15)!==cy){e.msg="unknown compression method",n.mode=Bt;break}if(u>>>=4,d-=4,T=(u&15)+8,n.wbits===0&&(n.wbits=T),T>15||T>n.wbits){e.msg="invalid window size",n.mode=Bt;break}n.dmax=1<>8&1),n.flags&512&&n.wrap&4&&(I[0]=u&255,I[1]=u>>>8&255,n.check=Zt(n.check,I,2,0)),u=0,d=0,n.mode=dy;case dy:for(;d<32;){if(o===0)break e;o--,u+=r[l++]<>>8&255,I[2]=u>>>16&255,I[3]=u>>>24&255,n.check=Zt(n.check,I,4,0)),u=0,d=0,n.mode=hy;case hy:for(;d<16;){if(o===0)break e;o--,u+=r[l++]<>8),n.flags&512&&n.wrap&4&&(I[0]=u&255,I[1]=u>>>8&255,n.check=Zt(n.check,I,2,0)),u=0,d=0,n.mode=py;case py:if(n.flags&1024){for(;d<16;){if(o===0)break e;o--,u+=r[l++]<>>8&255,n.check=Zt(n.check,I,2,0)),u=0,d=0}else n.head&&(n.head.extra=null);n.mode=my;case my:if(n.flags&1024&&(s=n.length,s>o&&(s=o),s&&(n.head&&(T=n.head.extra_len-n.length,n.head.extra||(n.head.extra=new Uint8Array(n.head.extra_len)),n.head.extra.set(r.subarray(l,l+s),T)),n.flags&512&&n.wrap&4&&(n.check=Zt(n.check,r,s,l)),o-=s,l+=s,n.length-=s),n.length))break e;n.length=0,n.mode=gy;case gy:if(n.flags&2048){if(o===0)break e;s=0;do T=r[l+s++],n.head&&T&&n.length<65536&&(n.head.name+=String.fromCharCode(T));while(T&&s>9&1,n.head.done=!0),e.adler=n.check=0,n.mode=Ur;break;case by:for(;d<32;){if(o===0)break e;o--,u+=r[l++]<>>=d&7,d-=d&7,n.mode=sd;break}for(;d<3;){if(o===0)break e;o--,u+=r[l++]<>>=1,d-=1,u&3){case 0:n.mode=_y;break;case 1:if(_X(n),n.mode=Su,t===Cu){u>>>=2,d-=2;break e}break;case 2:n.mode=Cy;break;case 3:e.msg="invalid block type",n.mode=Bt}u>>>=2,d-=2;break;case _y:for(u>>>=d&7,d-=d&7;d<32;){if(o===0)break e;o--,u+=r[l++]<>>16^65535)){e.msg="invalid stored block lengths",n.mode=Bt;break}if(n.length=u&65535,u=0,d=0,n.mode=od,t===Cu)break e;case od:n.mode=wy;case wy:if(s=n.length,s){if(s>o&&(s=o),s>a&&(s=a),s===0)break e;i.set(r.subarray(l,l+s),g),o-=s,l+=s,a-=s,g+=s,n.length-=s;break}n.mode=Ur;break;case Cy:for(;d<14;){if(o===0)break e;o--,u+=r[l++]<>>=5,d-=5,n.ndist=(u&31)+1,u>>>=5,d-=5,n.ncode=(u&15)+4,u>>>=4,d-=4,n.nlen>286||n.ndist>30){e.msg="too many length or distance symbols",n.mode=Bt;break}n.have=0,n.mode=Sy;case Sy:for(;n.have>>=3,d-=3}for(;n.have<19;)n.lens[L[n.have++]]=0;if(n.lencode=n.lendyn,n.lenbits=7,D={bits:n.lenbits},E=$s(uX,n.lens,0,19,n.lencode,0,n.work,D),n.lenbits=D.bits,E){e.msg="invalid code lengths set",n.mode=Bt;break}n.have=0,n.mode=xy;case xy:for(;n.have>>24,b=m>>>16&255,_=m&65535,!(y<=d);){if(o===0)break e;o--,u+=r[l++]<>>=y,d-=y,n.lens[n.have++]=_;else{if(_===16){for(N=y+2;d>>=y,d-=y,n.have===0){e.msg="invalid bit length repeat",n.mode=Bt;break}T=n.lens[n.have-1],s=3+(u&3),u>>>=2,d-=2}else if(_===17){for(N=y+3;d>>=y,d-=y,T=0,s=3+(u&7),u>>>=3,d-=3}else{for(N=y+7;d>>=y,d-=y,T=0,s=11+(u&127),u>>>=7,d-=7}if(n.have+s>n.nlen+n.ndist){e.msg="invalid bit length repeat",n.mode=Bt;break}for(;s--;)n.lens[n.have++]=T}}if(n.mode===Bt)break;if(n.lens[256]===0){e.msg="invalid code -- missing end-of-block",n.mode=Bt;break}if(n.lenbits=9,D={bits:n.lenbits},E=$s(G2,n.lens,0,n.nlen,n.lencode,0,n.work,D),n.lenbits=D.bits,E){e.msg="invalid literal/lengths set",n.mode=Bt;break}if(n.distbits=6,n.distcode=n.distdyn,D={bits:n.distbits},E=$s(q2,n.lens,n.nlen,n.ndist,n.distcode,0,n.work,D),n.distbits=D.bits,E){e.msg="invalid distances set",n.mode=Bt;break}if(n.mode=Su,t===Cu)break e;case Su:n.mode=xu;case xu:if(o>=6&&a>=258){e.next_out=g,e.avail_out=a,e.next_in=l,e.avail_in=o,n.hold=u,n.bits=d,rX(e,f),g=e.next_out,i=e.output,a=e.avail_out,l=e.next_in,r=e.input,o=e.avail_in,u=n.hold,d=n.bits,n.mode===Ur&&(n.back=-1);break}for(n.back=0;m=n.lencode[u&(1<>>24,b=m>>>16&255,_=m&65535,!(y<=d);){if(o===0)break e;o--,u+=r[l++]<>w)],y=m>>>24,b=m>>>16&255,_=m&65535,!(w+y<=d);){if(o===0)break e;o--,u+=r[l++]<>>=w,d-=w,n.back+=w}if(u>>>=y,d-=y,n.back+=y,n.length=_,b===0){n.mode=Ay;break}if(b&32){n.back=-1,n.mode=Ur;break}if(b&64){e.msg="invalid literal/length code",n.mode=Bt;break}n.extra=b&15,n.mode=Ty;case Ty:if(n.extra){for(N=n.extra;d>>=n.extra,d-=n.extra,n.back+=n.extra}n.was=n.length,n.mode=ky;case ky:for(;m=n.distcode[u&(1<>>24,b=m>>>16&255,_=m&65535,!(y<=d);){if(o===0)break e;o--,u+=r[l++]<>w)],y=m>>>24,b=m>>>16&255,_=m&65535,!(w+y<=d);){if(o===0)break e;o--,u+=r[l++]<>>=w,d-=w,n.back+=w}if(u>>>=y,d-=y,n.back+=y,b&64){e.msg="invalid distance code",n.mode=Bt;break}n.offset=_,n.extra=b&15,n.mode=Ey;case Ey:if(n.extra){for(N=n.extra;d>>=n.extra,d-=n.extra,n.back+=n.extra}if(n.offset>n.dmax){e.msg="invalid distance too far back",n.mode=Bt;break}n.mode=Oy;case Oy:if(a===0)break e;if(s=f-a,n.offset>s){if(s=n.offset-s,s>n.whave&&n.sane){e.msg="invalid distance too far back",n.mode=Bt;break}s>n.wnext?(s-=n.wnext,h=n.wsize-s):h=n.wnext-s,s>n.length&&(s=n.length),v=n.window}else v=i,h=g-n.offset,s=n.length;s>a&&(s=a),a-=s,n.length-=s;do i[g++]=v[h++];while(--s);n.length===0&&(n.mode=xu);break;case Ay:if(a===0)break e;i[g++]=n.length,a--,n.mode=xu;break;case sd:if(n.wrap){for(;d<32;){if(o===0)break e;o--,u|=r[l++]<{if(Ga(e))return tr;let t=e.state;return t.window&&(t.window=null),e.state=null,Fa},SX=(e,t)=>{if(Ga(e))return tr;const n=e.state;return(n.wrap&2)===0?tr:(n.head=t,t.done=!1,Fa)},xX=(e,t)=>{const n=t.length;let r,i,l;return Ga(e)||(r=e.state,r.wrap!==0&&r.mode!==Oc)?tr:r.mode===Oc&&(i=1,i=yl(i,t,n,0),i!==r.check)?Z2:(l=aC(e,t,n,n),l?(r.mode=Q2,J2):(r.havedict=1,Fa))};var TX=nC,kX=rC,EX=tC,OX=bX,AX=iC,PX=wX,IX=CX,MX=SX,LX=xX,RX="pako inflate (from Nodeca project)",qr={inflateReset:TX,inflateReset2:kX,inflateResetKeep:EX,inflateInit:OX,inflateInit2:AX,inflate:PX,inflateEnd:IX,inflateGetHeader:MX,inflateSetDictionary:LX,inflateInfo:RX};function DX(){this.text=0,this.time=0,this.xflags=0,this.os=0,this.extra=null,this.extra_len=0,this.name="",this.comment="",this.hcrc=0,this.done=!1}var $X=DX;const oC=Object.prototype.toString,{Z_NO_FLUSH:BX,Z_FINISH:FX,Z_OK:wl,Z_STREAM_END:cd,Z_NEED_DICT:fd,Z_STREAM_ERROR:zX,Z_DATA_ERROR:Ry,Z_MEM_ERROR:NX}=qo;function Wl(e){this.options=uf.assign({chunkSize:1024*64,windowBits:15,to:""},e||{});const t=this.options;t.raw&&t.windowBits>=0&&t.windowBits<16&&(t.windowBits=-t.windowBits,t.windowBits===0&&(t.windowBits=-15)),t.windowBits>=0&&t.windowBits<16&&!(e&&e.windowBits)&&(t.windowBits+=32),t.windowBits>15&&t.windowBits<48&&(t.windowBits&15)===0&&(t.windowBits|=15),this.err=0,this.msg="",this.ended=!1,this.chunks=[],this.strm=new X2,this.strm.avail_out=0;let n=qr.inflateInit2(this.strm,t.windowBits);if(n!==wl)throw new Error($a[n]);if(this.header=new $X,qr.inflateGetHeader(this.strm,this.header),t.dictionary&&(typeof t.dictionary=="string"?t.dictionary=_l.string2buf(t.dictionary):oC.call(t.dictionary)==="[object ArrayBuffer]"&&(t.dictionary=new Uint8Array(t.dictionary)),t.raw&&(n=qr.inflateSetDictionary(this.strm,t.dictionary),n!==wl)))throw new Error($a[n])}Wl.prototype.push=function(e,t){const n=this.strm,r=this.options.chunkSize,i=this.options.dictionary;let l,g,o;if(this.ended)return!1;for(t===~~t?g=t:g=t===!0?FX:BX,oC.call(e)==="[object ArrayBuffer]"?n.input=new Uint8Array(e):n.input=e,n.next_in=0,n.avail_in=n.input.length;;){for(n.avail_out===0&&(n.output=new Uint8Array(r),n.next_out=0,n.avail_out=r),l=qr.inflate(n,g),l===fd&&i&&(l=qr.inflateSetDictionary(n,i),l===wl?l=qr.inflate(n,g):l===Ry&&(l=fd));n.avail_in>0&&l===cd&&n.state.wrap>0&&e[n.next_in]!==0;)qr.inflateReset(n),l=qr.inflate(n,g);switch(l){case zX:case Ry:case fd:case NX:return this.onEnd(l),this.ended=!0,!1}if(o=n.avail_out,n.next_out&&(n.avail_out===0||l===cd))if(this.options.to==="string"){let a=_l.utf8border(n.output,n.next_out),u=n.next_out-a,d=_l.buf2string(n.output,a);n.next_out=u,n.avail_out=r-u,u&&n.output.set(n.output.subarray(a,a+u),0),this.onData(d)}else this.onData(n.output.length===n.next_out?n.output:n.output.subarray(0,n.next_out));if(!(l===wl&&o===0)){if(l===cd)return l=qr.inflateEnd(this.strm),this.onEnd(l),this.ended=!0,!0;if(n.avail_in===0)break}}return!0};Wl.prototype.onData=function(e){this.chunks.push(e)};Wl.prototype.onEnd=function(e){e===wl&&(this.options.to==="string"?this.result=this.chunks.join(""):this.result=uf.flattenChunks(this.chunks)),this.chunks=[],this.err=e,this.msg=this.strm.msg};function Sm(e,t){const n=new Wl(t);if(n.push(e),n.err)throw n.msg||$a[n.err];return n.result}function jX(e,t){return t=t||{},t.raw=!0,Sm(e,t)}var VX=Wl,UX=Sm,HX=jX,WX=Sm,YX=qo,XX={Inflate:VX,inflate:UX,inflateRaw:HX,ungzip:WX,constants:YX};const{Deflate:oq,deflate:sq,deflateRaw:KX,gzip:lq}=tX,{Inflate:uq,inflate:cq,inflateRaw:GX,ungzip:fq}=XX;var qX=KX,ZX=GX;function Dy(e){const t=new Map;for(const n of e){const[r,i]=n.split("="),l=decodeURIComponent(i);t.set(r,l)}return t}const JX=function(){if(!window.location.search.includes("?"))return;const t=window.location.search.replace("?","").split("&");let n=Dy(t);if(console.log("URL params:",n),n.get("share")){const l=ZX(new Uint8Array(atob(n.get("share")).split("").map(g=>g.charCodeAt(0))),{to:"string"});if(!l){Ut().raiseError("Error when trying to decode share parameter!",!1);return}n=Dy(l.split("&")),console.log("Share URL params:",n)}const r={id:-1,image:"",prompt:n.get("prompt")||"",sampler_name:n.get("sampler_name")||"k_euler",seed:Number(n.get("seed"))||-1,steps:Number(n.get("steps")||20),cfg_scale:Number(n.get("cfg_scale")||5),height:Number(n.get("height")||512),width:Number(n.get("width")||512),clip_skip:Number(n.get("clip_skip")||0),frames:Number(n.get("frames")||1),fps:Number(n.get("fps")||16),scheduler:n.get("scheduler")||"default"};Jt().generateText2Img(r,!1)},QX=10;function Ku(e,t,n,r,i=l=>Ut().raiseError(l,!1)){if(e.status===n&&t)return!0;if(!t.message)return i(`${r}: Got response code ${e.status}`);if(!t.errors)return i(`${r}: ${t.message}`);const l=Object.entries(t.errors).map(g=>`${g[0]} - ${g[1]}`).join(" | ");return i(`${r}: ${t.message} (${l})`)}const eK=zo("interrogate",()=>{const e=oe({}),t=oe(!1);async function n(g){Ut().raiseError(g,!1),t.value=!1,e.value={}}async function r(){const g=zt(),{source_image:o}=e.value;if(!o)return n("Failed to get interrogation ID: No image supplied.");t.value=!0;const a=await fetch(`${g.baseURL.length===0?".":g.baseURL}/sdapi/v1/interrogate`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({image:o.split(",")[1],model:"clip"})}),u=await a.json();!Ku(a,u,200,"Failed to get interrogation",n)||(e.value.id=u.id,e.value.status=u.caption)}function i(){e.value={},t.value=!1}function l(){return e.value.status||!1}return{currentInterrogation:e,interrogating:t,interrogateImage:r,getFormStatus:l,resetInterrogation:i}}),xm=e=>(ui("data-v-db184ac6"),e=e(),ci(),e),tK={key:0,style:{"margin-top":"16px"}},nK=xm(()=>ne("div",null,[He("Drop file here OR "),ne("em",null,"click to upload")],-1)),rK={key:1,style:{"margin-top":"16px"}},iK={key:2},aK={style:{"margin-top":"8px"}},oK=xm(()=>ne("h2",{style:{margin:"16px 0 8px 0"}},"Interrogation Results",-1)),sK={key:0},lK=xm(()=>ne("h3",null,"Caption",-1)),uK={key:0},cK={key:1},fK=Ee({__name:"InterrogationView",setup(e){const t=eK(),n=Jt(),r=Ut(),i=oe();async function l(u){if(i.value.clearFiles(),!u.raw.type.includes("image")){r.raiseError("Uploaded file needs to be a image!",!1);return}const d=await Tc(u.raw);t.currentInterrogation.source_image=d,t.interrogateImage()}function g(){n.generateText2Img({prompt:o.value})}const o=ee(()=>t.getFormStatus()),{ellipsis:a}=x2();return(u,d)=>C(t).currentInterrogation.source_image?C(t).currentInterrogation.status?(z(),se("div",iK,[ne("div",aK,[le(C(at),{icon:C(Mg),onClick:C(t).resetInterrogation},{default:he(()=>[He("New Interrogation")]),_:1},8,["icon","onClick"]),C(o)?(z(),be(C(at),{key:0,icon:C(Mg),onClick:g,disabled:!C(o)},{default:he(()=>[He("Text2Img (Caption)")]),_:1},8,["icon","disabled"])):ye("",!0)]),oK,le(C(ef),{src:C(t).currentInterrogation.source_image,alt:"Uploaded Image"},null,8,["src"]),C(o)?(z(),se("div",sK,[lK,C(o)?(z(),se("div",cK,[ne("strong",null,Ae(C(o)),1)])):(z(),se("div",uK,"Processing"+Ae(C(a)),1))])):ye("",!0)])):(z(),se("div",rK,[ne("strong",null,"Uploading image"+Ae(C(a)),1)])):(z(),se("div",tK,[ne("div",null,[le(C(Yp),{onChange:l,"auto-upload":!1,limit:1,class:"interrogation-upload",ref_key:"upload",ref:i,multiple:"",drag:""},{default:he(()=>[le(C(Le),{size:100},{default:he(()=>[le(C(dp))]),_:1}),nK]),_:1},512)])]))}});const dK=qt(fK,[["__scopeId","data-v-db184ac6"]]);function hK(e,t,n){if(e===0)return"0"+(t?"s":"seconds");if(e==null)return"?";const r=Math.floor(e/86400),i=Math.floor(e%86400/3600),l=Math.floor(e%86400%3600/60),g=Math.floor(e%86400%3600%60),o=r>0?r+(t?"d":"days"):"",a=i>0?i+(t?"h":"hours"):"",u=l>0?l+(t?"m":"minutes"):"",d=g>0?g+(t?"s":"seconds"):"",c=[];return n!=null&&n.days&&c.push(o),n!=null&&n.hours&&c.push(a),n!=null&&n.minutes&&c.push(u),n!=null&&n.seconds&&c.push(d),c.join(" ")}const pK={class:"form"},mK={key:0,style:{"padding-bottom":"50px"}},gK=ne("h1",{style:{margin:"0"}},"Interrogation",-1),vK=ne("div",null,"Interrogate images to get their predicted descriptions.",-1),yK={class:"sidebar"},bK={class:"reference-images-header"},_K=ne("span",{class:"reference-images-label"},"Reference Images",-1),wK={class:"reference-images-actions"},CK={key:0,class:"reference-image-list"},SK={class:"reference-image-index"},xK=["title"],TK={class:"reference-image-controls"},kK={key:1,class:"reference-image-empty"},EK={class:"main"},OK={class:"image center-horizontal"},AK={key:0},PK=Ee({__name:"GenerateView",setup(e){const n=Ep(kp).smallerOrEqual("md"),r=Jt(),i=Ut(),l=Ls();zt();const g=gv(async()=>{const _=r.cacheVersion,S=(await r.getAvailableSamplers()).map(x=>x.name);return S.length===0?[]:u(S)},[]),o=gv(async()=>{const _=r.cacheVersion,S=(await r.getAvailableSchedulers()).map(x=>x.name);return S.length===0?[]:d(S)},[]),a=yt({prompt:[{required:!0,message:"Please input prompt",trigger:"change"}]});function u(_){return!r.params||!r.params.sampler_name||_.indexOf(r.params.sampler_name)===-1&&(r.params.sampler_name=_[0]),_}function d(_){return!r.params||!r.params.scheduler||_.indexOf(r.params.scheduler)===-1&&(r.params.scheduler=_[0]),_}function c(_){return"Elapsed: "+hK(_,!0,{days:!0,hours:!0,minutes:!0,seconds:!0})}function f(){r.validGeneratorTypes.includes(r.generatorType)||(i.showGeneratorBadge=!1)}function s(_){r.generatorType=_,f(),console.log(_)}function h(){l.showCropPreview=!0,l.updateCropPreview()}function v(){var _;(_=document.getElementById("extra_image_input"))==null||_.click()}function m(_){var w;return(w=_.dataUrl)==null?void 0:w.startsWith("data:image")}function y(){return r.referenceImages.filter(m).map(_=>_.dataUrl)}function b(_){return r.referenceImages.slice(0,_).filter(m).length}return f(),JX(),(_,w)=>(z(),se(je,null,[le(C(mw),{"default-active":C(r).generatorType,collapse:!0,onSelect:s,mode:C(n)?"horizontal":"vertical",class:de(C(n)?"mobile-generator-types":"generator-types"),style:ze(C(n)?"overflow-x: auto":"")},{default:he(()=>[le(bu,{index:"Text2Img","icon-one":C(EE),"icon-two":C(tu),isMobile:C(n)},null,8,["icon-one","icon-two","isMobile"]),le(bu,{index:"Img2Img","icon-one":C(tu),"icon-two":C(tu),isMobile:C(n)},null,8,["icon-one","icon-two","isMobile"]),le(bu,{index:"Inpainting","icon-one":S2,"icon-two":C(tu),isMobile:C(n)},null,8,["icon-two","isMobile"]),le(bu,{index:"Interrogation","icon-one":gH,isMobile:C(n)},null,8,["isMobile"])]),_:1},8,["default-active","mode","class","style"]),ne("div",pK,[C(r).generatorType==="Interrogation"?(z(),se("div",mK,[gK,vK,le(dK)])):(z(),be(C(jp),{key:1,"label-position":"left","label-width":"140px",model:C(r),class:"container",rules:a,onSubmit:w[31]||(w[31]=et(()=>{},["prevent"]))},{default:he(()=>[ne("div",yK,[le(RH),le(kh,{label:"Negative Prompt",prop:"negativePrompt",modelValue:C(r).negativePrompt,"onUpdate:modelValue":w[0]||(w[0]=S=>C(r).negativePrompt=S),autosize:{maxRows:15},resize:"vertical",type:"textarea",placeholder:"Enter negative prompt here",info:"What to exclude from the image. Not working? Try increasing the guidance.","label-position":"top"},null,8,["modelValue"]),le(kh,{label:"Seed",prop:"seed",modelValue:C(r).params.seed,"onUpdate:modelValue":w[2]||(w[2]=S=>C(r).params.seed=S),placeholder:"Enter seed here",clearable:"","clear-icon":C(gE)},{append:he(()=>[le(C(zn),{content:"Randomize!",placement:"top"},{default:he(()=>[le(C(at),{icon:C(X3),onClick:w[1]||(w[1]=()=>C(r).params.seed=C(sC)())},null,8,["icon"])]),_:1})]),_:1},8,["modelValue","clear-icon"]),(z(),be(C(b0),{key:0,gutter:10},{default:he(()=>[C(r).multiSelect.sampler.state==="Multiple"?(z(),be(C(to),{key:0,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Sampler(s)",prop:"samplers",modelValue:C(r).multiSelect.sampler.selected,"onUpdate:modelValue":w[3]||(w[3]=S=>C(r).multiSelect.sampler.selected=S),options:C(g),info:"Multi-select enabled. Heun and DPM2 double generation time per step, but converge twice as fast.",multiple:""},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0),C(r).multiSelect.sampler.state==="Enabled"?(z(),be(C(to),{key:1,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Sampler",prop:"sampler",modelValue:C(r).params.sampler_name,"onUpdate:modelValue":w[4]||(w[4]=S=>C(r).params.sampler_name=S),options:C(g),info:"Heun and DPM2 double generation time per step, but converge twice as fast."},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0),C(r).multiSelect.scheduler.state==="Multiple"?(z(),be(C(to),{key:2,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Scheduler(s)",prop:"schedulers",modelValue:C(r).multiSelect.scheduler.selected,"onUpdate:modelValue":w[5]||(w[5]=S=>C(r).multiSelect.scheduler.selected=S),options:C(o),info:"Multi-select enabled. Experimental! KoboldCpp only, allows you to use a different scheduler. Leave as default otherwise.",multiple:""},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0),C(r).multiSelect.scheduler.state==="Enabled"?(z(),be(C(to),{key:3,span:C(n)?24:12},{default:he(()=>[le(po,{label:"Scheduler",prop:"scheduler",modelValue:C(r).params.scheduler,"onUpdate:modelValue":w[6]||(w[6]=S=>C(r).params.scheduler=S),options:C(o),info:"Experimental! KoboldCpp only, allows you to use a different scheduler. Leave as default otherwise."},null,8,["modelValue","options"])]),_:1},8,["span"])):ye("",!0)]),_:1})),le(bn,{label:"Batch Size",prop:"batchSize",modelValue:C(r).params.n,"onUpdate:modelValue":w[7]||(w[7]=S=>C(r).params.n=S),min:C(r).minImages,max:C(r).maxImages},null,8,["modelValue","min","max"]),C(r).multiSelect.steps.state==="Multiple"?(z(),be(bn,{key:1,label:"Steps(s)",prop:"multiSteps",modelValue:C(r).multiSelect.steps.selected,"onUpdate:modelValue":w[8]||(w[8]=S=>C(r).multiSelect.steps.selected=S),min:C(r).minSteps,max:C(r).maxSteps,info:"Multi-select enabled. Keep step count between 30 to 50 for optimal generation times. Coherence typically peaks between 60 and 90 steps, with a trade-off in speed.",multiple:""},null,8,["modelValue","min","max"])):C(r).multiSelect.steps.state==="Enabled"?(z(),be(bn,{key:2,label:"Steps",prop:"steps",modelValue:C(r).params.steps,"onUpdate:modelValue":w[9]||(w[9]=S=>C(r).params.steps=S),min:C(r).minSteps,max:C(r).maxSteps,info:"Keep step count between 30 to 50 for optimal generation times. Coherence typically peaks between 60 and 90 steps, with a trade-off in speed."},null,8,["modelValue","min","max"])):ye("",!0),le(bn,{label:"Width",prop:"width",modelValue:C(r).params.width,"onUpdate:modelValue":w[10]||(w[10]=S=>C(r).params.width=S),min:C(r).minDimensions,max:C(r).maxDimensions,step:64,change:h},null,8,["modelValue","min","max"]),le(bn,{label:"Height",prop:"height",modelValue:C(r).params.height,"onUpdate:modelValue":w[11]||(w[11]=S=>C(r).params.height=S),min:C(r).minDimensions,max:C(r).maxDimensions,step:64,change:h},null,8,["modelValue","min","max"]),C(r).multiSelect.guidance.state==="Multiple"?(z(),be(bn,{key:3,label:"Guidance(s)",prop:"cfgScales",modelValue:C(r).multiSelect.guidance.selected,"onUpdate:modelValue":w[12]||(w[12]=S=>C(r).multiSelect.guidance.selected=S),min:C(r).minCfgScale,max:C(r).maxCfgScale,info:"Multi-select enabled. Higher values will make the AI respect your prompt more. Lower values allow the AI to be more creative.",multiple:""},null,8,["modelValue","min","max"])):C(r).multiSelect.guidance.state==="Enabled"?(z(),be(bn,{key:4,label:"Guidance",prop:"cfgScale",modelValue:C(r).params.cfg_scale,"onUpdate:modelValue":w[13]||(w[13]=S=>C(r).params.cfg_scale=S),min:C(r).minCfgScale,max:C(r).maxCfgScale,step:.5,info:"Higher values will make the AI respect your prompt more. Lower values allow the AI to be more creative."},null,8,["modelValue","min","max","step"])):ye("",!0),C(r).multiSelect.eta.state==="Enabled"?(z(),be(bn,{key:5,label:"Eta",prop:"eta",modelValue:C(r).params.eta,"onUpdate:modelValue":w[14]||(w[14]=S=>C(r).params.eta=S),min:C(r).minEta,max:C(r).maxEta,step:.1,info:"Noise multiplier for ancestral samplers. 0 disables noise injection."},null,8,["modelValue","min","max","step"])):ye("",!0),C(r).multiSelect.clipSkip.state==="Multiple"?(z(),be(bn,{key:6,label:"CLIP Skip(s)",prop:"clipSkips",modelValue:C(r).multiSelect.clipSkip.selected,"onUpdate:modelValue":w[15]||(w[15]=S=>C(r).multiSelect.clipSkip.selected=S),min:C(r).minClipSkip,max:C(r).maxClipSkip,info:"Multi-select enabled. Last layers of CLIP to ignore. For most situations this can be left alone.",multiple:""},null,8,["modelValue","min","max"])):C(r).multiSelect.clipSkip.state==="Enabled"?(z(),be(bn,{key:7,label:"CLIP Skip",prop:"clipSkip",modelValue:C(r).params.clip_skip,"onUpdate:modelValue":w[16]||(w[16]=S=>C(r).params.clip_skip=S),min:C(r).minClipSkip,max:C(r).maxClipSkip,info:"Last layers of CLIP to ignore. For most situations this can be left alone."},null,8,["modelValue","min","max"])):ye("",!0),C(r).sourceGeneratorTypes.includes(C(r).generatorType)?(z(),be(bn,{key:8,label:"Init Strength",prop:"denoise",modelValue:C(r).params.denoising_strength,"onUpdate:modelValue":w[17]||(w[17]=S=>C(r).params.denoising_strength=S),min:C(r).minDenoise,max:C(r).maxDenoise,step:.01,info:"The final image will diverge from the starting image at higher values. 0=unchanged, 1=fullychanged"},null,8,["modelValue","min","max","step"])):ye("",!0),le(bn,{label:"Video Frames",prop:"frames",modelValue:C(r).params.frames,"onUpdate:modelValue":w[18]||(w[18]=S=>C(r).params.frames=S),min:C(r).minFrames,max:C(r).maxFrames,info:"Number of consecutive video frames to generate (Video models only). More frames increases memory usage."},null,8,["modelValue","min","max"]),C(r).params.frames>1?(z(),be(bn,{key:9,label:"FPS",prop:"fps",modelValue:C(r).params.fps,"onUpdate:modelValue":w[19]||(w[19]=S=>C(r).params.fps=S),min:C(r).minFps,max:C(r).maxFps,disabled:C(r).params.frames<=1,info:"Frames per second for video generation."},null,8,["modelValue","min","max","disabled"])):ye("",!0),ne("div",{class:"reference-images",onDragover:w[22]||(w[22]=et(()=>{},["prevent"])),onDrop:w[23]||(w[23]=et(S=>C(r).setExtraImage(S),["prevent"]))},[ne("div",bK,[_K,ne("div",wK,[ne("input",{class:"reference-images-input",type:"file",id:"extra_image_input",onChange:w[20]||(w[20]=S=>C(r).setExtraImage(S)),accept:"image/*",multiple:""},null,32),le(C(at),{onClick:v},{default:he(()=>[He(" Select or Drag Files ")]),_:1}),le(C(at),{onClick:w[21]||(w[21]=S=>C(r).clearExtraImage()),disabled:C(r).referenceImages.length===0},{default:he(()=>[He(" Clear Images ")]),_:1},8,["disabled"])])]),C(r).referenceImages.length>0?(z(),se("div",CK,[(z(!0),se(je,null,Nt(C(r).referenceImages,(S,x)=>(z(),se("div",{class:de(["reference-image-item",{"reference-file-item":!m(S)}]),key:S.id},[ne("span",SK,Ae(x+1),1),m(S)?(z(),be(C(ef),{key:0,class:"reference-image-thumb",src:S.dataUrl,fit:"cover","preview-src-list":y(),"initial-index":b(x),"preview-teleported":""},null,8,["src","preview-src-list","initial-index"])):ye("",!0),ne("span",{class:"reference-image-name",title:S.name},Ae(S.name),9,xK),ne("div",TK,[le(C(zn),{content:"Move up",placement:"top"},{default:he(()=>[le(C(at),{class:"small-btn",icon:C(f_),disabled:x===0,onClick:T=>C(r).moveExtraImage(x,-1)},null,8,["icon","disabled","onClick"])]),_:2},1024),le(C(zn),{content:"Move down",placement:"top"},{default:he(()=>[le(C(at),{class:"small-btn",icon:C(Tl),disabled:x===C(r).referenceImages.length-1,onClick:T=>C(r).moveExtraImage(x,1)},null,8,["icon","disabled","onClick"])]),_:2},1024),le(C(zn),{content:"Remove",placement:"top"},{default:he(()=>[le(C(at),{class:"small-btn",type:"danger",icon:C(Ol),plain:"",onClick:T=>C(r).removeExtraImage(x)},null,8,["icon","onClick"])]),_:2},1024)])],2))),128))])):(z(),se("div",kK," No reference images selected. "))],32),le(C(b0),null,{default:he(()=>[le(C(to),{span:C(n)?24:12},{default:he(()=>[le(qf,{label:"ESRGAN Upscale",prop:"enable_hr",modelValue:C(r).params.enable_hr,"onUpdate:modelValue":w[24]||(w[24]=S=>C(r).params.enable_hr=S),info:"Enable upscale with ESRGAN."},null,8,["modelValue"])]),_:1},8,["span"]),le(C(to),{span:C(n)?24:12},{default:he(()=>[C(r).generatorType==="Img2Img"?(z(),be(qf,{key:0,label:"Send as RefImg",prop:"send_as_refimg",modelValue:C(r).params.send_as_refimg,"onUpdate:modelValue":w[25]||(w[25]=S=>C(r).params.send_as_refimg=S),info:"Instead of regular Img2Img, send the image as a reference image for edit models."},null,8,["modelValue"])):ye("",!0),C(r).generatorType==="Img2Img"&&C(r).params.send_as_refimg&&C(r).params.frames>1?(z(),be(qf,{key:1,label:"Reverse RefImg",prop:"reverse_refimg",modelValue:C(r).params.reverse_refimg,"onUpdate:modelValue":w[26]||(w[26]=S=>C(r).params.reverse_refimg=S),info:"Use the reference image as the final frame instead of the first frame."},null,8,["modelValue"])):ye("",!0)]),_:1},8,["span"])]),_:1})]),ne("div",EK,[le(C(at),{onClick:w[27]||(w[27]=()=>{C(r).cancelled=!0,C(r).generating=!1,C(r).resetStore()}),class:"reset-btn"},{default:he(()=>[He("Reset")]),_:1}),le(C(at),{type:"primary",class:"generate-cancel-btn",style:ze(C(r).generating?"width: 55%;":""),onClick:w[28]||(w[28]=()=>C(r).generateImage(C(r).generatorType))},{default:he(()=>[ne("span",null," Generate "+Ae(C(r).totalImageCount)+" image"+Ae(C(r).totalImageCount===1?"":"s"),1)]),_:1},8,["style"]),C(r).generating?(z(),be(C(at),{key:0,type:"danger",class:"generate-cancel-btn",style:{width:"25%"},disabled:C(r).cancelled,onClick:w[29]||(w[29]=()=>{C(r).cancelled=!0,C(r).generating=!1,C(r).clearQueue()})},{default:he(()=>[He("Cancel all")]),_:1},8,["disabled"])):ye("",!0)]),ne("div",OK,[le(C(g$),{class:"center-both generated-image"},{default:he(()=>[!C(r).generating&&C(r).outputs.length==0?(z(),se("div",AK,[/Inpainting/.test(C(r).generatorType)?(z(),be(K0,{key:0})):ye("",!0),/Img2Img/.test(C(r).generatorType)?(z(),be(K0,{key:1})):ye("",!0)])):ye("",!0),!C(i).showGeneratedImages&&C(r).generating?(z(),be(xH,{key:1,generated:C(r).outputs.length,total:C(r).queue.length,elapsed:c(C(r).timer.seconds),onShowGenerated:w[30]||(w[30]=S=>C(i).showGeneratedImages=!0)},null,8,["generated","total","elapsed"])):ye("",!0),C(i).showGeneratedImages&&C(r).outputs.length!==0?(z(),be(HH,{key:2})):ye("",!0)]),_:1})])]),_:1},8,["model","rules"]))])],64))}});const Bs=uk({history:kT("./"),routes:[{path:"/",name:"generate",component:PK},{path:"/images",name:"images",component:()=>Gf(()=>Promise.resolve().then(()=>RG),void 0,import.meta.url)},{path:"/about",name:"about",component:()=>Gf(()=>Promise.resolve().then(()=>XG),void 0,import.meta.url)},{path:"/options",name:"options",component:()=>Gf(()=>Promise.resolve().then(()=>nq),void 0,import.meta.url)},{path:"/return",name:"return",redirect:e=>(window.location.href=window.location.pathname.endsWith("/")?"..":".","/")}]});function IK(e){const t=[],n=/]+):([^>]+)>/g;return[e.replace(n,(i,l,g)=>{if(g.trim()==="")return"";const o=Number(g);if(isNaN(o))return"";let a=l,u=!1;const d="|high_noise|";return a.startsWith(d)&&(a=a.substring(d.length),u=!0),t.push({name:a,multiplier:o,...u?{is_high_noise:!0}:{}}),""}),t]}function Tu(){return{steps:20,n:1,sampler_name:"Euler",width:512,height:512,cfg_scale:5,eta:1,clip_skip:0,seed:-1,denoising_strength:.6,frames:1,fps:16,enable_hr:!1,send_as_refimg:!0,reverse_refimg:!1,scheduler:"default"}}function sC(){return Math.floor(Math.random()*9999999)+1}const Jt=zo("generator",()=>{const e=["Text2Img","Img2Img","Inpainting"],t=["Img2Img","Inpainting"],n=oe("Text2Img"),r=oe(""),i=kn("promptHistory",[]),l=oe(""),g=kn("negativeLibrary",[]),o=oe(Tu()),a=oe({interval:0,seconds:0}),u=oe({sampler:{name:"Sampler",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.sampler_name],mapToParam:Ce=>Ce.sampler_name},scheduler:{name:"Scheduler",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.scheduler],mapToParam:Ce=>Ce.scheduler},steps:{name:"Steps",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.steps],mapToParam:Ce=>Ce.steps},guidance:{name:"CFG Scale",state:"Enabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.cfg_scale],mapToParam:Ce=>Ce.cfg_scale},clipSkip:{name:"Clip Skip",state:"Disabled",allowedStates:["Disabled","Enabled","Multiple"],selected:[o.value.clip_skip],mapToParam:Ce=>Ce.clip_skip},eta:{name:"Eta",state:"Disabled",allowedStates:["Disabled","Enabled"],selected:[o.value.eta],mapToParam:Ce=>Ce.eta}}),d=()=>({sourceProcessing:void 0,sourceImage:void 0,maskImage:void 0}),c=oe({...d(),sourceProcessing:"inpainting"}),f=oe({...d(),sourceProcessing:"img2img"}),s=Ce=>Ce==="Inpainting"?c.value:Ce==="Img2Img"?f.value:d(),h=ee(()=>s(n.value)),v=oe(""),m=oe(!1),y=oe(!1),b=oe([]),_=oe([]),w=oe(64),S=ee(()=>zt().allowLargerParams==="Enabled"?3072:1024),x=oe(1),T=oe(20),E=oe(1),I=ee(()=>zt().allowLargerParams==="Enabled"?150:50),D=oe(1),N=oe(24),L=oe(0),F=oe(1),O=oe(0),V=oe(1),re=oe(0),J=oe(10),ue=oe(1),M=ee(()=>zt().allowLargerParams==="Enabled"?400:200),U=oe(16),R=ee(()=>zt().allowLargerParams==="Enabled"?32:24),P=(Ce,Ve,Ne)=>Array.from({length:(Ve-Ce+1)/Ne},(Ue,pt)=>(pt+Ce)*Ne),Q=oe(P(re.value,J.value,1)),j=oe(P(D.value,N.value,.5)),A=ee(()=>{const Ce=(yr,Ln,es=1)=>yr*(Ln.state==="Multiple"&&Ln.selected.length>0?Ln.selected.length:es),Ne=o.value.n*B().length,Ue=Ce(Ne,u.value.sampler),pt=Ce(Ue,u.value.scheduler),Ze=Ce(pt,u.value.steps),ln=Ce(Ze,u.value.guidance);return Ce(ln,u.value.clipSkip)});function q(){return o.value=Tu(),c.value=d(),f.value=d(),b.value=[],Ut().showGeneratedImages=!1,G(),!0}function G(){_.value=[]}function te(){b.value=[]}async function fe(){const Ce=zt(),Ve=Ce.baseURL.length===0?".":Ce.baseURL,Ne=await fetch(`${Ve}/sdapi/v1/loras`),Ue=await Ne.json();return Ku(Ne,Ue,200,"Failed to get available LoRAs")?Ue:[]}async function pe(Ce){if(!e.includes(Ce))return[];if(r.value==="")return H("Failed to generate: No prompt submitted.");const Ve=Ls(),Ne=Ut();Ve.saveImages();const{sourceImage:Ue,maskImage:pt,sourceProcessing:Ze}=s(Ce);_e(r.value);const ln=[],yr=B().map(rt=>{const Vt=rt.split(" ### ");return{full_prompt:rt,prompt:Vt[0],negative_prompt:Vt[1]||""}}).map(rt=>{const[Vt,zr]=IK(rt.prompt);return{...rt,prompt:Vt,extractedLoras:zr}}),Ln=yr.some(rt=>rt.extractedLoras.length>0)?await fe():[],es=yr.map(({extractedLoras:rt,...Vt})=>{const zr=rt.length>0&&Ln.length>0?rt.map(rr=>{const Ji=Ln.find(Qi=>Qi.name===rr.name||Qi.path===rr.name);return{path:Ji?Ji.path:rr.name,multiplier:rr.multiplier,...rr.is_high_noise?{is_high_noise:!0}:{}}}):[];return{...Vt,...zr.length>0?{lora:zr}:{}}}),{seed:Zi,cfg_scale:br,eta:km,steps:Fr,clip_skip:pC,sampler_name:mC,scheduler:gC,n:vC,...Xl}=o.value,df=parseInt(Zi.toString()),yC=isNaN(df)||df<0?sC():df,bC=Array.from({length:vC},(rt,Vt)=>yC+Vt),Za=(rt,Vt)=>rt.state==="Disabled"?[]:rt.state==="Enabled"?[Vt]:rt.state==="Multiple"&&rt.selected.length===0?[]:rt.selected,_C={promptVariant:es,seed:bC,cfg_scale:Za(u.value.guidance,br),eta:Za(u.value.eta,km),steps:Za(u.value.steps,Fr),clip_skip:Za(u.value.clipSkip,pC),sampler_name:Za(u.value.sampler,mC),scheduler:Za(u.value.scheduler,gC)},wC=(rt=>Object.entries(rt).filter(([zr,rr])=>rr.length>0).reduce((zr,[rr,Ji])=>{const Qi=[];for(const nn of zr)for(const Kl of Ji)Qi.push({...nn,[rr]:Kl});return Qi},[{}]))(_C),CC=[await ie()];for(const rt of wC){const{promptVariant:{full_prompt:Vt,...zr},...rr}=rt,Ji=Ce==="Img2Img"?Xl.send_as_refimg:!1,Qi=Ji&&Xl.frames>1?Xl.reverse_refimg:!1;let nn={prompt:Vt,params:{...Xl,send_as_refimg:Ji,reverse_refimg:Qi,...rr,...zr,init_images:Ue?[Ue.split(",")[1]]:[],mask:pt,inpainting_mask_invert:pt?0:null,inpainting_fill:pt?1:null},source_image:Ue==null?void 0:Ue.split(",")[1],source_mask:pt,source_processing:Ze,models:CC};nn.params.sampler_name=="default"&&delete nn.params.sampler_name,nn.params.scheduler=="default"&&delete nn.params.scheduler,!nn.params.frames||nn.params.frames<=1?(delete nn.params.frames,delete nn.params.fps):nn.params.fps=Math.min(R.value,Math.max(U.value,Math.round(Number(nn.params.fps)||Tu().fps)));const Kl=Me.value.map(SC=>SC.base64);Kl.length>0&&(nn.params.extra_images=Kl),zt().alsoRequestAvi==="Enabled"&&nn.params.frames&&nn.params.frames>1&&(nn.params.video_output_type=2),ln.push(nn)}let Em=!1;m.value||(Em=!0,b.value=[]),m.value=!0,Ne.showGeneratedImages=!1;let Om=_.value.filter(rt=>!rt.gathered&&!rt.failed).length;for(let rt=0;rt{a.value.seconds++},1e3);!_.value.every(rt=>rt.gathered||rt.failed)&&!y.value;){const rt=_.value.find(Vt=>!Vt.gathered&&!Vt.failed);if(!rt)break;rt.gathered=!0;try{const Vt=await K(rt.params);if(!Vt){rt.failed=!0;continue}ve([{...Vt,...rt}])}catch(Vt){rt.failed=!0,console.error("Error fetching image:",Vt)}}}async function ve(Ce){const Ve=Da();console.log(Ce);const Ne=await Promise.all(Ce.map(async Ze=>{const ln=Ze.images[0],yr=!!Ze.animated?"gif":"png",Ln=Ze.extra_data?`data:video/avi;base64,${Ze.extra_data}`:"",es=Ze.final_frame?`data:image/jpeg;base64,${Ze.final_frame}`:"";let Zi={id:-1,image:`data:image/${yr};base64,${ln}`,prompt:Ze.prompt,modelName:Ze.models[0],frames:Ze.params.frames,fps:Ze.params.fps,extra_avi:Ln,final_frame:es,enable_hr:Ze.params.enable_hr,send_as_refimg:Ze.params.send_as_refimg,reverse_refimg:Ze.params.reverse_refimg};if(Ze.info&&typeof Ze.info=="string"&&Ze.info.trim()!=="")try{const br=JSON.parse(Ze.info);["seed","steps","sampler_name","cfg_scale","eta","width","height","clip_skip","fps"].forEach(Fr=>{br[Fr]!=null&&br[Fr]!=null?Zi[Fr]=br[Fr]:Ze.params[Fr]!=null&&(Zi[Fr]=Ze.params[Fr])}),br.extra_generation_params&&br.extra_generation_params["Schedule type"]?Zi.scheduler=br.extra_generation_params["Schedule type"]:Zi.scheduler=Ze.params.scheduler}catch(br){console.warn("Failed to parse info JSON:",br)}return Zi})),Ue=await Ve.pushOutputs(Ne),pt=0;return b.value=[...Ue.map(Ze=>({type:"image",index:pt,output:Ze})),...b.value].sort((Ze,ln)=>Ze.index-ln.index),b.value.length===_.value.length&&(_.value=[],m.value=!1,Ut().showGeneratedImages=!0,clearInterval(a.value.interval),a.value.interval=0,a.value.seconds=0),Ne}async function H(Ce){const Ve=Ut();return Ce&&Ve.raiseError(Ce,!1),[]}function W(Ce,Ve,Ne,Ue){return Ve<=Ne?Ve:(Ut().raiseWarning(`This image was generated using the 'Larger Values' option. Setting '${Ce}' to its default value instead of ${Ve}.`,!0),Ue)}async function k(Ce,Ve=!0){const Ne=Tu();if(n.value="Text2Img",u.value.guidance.state="Enabled",u.value.sampler.state="Enabled",u.value.steps.state="Enabled",u.value.clipSkip.state="Disabled",u.value.scheduler.state="Enabled",Bs.push("/"),Ve&&(Ce.width=Ce.width||Ne.width,Ce.height=Ce.height||Ne.height),Ce.prompt){const Ue=Ce.prompt.split(" ### ");r.value=Ue[0],l.value=Ue[1]||""}if(Ce.sampler_name){o.value.sampler_name=Ce.sampler_name;const pt=(await we()).find(Ze=>Ze.aliases&&Ze.aliases.includes(Ce.sampler_name));pt&&(o.value.sampler_name=pt.name)}Ce.steps&&(o.value.steps=W("steps",Ce.steps,I.value,Ne.steps)),Ce.cfg_scale&&(o.value.cfg_scale=Ce.cfg_scale),(Ce.eta||Ce.eta===0)&&(o.value.eta=Ce.eta),Ce.width&&(o.value.width=W("width",Ce.width,S.value,Ne.width)),Ce.height&&(o.value.height=W("height",Ce.height,S.value,Ne.height)),Ce.seed&&(o.value.seed=Ce.seed),Ce.clip_skip&&(o.value.clip_skip=W("clip_skip",Ce.clip_skip,J.value,Ne.clip_skip)),Ce.scheduler&&(o.value.scheduler=Ce.scheduler),Ce.frames&&(o.value.frames=W("frames",Ce.frames,M.value,Ne.frames)),Ce.fps&&(o.value.fps=Math.min(R.value,Math.max(U.value,Math.round(W("fps",Ce.fps,R.value,Ne.fps)))))}function X(Ce){const Ve=Ls();n.value="Img2Img",f.value.sourceImage=Ce,Ve.drawing=!1,b.value=[],Bs.push("/"),Kn.fabric.Image.fromURL(Ce,Ve.newImage)}function Y(Ce){const Ve=Ls();b.value=[],c.value.sourceImage=Ce,n.value="Inpainting",Bs.push("/"),Kn.fabric.Image.fromURL(Ce,Ve.newImage)}function $(){return l.value===""?r.value:`${r.value} ### ${l.value}`}function B(){const Ce=$(),Ve=Ce.match(/\{(.*?)\}/g)||[];if(Ve.length===0)return[Ce];let Ne=[];return Ve.forEach(Ue=>{const pt=[],Ze=Ue.replace("{","").replace("}","").split("|");Ne.length===0?Ze.forEach(ln=>{const gn=Ce.replace(Ue,ln);pt.push(gn)}):Ne.forEach(ln=>{Ze.forEach(gn=>{const yr=ln.replace(Ue,gn);pt.push(yr)})}),Ne=[...pt]}),Ne}async function K(Ce){const Ve=zt();try{const Ne=await fetch(`${Ve.baseURL.length===0?".":Ve.baseURL}/sdapi/v1/${Ce.init_images.length>0?"img":"txt"}2img`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify(Ce)}),Ue=await Ne.json();return Ku(Ne,Ue,200,"Failed to fetch",ce)?Ue:!1}catch{return!1}}function ce(Ce){return Ut().raiseError(Ce,!1),y.value=!1,b.value=[],!1}async function ie(){const Ce=zt(),Ve=await fetch(`${Ce.baseURL.length===0?".":Ce.baseURL}/sdapi/v1/sd-models`),Ne=await Ve.json();if(!!Ku(Ve,Ne,200,"Failed to get available models"))return Ne.length===0?"(No model loaded)":Ne[0].model_name}const Z=oe(0),me=new Map;function ae(){me.clear(),Z.value++}xe(()=>zt().baseURL,()=>{ae()});async function ge(Ce){const Ve=zt(),Ue=((Ve.baseURL.length===0?".":Ve.baseURL).replace(/\/+$/,"")||".")+"/"+Ce.replace(/^\/+/,"");if(me.has(Ue))return me.get(Ue);const pt=(async()=>{try{const Ze=await fetch(Ue);if(Ze.ok)return await Ze.json();console.error(`API Error: ${Ze.status} ${Ze.statusText} at ${Ue}`)}catch(Ze){console.error(`Fetch error for ${Ue}:`,Ze)}return me.delete(Ue),null})();return me.set(Ue,pt),pt}async function we(){const Ce=await ge("/sdapi/v1/samplers");return Array.isArray(Ce)?Ce:[]}async function ke(){const Ce=await ge("/sdapi/v1/schedulers");return Array.isArray(Ce)?Ce:[]}function Oe(Ce){g.value.indexOf(Ce)===-1&&(g.value=[...g.value,Ce])}function Fe(Ce){g.value=g.value.filter(Ve=>Ve!=Ce)}function _e(Ce){if(i.value.findIndex(Ve=>Ve.prompt===Ce)===-1){if(i.value.length>=10+i.value.filter(Ve=>Ve.starred).length){const Ve=i.value.filter(Ue=>!Ue.starred),Ne=i.value.findIndex(Ue=>Ue===Ve[Ve.length-1]);i.value.splice(Ne,1)}i.value=[...i.value,{starred:!1,timestamp:Date.now(),prompt:Ce}]}}function Se(Ce){i.value=i.value.filter(Ve=>Ve.prompt!=Ce&&Ve!=Ce)}function De(){return!1}const Me=oe([]);async function bt(Ce){var Ze,ln;const Ve=Ce.target,Ne=(ln=Ve.files)!=null?ln:(Ze=Ce.dataTransfer)==null?void 0:Ze.files,Ue=Array.from(Ne!=null?Ne:[]);if(Ue.length===0)return;const pt=await Promise.all(Ue.map(async(gn,yr)=>{const Ln=await Tc(gn);return{id:`${Date.now()}-${yr}-${gn.name}`,name:gn.name,size:gn.size,dataUrl:Ln,base64:Ln.includes("data:image")?Ln.split(",")[1]:Ln}}));Me.value||(Me.value=[]);for(let gn=0;gn=Me.value.length)return;const Ue=[...Me.value],[pt]=Ue.splice(Ce,1);Ue.splice(Ne,0,pt),Me.value=Ue}function tn(){Me.value=[];const Ce=document.getElementById("extra_image_input");Ce&&(Ce.value="")}return{generatorType:n,prompt:r,params:o,outputs:b,inpainting:c,img2img:f,uploadDimensions:v,cancelled:y,multiSelect:u,referenceImages:Me,negativePrompt:l,generating:m,negativePromptLibrary:g,minDimensions:w,maxDimensions:S,minImages:x,maxImages:T,minSteps:E,maxSteps:I,minCfgScale:D,maxCfgScale:N,minDenoise:O,maxDenoise:V,minClipSkip:re,maxClipSkip:J,minFrames:ue,maxFrames:M,minFps:U,maxFps:R,minEta:L,maxEta:F,clipSkipList:Q,cfgList:j,queue:_,promptHistory:i,timer:a,validGeneratorTypes:e,sourceGeneratorTypes:t,currentImageProps:h,totalImageCount:A,generateImage:pe,generateText2Img:k,generateImg2Img:X,generateInpainting:Y,getPrompt:De,resetStore:q,clearQueue:G,clearOutputs:te,pushToNegativeLibrary:Oe,removeFromNegativeLibrary:Fe,pushToPromptHistory:_e,removeFromPromptHistory:Se,setExtraImage:bt,removeExtraImage:_t,moveExtraImage:ht,clearExtraImage:tn,getAvailableSamplers:we,getAvailableSchedulers:ke,cacheVersion:Z,invalidateApiCaches:ae,getCachedEndpoint:ge}});"stream"in Blob.prototype||Object.defineProperty(Blob.prototype,"stream",{value(){return new Response(this).body}});var ff=e=>new DataView(new ArrayBuffer(e)),qa=e=>new Uint8Array(e.buffer||e),mo=e=>new TextEncoder().encode(String(e));function MK(e,t){if(t===void 0||t instanceof Date||(t=new Date(t)),e instanceof File)return{t:t||new Date(e.lastModified),o:e.stream()};if(e instanceof Response)return{t:t||new Date(e.headers.get("Last-Modified")||Date.now()),o:e.body};if(t===void 0)t=new Date;else if(isNaN(t))throw new Error("Invalid modification date.");if(typeof e=="string")return{t,o:mo(e)};if(e instanceof Blob)return{t,o:e.stream()};if(e instanceof Uint8Array||e instanceof ReadableStream)return{t,o:e};if(e instanceof ArrayBuffer||ArrayBuffer.isView(e))return{t,o:qa(e)};if(Symbol.asyncIterator in e)return{t,o:lC(e)};throw new TypeError("Unsupported input format.")}function lC(e){const t="next"in e?e:e[Symbol.asyncIterator]();return new ReadableStream({async pull(n){let r=0;for(;n.desiredSize>r;){const i=await t.next();if(!i.value){n.close();break}{const l=LK(i.value);n.enqueue(l),r+=l.byteLength}}}})}function LK(e){return typeof e=="string"?mo(e):e instanceof Uint8Array?e:qa(e)}function RK(e,t,n){if(t===void 0||t instanceof Uint8Array||(t=mo(t)),e instanceof File)return{i:t||mo(e.name),A:e.size};if(e instanceof Response){const r=e.headers.get("content-disposition"),i=r&&r.match(/;\s*filename\*?=["']?(.*?)["']?$/i),l=i&&i[1]||new URL(e.url).pathname.split("/").pop(),g=l&&decodeURIComponent(l),o=n||+e.headers.get("content-length");return{i:t||mo(g),A:o}}if(!t||t.length===0)throw new Error("The file must have a name.");return typeof e=="string"?{i:t,A:mo(e).length}:e instanceof Blob?{i:t,A:e.size}:e instanceof ArrayBuffer||ArrayBuffer.isView(e)?{i:t,A:e.byteLength}:{i:t,A:n}}var DK=new WebAssembly.Instance(new WebAssembly.Module(Uint8Array.from(atob("AGFzbQEAAAABCgJgAABgAn9/AXwDAwIAAQUDAQACBwkCAW0CAAFjAAEIAQAKlQECSQEDfwNAIAEhAEEAIQIDQCAAQQF2IABBAXFBoIbi7X5scyEAIAJBAWoiAkEIRw0ACyABQQJ0IAA2AgAgAUEBaiIBQYACRw0ACwtJAQF/IAFBf3MhAUGAgAQhAkGAgAQgAGohAANAIAFB/wFxIAItAABzQQJ0KAIAIAFBCHZzIQEgAkEBaiICIABJDQALIAFBf3O4Cw"),e=>e.charCodeAt(0)))),{c:$K,m:BK}=DK.exports,FK=qa(BK).subarray(65536);function $y(e,t=0){for(const n of function*(r){for(;r.length>65536;)yield r.subarray(0,65536),r=r.subarray(65536);r.length&&(yield r)}(e))FK.set(n),t=$K(n.length,t);return t}function uC(e,t,n=0){const r=e.getSeconds()>>1|e.getMinutes()<<5|e.getHours()<<11,i=e.getDate()|e.getMonth()+1<<5|e.getFullYear()-1980<<9;t.setUint16(n,r,1),t.setUint16(n+2,i,1)}function zK(e){const t=ff(30);return t.setUint32(0,1347093252),t.setUint32(4,335546368),uC(e.t,t,10),t.setUint16(26,e.i.length,1),qa(t)}async function*NK(e){let{o:t}=e;if("then"in t&&(t=await t),t instanceof Uint8Array)yield t,e.u=$y(t,0),e.A=t.length;else{e.A=0;const n=t.getReader();for(;;){const{value:r,done:i}=await n.read();if(i)break;e.u=$y(r,e.u),e.A+=r.length,yield r}}}function jK(e){const t=ff(16);return t.setUint32(0,1347094280),t.setUint32(4,e.u,1),t.setUint32(8,e.A,1),t.setUint32(12,e.A,1),qa(t)}function VK(e,t){const n=ff(46);return n.setUint32(0,1347092738),n.setUint32(4,352523264),n.setUint16(8,2048),uC(e.t,n,12),n.setUint32(16,e.u,1),n.setUint32(20,e.A,1),n.setUint32(24,e.A,1),n.setUint16(28,e.i.length,1),n.setUint16(40,33204,1),n.setUint32(42,t,1),qa(n)}function UK(e){return e instanceof File||e instanceof Response?[[e],[e]]:[[e.input,e.name,e.size],[e.input,e.lastModified]]}function HK(e,t={}){const n={"Content-Type":"application/zip","Content-Disposition":"attachment"};return Number.isInteger(t.length)&&t.length>0&&(n["Content-Length"]=t.length),t.metadata&&(n["Content-Length"]=p(t.metadata)),new Response(lC(async function*(r){const i=[];let l=0,g=0;for await(const u of r)yield zK(u),yield u.i,yield*NK(u),yield jK(u),i.push(VK(u,l)),i.push(u.i),g++,l+=46+u.i.length+u.A;let o=0;for(const u of i)yield u,o+=u.length;const a=ff(22);a.setUint32(0,1347093766),a.setUint16(8,g,1),a.setUint16(10,g,1),a.setUint32(12,o,1),a.setUint32(16,l,1),yield qa(a)}(async function*(r){for await(const i of r){const[l,g]=UK(i);yield Object.assign(MK(...g),RK(...l))}}(e))),{headers:n})}async function cC(e,t=!0,n){const r=zt();t&&Wi({message:`Downloading ${e.length} image(s)...`,type:"info"});const i=[];for(let o=0;o]/g,"").substring(0,128).trimEnd();let s=r.imageDownloadType;u.startsWith("data:image/gif")&&(s="GIF"),s==="PNG"?i.push({name:f+".png",input:await ba(u,"image/png")}):s==="JPG"?i.push({name:f+".jpg",input:await ba(u,"image/jpeg")}):s==="GIF"?i.push({name:f+".gif",input:await ba(u,"image/gif")}):i.push({name:f+".webp",input:await ba(u,"image/webp")}),i.push({name:f+".json",input:JSON.stringify(c,void 0,4)}),n&&n()}const l=await HK(i).blob(),g=document.createElement("a");g.href=URL.createObjectURL(l),g.download="sdui_images.zip",g.click()}async function fC(e,t){const n=zt(),r=document.createElement("a");let i,l=n.imageDownloadType;e.startsWith("data:image/gif")&&(l="GIF"),l==="PNG"?(i=await ba(e,"image/png"),r.href=URL.createObjectURL(i),r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".png"):l==="JPG"?(i=await ba(e,"image/jpeg"),r.href=URL.createObjectURL(i),r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".jpg"):l==="GIF"?(i=await ba(e,"image/gif"),r.href=URL.createObjectURL(i),r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".gif"):(r.href=e,r.download=t.replace(/[/\\:*?"<>]/g,"").substring(0,128).trimEnd()+".webp"),r.click(),i&&URL.revokeObjectURL(r.href)}function dC(e,t){const n=e;if(!n)return;const r=atob(n),i=r.length,l=new Uint8Array(i);for(let u=0;u{Ew.confirm("This action will permanently delete this image. Continue?","Warning",{confirmButtonText:"OK",cancelButtonText:"Cancel",type:"warning"}).then(()=>{r.deleteOutput(t.imageData.id),t.onDelete!==void 0&&t.onDelete(t.imageData.id),Wi({type:"success",message:"Deleted Image"})})},l=a=>{if(a.extra_avi){const u=a.extra_avi.split(",")[1];if(!u)return;const d=`${a.seed}-${a.prompt}.avi`;dC(u,d);return}else fC(a.image,`${a.seed}-${a.prompt}`)},g=()=>{Jt().clearOutputs(),Ut().showGeneratedImages=!1,Jt().clearQueue()};async function o(a){const u=window.location.origin,d={prompt:a.prompt,width:a.width?a.width:void 0,height:a.height?a.height:void 0,steps:a.steps,cfg_scale:a.cfg_scale,sampler_name:a.sampler_name,model_name:a.modelName,seed:a.seed,clip_skip:a.clip_skip,frames:a.frames,fps:a.fps,scheduler:a.scheduler,extra_avi:a.extra_avi,final_frame:a.final_frame,enable_hr:a.enable_hr,send_as_refimg:a.send_as_refimg,reverse_refimg:a.reverse_refimg},c=window.location.pathname.replace("images","");let f=`${u}${c}?share=`,s="",h="";for(const[m,y]of Object.entries(d)){if(!y)continue;let b=y;typeof y=="string"?b=encodeURIComponent(y):Array.isArray(y)&&(b=JSON.stringify(y)),s+=`${h}${m}=${b}`,h="&"}f+=btoa(String.fromCharCode.apply(null,Array.from(qX(s)))),await navigator.clipboard.writeText(f),Wi({type:"success",message:"Copied shareable link to clipboard"})}return(a,u)=>(z(),se(je,null,[le(C(at),{class:"compact-button",onClick:i,type:"danger",size:"small",icon:C(Ol),plain:""},{default:he(()=>[He("Delete")]),_:1},8,["icon"]),le(C(at),{class:"compact-button",onClick:u[0]||(u[0]=d=>l(e.imageData)),type:"success",size:"small",icon:C(Ks),plain:""},{default:he(()=>[He("Download")]),_:1},8,["icon"]),e.imageData.starred?ye("",!0):(z(),be(C(at),{key:0,class:"compact-button",onClick:u[1]||(u[1]=d=>C(r).toggleStarred(e.imageData.id)),type:"warning",size:"small",icon:C(w4),plain:""},{default:he(()=>[He("Star")]),_:1},8,["icon"])),e.imageData.starred?(z(),be(C(at),{key:1,class:"compact-button",onClick:u[2]||(u[2]=d=>C(r).toggleStarred(e.imageData.id)),type:"warning",size:"small",icon:C(__),plain:""},{default:he(()=>[He("Unstar")]),_:1},8,["icon"])):ye("",!0),le(C(at),{class:"compact-button",onClick:u[3]||(u[3]=d=>C(n).generateText2Img(e.imageData)),type:"success",size:"small",plain:""},{default:he(()=>[He("Txt2img")]),_:1}),le(C(at),{class:"compact-button",onClick:u[4]||(u[4]=d=>C(n).generateImg2Img(e.imageData.image)),type:"success",size:"small",plain:""},{default:he(()=>[He("Img2img")]),_:1}),le(C(at),{class:"compact-button",onClick:u[5]||(u[5]=d=>C(n).generateInpainting(e.imageData.image)),type:"success",size:"small",plain:""},{default:he(()=>[He("Inpaint")]),_:1}),e.showDismiss?(z(),be(C(at),{key:2,class:"compact-button",onClick:u[6]||(u[6]=d=>g()),type:"success",size:"small",plain:""},{default:he(()=>[He("Dismiss")]),_:1})):ye("",!0),le(C(at),{class:"compact-button",onClick:u[7]||(u[7]=d=>o(e.imageData)),type:"success",icon:C($3),size:"small",plain:""},{default:he(()=>[He("Share")]),_:1},8,["icon"])],64))}});const hC=qt(WK,[["__scopeId","data-v-4d472d34"]]),YK={class:"main-output",style:{position:"relative",display:"flex","align-items":"center","justify-content":"center"}},XK=["src"],KK={style:{"font-size":"16px","font-weight":"500"}},GK={style:{"font-family":"'Segoe UI', Tahoma, Geneva, Verdana, sans-serif","letter-spacing":"0.025em"}},qK={key:0},ZK=ne("br",null,null,-1),JK={key:0},QK=["onClick"],eG={key:1},tG=["onClick"],nG={key:2},rG=["onClick"],iG=Ee({__name:"ImageDialog",setup(e){const t=Da(),n=Ut(),r=oe();fM(r,{onSwipeEnd(d,c){c==="RIGHT"&&n.openModalToLeft(),c==="LEFT"&&n.openModalToRight()}});const i=ee({get(){return n.activeModal!==-1},set(){n.activeModal=-1}}),l=oe(t.currentOutputs[0]);xe(()=>n.activeModal,async()=>{const d=t.currentOutputs.find(c=>c.id===n.activeModal);if(d)return l.value=d;l.value=await jt.outputs.get(n.activeModal)||t.currentOutputs[0]});function g(){i.value=!1}function o(){var f;if(!((f=l.value)!=null&&f.final_frame))return;const d=l.value.final_frame;Jt().generateImg2Img(d)}function a(){var d;!((d=l.value)!=null&&d.image)||fC(l.value.image,`${l.value.seed}-${l.value.prompt}`)}function u(){var f,s;if(!((f=l.value)!=null&&f.extra_avi))return;const d=l.value.extra_avi.split(",")[1];if(!d)return;const c=`output-${(s=l.value.id)!=null?s:"video"}.avi`;dC(d,c)}return(d,c)=>{var f;return z(),be(C(WB),{"model-value":C(i),width:(f=l.value)==null?void 0:f.width,class:"image-viewer",onClosed:g,"align-center":""},{footer:he(()=>[le(hC,{"image-data":l.value,"on-delete":g},null,8,["image-data"])]),default:he(()=>{var s,h,v,m;return[ne("div",{class:"main-output-container",ref_key:"target",ref:r},[ne("div",YK,[(s=l.value)!=null&&s.image?(z(),se("img",{key:0,src:l.value.image,alt:"Output image",style:{"max-width":"100%","max-height":"100%","object-fit":"contain"}},null,8,XK)):ye("",!0)])],512),ne("div",KK,Ae(((h=l.value.prompt)==null?void 0:h.split("###")[0])||"Unknown Creation"),1),ne("div",GK,[ne("div",null,"Negative Prompt: "+Ae(((v=l.value.prompt)==null?void 0:v.split("###")[1])||"None"),1),ne("span",null,"Model: "+Ae(l.value.modelName||"Unknown")+" - ",1),ne("span",null,"Sampler: "+Ae(l.value.sampler_name||"Unknown")+" - ",1),ne("span",null,"Scheduler: "+Ae(l.value.scheduler||"Unknown")+" - ",1),ne("span",null,"Seed: "+Ae(l.value.seed||"Unknown")+" - ",1),ne("span",null,"Steps: "+Ae(l.value.steps||"Unknown")+" - ",1),ne("span",null,"CFG Scale: "+Ae(l.value.cfg_scale||"Unknown")+" - ",1),ne("span",null,"Clip Skip: "+Ae((m=l.value.clip_skip)!=null?m:"Unknown")+" - ",1),ne("span",null,"Dimensions: "+Ae(l.value.width||"???")+"x"+Ae(l.value.height||"???")+" - ",1),ne("span",null,"Frames: "+Ae(l.value.frames||"1"),1),l.value.frames&&l.value.frames>1?(z(),se("span",qK," - FPS: "+Ae(l.value.fps||"Unknown"),1)):ye("",!0),ZK,ne("b",null,[l.value.frames&&l.value.frames>1?(z(),se("span",JK,[ne("a",{href:"#",onClick:et(a,["prevent"]),style:{cursor:"pointer",color:"var(--el-color-primary)"}},"[Download GIF]",8,QK)])):ye("",!0),l.value.extra_avi?(z(),se("span",eG,[He(" - "),ne("a",{href:"#",onClick:et(u,["prevent"]),style:{cursor:"pointer",color:"var(--el-color-primary)"}},"[Download AVI]",8,tG)])):ye("",!0),l.value.final_frame?(z(),se("span",nG,[He(" - "),ne("a",{href:"#",onClick:et(o,["prevent"]),style:{cursor:"pointer",color:"var(--el-color-primary)"}},"[Extend Video]",8,rG)])):ye("",!0)])])]}),_:1},8,["model-value","width"])}}});const aG=e=>(ui("data-v-8f4d2380"),e=e(),ci(),e),oG=aG(()=>ne("div",{style:{"font-size":"20px"}},"Stable UI",-1)),sG={class:"generator-icons"},lG=Ee({__name:"App",setup(e){const n=Ep(kp).smallerOrEqual("md"),r=Ut();zt();const i=fk(),l=oe();return xe(()=>i.path,g=>{l.value&&l.value.open(g)}),(g,o)=>(z(),se(je,null,[ne("div",{class:de({"menu-container":!C(n)})},[le(C(mw),{"default-active":C(i).path,mode:"horizontal",router:!0,ellipsis:!C(n),class:de(C(n)?"mobile-menu":"menu"),ref_key:"menuRef",ref:l},{default:he(()=>[C(n)?ye("",!0):(z(),be(C(Up),{key:0,class:"remove-item-styling center-vertical"},{title:he(()=>[oG]),_:1})),le(ls,{isMobile:C(n),index:"/"},{icon:he(()=>[ne("div",sG,[le(C(Le),null,{default:he(()=>[le(C(wO))]),_:1}),C(r).showGeneratorBadge?(z(),be(C(Le),{key:0,class:"generator-badge",size:10},{default:he(()=>[le(tH)]),_:1})):ye("",!0)])]),title:he(()=>[He("Generate")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/images"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(Q3))]),_:1})]),title:he(()=>[He("Images")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/about"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(g_))]),_:1})]),title:he(()=>[He("About")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/options"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(M4))]),_:1})]),title:he(()=>[He("Options")]),_:1},8,["isMobile"]),le(ls,{isMobile:C(n),index:"/return"},{icon:he(()=>[le(C(Le),null,{default:he(()=>[le(C(d_))]),_:1})]),title:he(()=>[He("Return to Lite")]),_:1},8,["isMobile"])]),_:1},8,["default-active","ellipsis","class"])],2),ne("div",{class:de({view:!C(n)})},[le(C(c_))],2),le(iG)],64))}});const uG=qt(lG,[["__scopeId","data-v-8f4d2380"]]);const Tm=tT(uG);Tm.use(iT());Tm.use(Bs);Tm.mount("#app");Bs.replace("/");window.addEventListener("beforeunload",e=>{Jt().generating&&(e.preventDefault(),e.returnValue="")});const cG={key:1,class:"image-action"},fG=Ee({__name:"CustomImage",props:{imageData:null},setup(e){const t=e,n=Ut(),r=oe(null);VI(r,n.toggleMultiSelect,{modifiers:{prevent:!0}});const i=oe(!1);cM(r,([{isIntersecting:g}])=>{g&&(i.value=g)},{rootMargin:"500px"});const l=ee(()=>n.selected.includes(t.imageData.id));return(g,o)=>(z(),se("div",{class:"relative",ref_key:"containerRef",ref:r},[i.value?(z(),be(C(ef),{key:0,class:"thumbnail",src:e.imageData.image,onClick:o[0]||(o[0]=a=>C(n).activeModal=e.imageData.id),fit:"cover",loading:"lazy",style:ze(`${C(l)&&"opacity: 0.5"}`)},null,8,["src","style"])):ye("",!0),i.value?(z(),se("div",cG,[e.imageData.starred?(z(),be(C(Le),{key:0,class:"starred-icon",size:35,color:"var(--el-color-warning)"},{default:he(()=>[le(C(__))]),_:1})):ye("",!0),C(n).multiSelect?(z(),se("div",{key:1,class:"select-container",onClick:o[1]||(o[1]=a=>C(n).toggleSelection(e.imageData.id))},[le(C(Le),{class:"select-icon",size:35,color:`rgba(255, 255, 255, ${C(l)?"1":"0.5"})`},{default:he(()=>[C(l)?ye("",!0):(z(),be(C(El),{key:0})),C(l)?(z(),be(C(p_),{key:1})):ye("",!0)]),_:1},8,["color"])])):ye("",!0)])):ye("",!0)],512))}});const By=qt(fG,[["__scopeId","data-v-b9569bbd"]]);function dG({mobileWidth:e=768,hideAfterDistanceFromTop:t=100,hideAfterScroll:n=100}={}){const{width:r}=a1(),i=ee(()=>r.value<=e),l=oe(!0),{y:g}=uM(window);let o=g.value,a=0;return xe(g,u=>{if(!i.value)return;const d=u-o;if(o=u,d>0&&u>t){a+=d,a>=n&&(l.value=!1,a=0);return}d<0&&(a=0,l.value=!0)}),{isVisible:l,isMobile:i}}const hG={},pG={xmlns:"http://www.w3.org/2000/svg","xmlns:xlink":"http://www.w3.org/1999/xlink",viewBox:"0 0 1024 1024"},mG=ne("path",{d:"M880 112H144c-17.7 0-32 14.3-32 32v736c0 17.7 14.3 32 32 32h736c17.7 0 32-14.3 32-32V144c0-17.7-14.3-32-32-32zm-696 72h136v656H184V184zm656 656H384V384h456v456zM384 320V184h456v136H384z",fill:"currentColor"},null,-1),gG=[mG];function vG(e,t){return z(),se("svg",pG,gG)}const yG=qt(hG,[["render",vG]]),bG=e=>(ui("data-v-b39b47c5"),e=e(),ci(),e),_G={class:"options"},wG=["onClick"],CG=["onClick"],SG=["onClick"],xG={key:1,class:"center-both",style:{gap:"12px"}},TG={key:2},kG=bG(()=>ne("em",{style:{"font-size":"14px"}},"(long press to select multiple images)",-1)),EG=[kG],OG={key:0},AG={key:0,style:{display:"flex",gap:"8px"}},PG={key:1,class:"images"},IG={key:1},MG=Ee({__name:"ImagesView",setup(e){const{width:t}=a1(),{isVisible:n,isMobile:r}=dG(),i=Da(),l=zt(),g=Ut();function o(){g.selected=g.selected.filter(h=>!i.currentOutputs.map(v=>v.id).includes(h)),g.selected=[...g.selected,...i.currentOutputs.map(h=>h.id)],g.multiSelect=!0}async function a(){const h=await jt.outputs.toCollection().primaryKeys();g.selected=h,g.multiSelect=!0}function u(){g.selected=g.selected.filter(h=>!i.currentOutputs.map(v=>v.id).includes(h)),g.selected.length===0&&(g.multiSelect=!1)}function d(){g.selected=[],g.multiSelect=!1}const c=()=>{Ew.confirm(`This action will permanently delete ${g.selected.length} images. Continue?`,"Warning",{confirmButtonText:"OK",cancelButtonText:"Cancel",type:"warning"}).then(()=>{i.deleteMultipleOutputs(g.selected)})};vv(["a","A","ArrowLeft"],g.openModalToLeft),vv(["d","D","ArrowRight"],g.openModalToRight);async function f(){cC(g.selected)}const s=ee(()=>{let h=2;t.value>1440?h=6:t.value>1280?h=5:t.value>768?h=4:t.value>480&&(h=3);const v=[];for(let m=0;m(z(),se(je,null,[ne("div",{class:de(["images-top-bar",{"mobile-hidden":C(r)&&!C(n)}])},[ne("div",_G,[le(C(cu),{placement:"bottom",title:"Sort By",trigger:"click",width:200,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[le(C(c4))]),_:1})]),_:1})]),default:he(()=>[(z(),se(je,null,Nt(["Newest","Oldest"],m=>ne("div",{key:m,onClick:()=>C(i).sortBy=m,class:de(`el-select-dropdown__item ${C(i).sortBy===m?"selected":""}`)},Ae(m),11,wG)),64))]),_:1}),le(C(cu),{placement:"bottom",title:"Filter By",trigger:"click",width:240,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[le(C(d3))]),_:1})]),_:1})]),default:he(()=>[(z(),se(je,null,Nt(["all","favourited","unfavourited","unrated"],m=>ne("div",{key:m,onClick:()=>C(i).filterBy=m,class:de(`el-select-dropdown__item ${C(i).filterBy===m?"selected":""}`)},Ae(C(i).filterBy===m?"Showing":"Show")+" "+Ae(m),11,CG)),64))]),_:1}),le(C(cu),{placement:"bottom",title:"Image Layout",trigger:"click",width:240,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[le(yG)]),_:1})]),_:1})]),default:he(()=>[(z(),se(je,null,Nt([{label:"Square Grid",value:"grid"},{label:"Dynamic Layout",value:"dynamic"}],m=>ne("div",{key:m.value,onClick:()=>C(i).currentLayout=m.value,class:de(`el-select-dropdown__item ${C(i).currentLayout===m.value?"selected":""}`)},Ae(m.label),11,SG)),64))]),_:1}),le(C(cu),{placement:"bottom",title:"Selection",trigger:"click",width:240,transition:"none","hide-after":0},{reference:he(()=>[le(C(at),{class:"btn-select"},{default:he(()=>[le(C(Le),{size:16},{default:he(()=>[C(g).multiSelect?(z(),be(C(p_),{key:0})):(z(),be(C(El),{key:1}))]),_:1})]),_:1})]),default:he(()=>[C(g).multiSelect?(z(),se("div",{key:0,class:"el-select-dropdown__item selected",onClick:v[0]||(v[0]=(...m)=>C(g).toggleMultiSelect&&C(g).toggleMultiSelect(...m))},"Disable multi-select")):(z(),se("div",{key:1,class:"el-select-dropdown__item",onClick:v[1]||(v[1]=(...m)=>C(g).toggleMultiSelect&&C(g).toggleMultiSelect(...m))},"Enable multi-select")),C(g).selected.length>0?(z(),se("div",{key:2,class:"el-select-dropdown__item selected",onClick:d},"Deselect All")):(z(),se("div",{key:3,class:"el-select-dropdown__item",onClick:a},"Select All")),C(g).selected.every(m=>!C(i).currentOutputs.map(y=>y.id).includes(m))?(z(),se("div",{key:5,class:"el-select-dropdown__item",onClick:o},"Select Page")):(z(),se("div",{key:4,class:"el-select-dropdown__item selected",onClick:u},"Deselect Page"))]),_:1})]),C(l).pageless==="Disabled"?(z(),be(C(m9),{key:0,layout:"prev, pager, next",total:C(i).outputsLength,"page-size":C(l).pageSize,"current-page":C(i).currentPage,"onUpdate:currentPage":v[2]||(v[2]=m=>C(i).currentPage=m),"hide-on-single-page":""},null,8,["total","page-size","current-page"])):ye("",!0),C(g).multiSelect?(z(),se("div",xG,[ne("div",null,Ae(C(g).selected.length)+" selected",1),le(C(at),{type:"danger",onClick:c,icon:C(Ol),plain:""},{default:he(()=>[He("Delete")]),_:1},8,["icon"]),le(C(at),{type:"success",onClick:f,icon:C(Ks),plain:"",style:{margin:"0"}},{default:he(()=>[He("Download")]),_:1},8,["icon"])])):(z(),se("div",TG,EG))],2),C(i).outputsLength!=0?(z(),se("div",OG,[C(i).currentLayout==="dynamic"?(z(),se("div",AG,[(z(!0),se(je,null,Nt(C(s),(m,y)=>(z(),se("div",{key:y,style:{flex:"1 1 0%"}},[(z(!0),se(je,null,Nt(m,b=>(z(),be(By,{key:b.id,"image-data":b,style:{"margin-bottom":"8px"}},null,8,["image-data"]))),128))]))),128))])):ye("",!0),C(i).currentLayout==="grid"?(z(),se("div",PG,[(z(!0),se(je,null,Nt(C(i).currentOutputs,m=>(z(),be(By,{key:m.id,"image-data":m,style:{width:"200px",height:"200px"}},null,8,["image-data"]))),128))])):ye("",!0)])):ye("",!0),C(i).outputsLength==0?(z(),se("div",IG,[le(C(FF),{description:"No Images Found"})])):ye("",!0)],64))}});const LG=qt(MG,[["__scopeId","data-v-b39b47c5"]]),RG=Object.freeze(Object.defineProperty({__proto__:null,default:LG},Symbol.toStringTag,{value:"Module"})),DG=["href"],$G=Ee({__name:"BaseLink",props:{href:null,router:{type:Boolean}},setup(e){return(t,n)=>{const r=gt("router-link");return z(),se(je,null,[e.router?ye("",!0):(z(),se("a",{key:0,target:"_blank",rel:"noreferrer noopener",href:e.href},[Te(t.$slots,"default",{},void 0,!0)],8,DG)),e.router?(z(),be(r,{key:1,to:e.href},{default:he(()=>[Te(t.$slots,"default",{},void 0,!0)]),_:3},8,["to"])):ye("",!0)],64)}}});const BG=qt($G,[["__scopeId","data-v-17b53b7d"]]),Yl=e=>(ui("data-v-ecc278c5"),e=e(),ci(),e),FG={class:"about"},zG={class:"about-content"},NG=Yl(()=>ne("h1",{style:{"margin-top":"0"}},"Stable UI",-1)),jG=Yl(()=>ne("div",null,[He("This tool was originally a front-end for the AI Horde and has since been converted for local generations with the A1111 API, such as in "),ne("a",{href:"https://github.com/LostRuins/koboldcpp"},"KoboldCpp"),He(".")],-1)),VG=Yl(()=>ne("br",null,null,-1)),UG=Yl(()=>ne("div",null,"If you want to help improve this tool, you can find the currently maintained source code from this modified version on https://github.com/LostRuins/stable-ui and https://github.com/henk717/stable-ui, which is based off https://github.com/ayunami2000/stable-ui, which derives from the original AI Horde version on https://github.com/aqualxx/stable-ui (Original author aqualxx#5004). Feel free to contribute!",-1)),HG=Yl(()=>ne("br",null,null,-1)),WG=Ee({__name:"AboutView",setup(e){return(t,n)=>(z(),se("div",FG,[ne("div",zG,[NG,jG,VG,UG,HG,ne("div",null,[He("You can find the KoboldAI community and authors of this fork on the "),le(BG,{href:"https://koboldai.org/discord"},{default:he(()=>[He("KoboldAI Discord")]),_:1})])])]))}});const YG=qt(WG,[["__scopeId","data-v-ecc278c5"]]),XG=Object.freeze(Object.defineProperty({__proto__:null,default:YG},Symbol.toStringTag,{value:"Module"}));const so=Ee({__name:"FormRadio",props:{label:null,modelValue:null,prop:null,useBoolean:{type:Boolean},options:null,disabled:{type:Boolean},info:null,labelStyle:null,change:null},emits:["update:modelValue"],setup(e,{emit:t}){const n=e;function r(l){if(n.useBoolean&&l==="Enabled"?t("update:modelValue",!0):n.useBoolean&&l==="Disabled"?t("update:modelValue",!1):t("update:modelValue",l),!!n.change)return n.useBoolean&&l==="Enabled"?n.change(!0):n.useBoolean&&l==="Disabled"?n.change(!1):n.change(l)}const i=ee(()=>n.useBoolean?n.modelValue===!0?"Enabled":n.modelValue===!1?"Disabled":n.modelValue:n.modelValue);return(l,g)=>(z(),be(C(Di),{prop:e.prop},{label:he(()=>[le(jl,{info:e.info,"label-style":e.labelStyle},{default:he(()=>[Te(l.$slots,"label",{},()=>[He(Ae(e.label),1)])]),_:3},8,["info","label-style"])]),default:he(()=>[le(C(H$),{disabled:e.disabled,"model-value":C(i),onChange:r},{default:he(()=>[(z(!0),se(je,null,Nt(e.options,o=>(z(),be(C(W$),{key:o,label:o},null,8,["label"]))),128))]),_:1},8,["disabled","model-value"]),Te(l.$slots,"inline")]),_:3},8,["prop"]))}}),Qo=e=>(ui("data-v-0345ee1f"),e=e(),ci(),e),KG=Qo(()=>ne("h1",null,"Options",-1)),GG=Qo(()=>ne("h2",null,"Generation Options",-1)),qG=Qo(()=>ne("h3",null,"Parameter Controls",-1)),ZG=Qo(()=>ne("h2",null,"Image Options",-1)),JG=Qo(()=>ne("div",null,[He("Drop file here OR "),ne("em",null,"click to upload")],-1)),QG=Qo(()=>ne("h2",null,"General Options",-1)),eq=Ee({__name:"OptionsView",setup(e){const t=zt(),n=Da(),r=Jt(),i=[{value:"dark",label:"Dark"},{value:"light",label:"Light"},{value:"auto",label:"Auto"}],l=oe([]),g=oe(),o=oe(!1),a=oe(0);async function u(c){n.importFromZip(c),g.value.clearFiles()}async function d(){Wi({message:`Downloading ${n.outputsLength} image(s)... (this may take a while)`,type:"info"}),o.value=!0,a.value=0;const c=await jt.outputs.toCollection().primaryKeys();await cC(c,!1,()=>{a.value++}),o.value=!1,a.value=0}return(c,f)=>(z(),se(je,null,[KG,le(C(jp),{"label-position":"top",model:C(t).options,onSubmit:f[10]||(f[10]=et(()=>{},["prevent"]))},{default:he(()=>[le(C(zj),{type:"border-card",style:{"min-height":"50vh"}},{default:he(()=>[le(C(zf),{label:"\u{1F5A8}\uFE0F Generation"},{default:he(()=>[GG,le(C(Di),{label:"Base URL"},{default:he(()=>[le(C(Xa),{class:"apikey",prop:"baseURL",modelValue:C(t).baseURL,"onUpdate:modelValue":f[0]||(f[0]=s=>C(t).baseURL=s)},null,8,["modelValue"])]),_:1}),qG,(z(!0),se(je,null,Nt(C(r).multiSelect,(s,h)=>{var v;return z(),se("div",{key:h},[le(so,{label:s.name,prop:"pageless",modelValue:s.state,"onUpdate:modelValue":m=>s.state=m,options:(v=s.allowedStates)!=null?v:[]},null,8,["label","modelValue","onUpdate:modelValue","options"])])}),128)),le(so,{label:"Allow Larger Params",prop:"pageless",modelValue:C(t).allowLargerParams,"onUpdate:modelValue":f[1]||(f[1]=s=>C(t).allowLargerParams=s),options:["Enabled","Disabled"]},null,8,["modelValue"]),le(po,{label:"Image Resize Mode",prop:"imageResizeMode",modelValue:C(t).imageResizeMode,"onUpdate:modelValue":f[2]||(f[2]=s=>C(t).imageResizeMode=s),options:[{label:"No Scale",value:"NoScale"},{label:"Scale and Crop",value:"ScaleAndCrop"},{label:"Scale and Pad",value:"ScaleAndPad"},{label:"Stretch",value:"Stretch"},{label:"Original",value:"Original"}],info:"How to adapt input image dimensions to the requested image size. No Scale: do not scale, just crop or pad each dimension to fit (default behavior). Scale and Crop: scale to match the smaller dimension, preserving aspect ratio, then crop to fit. Scale and Pad: scale to match the larger dimension, preserving aspect ratio, then pad to fit. Stretch: stretch both dimensions to fit, possibly not preserving aspect ratio. Original: send the input image as-is to the server, with no scaling, cropping or padding."},null,8,["modelValue"]),le(so,{label:"Video Gen: Request AVI download",prop:"pageless",modelValue:C(t).alsoRequestAvi,"onUpdate:modelValue":f[3]||(f[3]=s=>C(t).alsoRequestAvi=s),options:["Enabled","Disabled"]},null,8,["modelValue"])]),_:1}),le(C(zf),{label:"\u{1F4F7} Images"},{default:he(()=>[ZG,le(bn,{label:"Images Per Page",prop:"pageSize",modelValue:C(t).pageSize,"onUpdate:modelValue":f[4]||(f[4]=s=>C(t).pageSize=s),min:10,max:50,step:5,disabled:C(t).pageless==="Enabled"},null,8,["modelValue","disabled"]),le(so,{label:"Pageless Format",prop:"pageless",modelValue:C(t).pageless,"onUpdate:modelValue":f[5]||(f[5]=s=>C(t).pageless=s),options:["Enabled","Disabled"]},null,8,["modelValue"]),le(so,{label:"Carousel Auto Cycle",prop:"autoCarousel",modelValue:C(t).autoCarousel,"onUpdate:modelValue":f[6]||(f[6]=s=>C(t).autoCarousel=s),options:["Enabled","Disabled"]},null,8,["modelValue"]),le(so,{label:"Image Download Format",prop:"downloadType",modelValue:C(t).imageDownloadType,"onUpdate:modelValue":f[7]||(f[7]=s=>C(t).imageDownloadType=s),options:["PNG","JPG","WEBP","GIF"]},null,8,["modelValue"]),le(C(Di),{label:"Export Images (ZIP File)"},{default:he(()=>[o.value?(z(),be(C(at),{key:1,icon:C(Ks),disabled:""},{default:he(()=>[He("Downloading... ("+Ae(a.value)+" / "+Ae(C(n).outputsLength)+" image(s))",1)]),_:1},8,["icon"])):(z(),be(C(at),{key:0,icon:C(Ks),onClick:f[8]||(f[8]=s=>d())},{default:he(()=>[He("Download "+Ae(C(n).outputsLength)+" image(s)",1)]),_:1},8,["icon"]))]),_:1}),le(C(Di),{label:"Import Images (ZIP File)"},{default:he(()=>[le(C(Yp),{drag:"",ref_key:"upload",ref:g,"auto-upload":!1,onChange:u,"file-list":l.value,limit:1,multiple:""},{default:he(()=>[le(C(Le),{size:100},{default:he(()=>[le(C(dp))]),_:1}),JG]),_:1},8,["file-list"])]),_:1})]),_:1}),le(C(zf),{label:"\u2699\uFE0F General"},{default:he(()=>[QG,le(po,{label:"Color Scheme",prop:"colorScheme",modelValue:C(t).options.colorMode,"onUpdate:modelValue":f[9]||(f[9]=s=>C(t).options.colorMode=s),options:i},null,8,["modelValue"])]),_:1})]),_:1})]),_:1},8,["model"])],64))}});const tq=qt(eq,[["__scopeId","data-v-0345ee1f"]]),nq=Object.freeze(Object.defineProperty({__proto__:null,default:tq},Symbol.toStringTag,{value:"Module"})); - From 8d2e5806322d5207c164717eda617de58a1729c1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 17 Jun 2026 19:38:55 +0300 Subject: [PATCH 026/292] metal : add f16 and bf16 support for concat operator (#24724) * metal : add f16 and bf16 support for concat operator Extend the Metal backend concat operator to support f16 and bf16 tensor types in addition to the existing f32 and i32 support. - Template kernel_concat on type T with specializations for float, half, bfloat, and int - Add type-specific pipeline getter ggml_metal_library_get_pipeline_concat() - Update device support check to allow f16 unconditionally and bf16 when device supports bfloat16 - Update dispatch to select the correct kernel specialization by type Assisted-by: pi:llama.cpp/Qwen3.6-27B * metal : extend concat operator to support f16, bf16, i8, i16 and i64 Assisted-by: pi:llama.cpp/Qwen3.6-27B --- ggml/src/ggml-metal/ggml-metal-device.cpp | 16 ++++++++++- ggml/src/ggml-metal/ggml-metal-device.h | 1 + ggml/src/ggml-metal/ggml-metal-device.m | 21 ++++++++++---- ggml/src/ggml-metal/ggml-metal-ops.cpp | 2 +- ggml/src/ggml-metal/ggml-metal.metal | 35 +++++++++++++++-------- 5 files changed, 56 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index 4f4f073cb..cbc9459c9 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -66,7 +66,6 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_base(ggml const char * op_str = "undefined"; switch (op) { case GGML_OP_ADD_ID: op_str = "add_id"; break; - case GGML_OP_CONCAT: op_str = "concat"; break; default: GGML_ABORT("fatal error"); }; @@ -211,6 +210,21 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat(ggml_meta return res; } +ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat(ggml_metal_library_t lib, ggml_type tsrc) { + char base[256]; + char name[256]; + + snprintf(base, 256, "kernel_concat_%s", ggml_type_name(tsrc)); + snprintf(name, 256, "%s", base); + + ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); + if (!res.pipeline) { + res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr); + } + + return res; +} + ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary(ggml_metal_library_t lib, const ggml_tensor * op) { char base[256]; char name[256]; diff --git a/ggml/src/ggml-metal/ggml-metal-device.h b/ggml/src/ggml-metal/ggml-metal-device.h index 4a3ebb556..d465f31c0 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.h +++ b/ggml/src/ggml-metal/ggml-metal-device.h @@ -115,6 +115,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_get_rows struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_set_rows (ggml_metal_library_t lib, enum ggml_type tidx, enum ggml_type tdst); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_diag (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_repeat (ggml_metal_library_t lib, enum ggml_type tsrc); +struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_concat (ggml_metal_library_t lib, enum ggml_type tsrc); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_unary (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_glu (ggml_metal_library_t lib, const struct ggml_tensor * op); struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_sum (ggml_metal_library_t lib, const struct ggml_tensor * op); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index d583bd6ef..26b0e77f2 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1123,13 +1123,24 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te return true; case GGML_OP_CONCAT: { - // kernel_concat copies one float-sized value per element. - // Other scalar types need a type-generic copy kernel first. const enum ggml_type src0_type = op->src[0]->type; const enum ggml_type src1_type = op->src[1]->type; - return src0_type == src1_type && - src0_type == op->type && - (src0_type == GGML_TYPE_F32 || src0_type == GGML_TYPE_I32); + if (src0_type != src1_type || src0_type != op->type) { + return false; + } + switch (src0_type) { + case GGML_TYPE_F32: + case GGML_TYPE_F16: + case GGML_TYPE_I8: + case GGML_TYPE_I16: + case GGML_TYPE_I32: + case GGML_TYPE_I64: + return true; + case GGML_TYPE_BF16: + return has_bfloat; + default: + return false; + } } case GGML_OP_ADD: case GGML_OP_SUB: diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index e2ce56e9e..b13bf2711 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -556,7 +556,7 @@ int ggml_metal_op_concat(ggml_metal_op_t ctx, int idx) { /*.dim =*/ dim, }; - auto pipeline = ggml_metal_library_get_pipeline_base(lib, GGML_OP_CONCAT); + auto pipeline = ggml_metal_library_get_pipeline_concat(lib, op->type); ggml_metal_encoder_set_pipeline(enc, pipeline); ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), 0); diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 072f14e68..97abecb47 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -7513,14 +7513,15 @@ template [[host_name("kernel_cpy_q5_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32< template [[host_name("kernel_cpy_q5_1_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; template [[host_name("kernel_cpy_q8_0_f16")]] kernel cpy_q_f_t kernel_cpy_q_f32; +template kernel void kernel_concat( - constant ggml_metal_kargs_concat & args, - device const char * src0, - device const char * src1, - device char * dst, - uint3 tgpig[[threadgroup_position_in_grid]], - ushort3 tpitg[[thread_position_in_threadgroup]], - ushort3 ntg[[threads_per_threadgroup]]) { + constant ggml_metal_kargs_concat & args, + device const char * src0, + device const char * src1, + device char * dst, + uint3 tgpig[[threadgroup_position_in_grid]], + ushort3 tpitg[[thread_position_in_threadgroup]], + ushort3 ntg[[threads_per_threadgroup]]) { const int i3 = tgpig.z; const int i2 = tgpig.y; @@ -7533,21 +7534,31 @@ kernel void kernel_concat( int o[4] = {0, 0, 0, 0}; o[args.dim] = args.dim == 0 ? args.ne00 : (args.dim == 1 ? args.ne01 : (args.dim == 2 ? args.ne02 : args.ne03)); - device const float * x; - for (int i0 = tpitg.x; i0 < args.ne0; i0 += ntg.x) { + device const T * x; + if (i0 < args.ne00 && i1 < args.ne01 && i2 < args.ne02 && i3 < args.ne03) { - x = (device const float *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00); + x = (device const T *)(src0 + (i3 )*args.nb03 + (i2 )*args.nb02 + (i1 )*args.nb01 + (i0 )*args.nb00); } else { - x = (device const float *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10); + x = (device const T *)(src1 + (i3 - o[3])*args.nb13 + (i2 - o[2])*args.nb12 + (i1 - o[1])*args.nb11 + (i0 - o[0])*args.nb10); } - device float * y = (device float *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + device T * y = (device T *)(dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); *y = *x; } } +typedef decltype(kernel_concat) kernel_concat_t; + +template [[host_name("kernel_concat_f32")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_f16")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_bf16")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i8")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i32")]] kernel kernel_concat_t kernel_concat; +template [[host_name("kernel_concat_i64")]] kernel kernel_concat_t kernel_concat; + template void kernel_mul_mv_q2_K_f32_impl( args_t args, From 0843245cb1ed0129c81c0ae58c633329c496a3d7 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 17 Jun 2026 20:36:05 +0300 Subject: [PATCH 027/292] metal : implement rope_back operator (#24725) Reuse existing rope kernels with a function constant to toggle forward/backward rotation, avoiding duplicate kernel code. Assisted-by: pi:llama.cpp/Qwen3.6-27B --- ggml/src/ggml-metal/ggml-metal-device.cpp | 7 +++++-- ggml/src/ggml-metal/ggml-metal-device.m | 1 + ggml/src/ggml-metal/ggml-metal-ops.cpp | 1 + ggml/src/ggml-metal/ggml-metal.metal | 4 ++++ 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-device.cpp b/ggml/src/ggml-metal/ggml-metal-device.cpp index cbc9459c9..0e1f1de45 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.cpp +++ b/ggml/src/ggml-metal/ggml-metal-device.cpp @@ -1703,7 +1703,9 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_norm(ggml_metal_ } ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_library_t lib, const ggml_tensor * op) { - assert(op->op == GGML_OP_ROPE); + assert(op->op == GGML_OP_ROPE || op->op == GGML_OP_ROPE_BACK); + + const bool is_back = op->op == GGML_OP_ROPE_BACK; char base[256]; char name[256]; @@ -1727,13 +1729,14 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rope(ggml_metal_ snprintf(base, 256, "kernel_rope_norm_%s", ggml_type_name(op->src[0]->type)); } - snprintf(name, 256, "%s_imrope=%d", base, is_imrope ? 1 : 0); + snprintf(name, 256, "%s_imrope=%d_is_back=%d", base, is_imrope ? 1 : 0, is_back ? 1 : 0); ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name); if (!res.pipeline) { ggml_metal_cv_t cv = ggml_metal_cv_init(); ggml_metal_cv_set_bool(cv, is_imrope, FC_ROPE + 0); + ggml_metal_cv_set_bool(cv, is_back, FC_ROPE + 1); res = ggml_metal_library_compile_pipeline(lib, base, name, cv); diff --git a/ggml/src/ggml-metal/ggml-metal-device.m b/ggml/src/ggml-metal/ggml-metal-device.m index 26b0e77f2..a7cbc60eb 100644 --- a/ggml/src/ggml-metal/ggml-metal-device.m +++ b/ggml/src/ggml-metal/ggml-metal-device.m @@ -1184,6 +1184,7 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te case GGML_OP_RMS_NORM: return has_simdgroup_reduction && (ggml_is_contiguous_rows(op->src[0])); case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: return true; case GGML_OP_IM2COL: return ggml_is_contiguous(op->src[1]) && op->src[1]->type == GGML_TYPE_F32 && (op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_F32); diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index b13bf2711..18656b346 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -375,6 +375,7 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) { n_fuse = ggml_metal_op_norm(ctx, idx); } break; case GGML_OP_ROPE: + case GGML_OP_ROPE_BACK: { n_fuse = ggml_metal_op_rope(ctx, idx); } break; diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 97abecb47..31b5326ec 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -4358,6 +4358,7 @@ template [[host_name("kernel_mul_mv_bf16_bf16_short")]] kernel mul_mv_t_t_short_ #endif constant bool FC_rope_is_imrope [[function_constant(FC_ROPE + 0)]]; +constant bool FC_rope_is_back [[function_constant(FC_ROPE + 1)]]; static float rope_yarn_ramp(const float low, const float high, const int i0) { const float y = (i0 / 2 - low) / max(0.001f, high - low); @@ -4381,6 +4382,9 @@ static void rope_yarn( } *cos_theta = cos(theta) * mscale; *sin_theta = sin(theta) * mscale; + if (FC_rope_is_back) { + *sin_theta *= -1.0f; + } } // Apparently solving `n_rot = 2pi * x * base^((2 * max_pos_emb) / n_dims)` for x, we get From 2e88c49c90f0add8796f633fea8c3d65b975f295 Mon Sep 17 00:00:00 2001 From: shalinib-ibm Date: Thu, 18 Jun 2026 00:15:19 +0530 Subject: [PATCH 028/292] ggml-cpu: Conditionally enable power11 backend based on compiler support (#24687) * ggml: Conditionally enable power11 backend based on compiler support Guard POWER11 backend creation behind a compiler flag check for -mcpu=power11. This avoids build failures on current GCC/Clang toolchains while preserving forward compatibility once POWER11 support becomes available. * Update CMakeLists.txt ggml-cpu: Use -mcpu=power10 for P10 and P11 --- ggml/src/CMakeLists.txt | 9 ++++++++- ggml/src/ggml-cpu/CMakeLists.txt | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index c26c3f147..89e5180d9 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -438,7 +438,14 @@ if (GGML_CPU_ALL_VARIANTS) ggml_add_cpu_backend_variant(power8_2 POWER8 VSX) ggml_add_cpu_backend_variant(power9 POWER9 VSX) ggml_add_cpu_backend_variant(power10 POWER10 VSX) - ggml_add_cpu_backend_variant(power11 POWER11 VSX) + # POWER11 backend: only if compiler supports -mcpu=power11 + check_cxx_compiler_flag("-mcpu=power11" GGML_CXX_SUPPORTS_POWER11) + if (GGML_CXX_SUPPORTS_POWER11) + message(STATUS "Compiler supports -mcpu=power11, enabling POWER11 backend") + ggml_add_cpu_backend_variant(power11 POWER11 VSX) + else() + message(STATUS "Skipping POWER11 backend: compiler does not support -mcpu=power11") + endif() else() message(FATAL_ERROR "Unsupported PowerPC target OS: ${CMAKE_SYSTEM_NAME}") endif() diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 8c735a045..f7c557af4 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -389,7 +389,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) string(REGEX MATCHALL "POWER *([0-9]+)" MATCHED_STRING "${POWER10_M_UPPER}") string(REGEX REPLACE "POWER *([0-9]+)" "\\1" EXTRACTED_NUMBER "${MATCHED_STRING}") - if (EXTRACTED_NUMBER GREATER_EQUAL 10) + if (EXTRACTED_NUMBER EQUAL 10 OR EXTRACTED_NUMBER EQUAL 11) list(APPEND ARCH_FLAGS -mcpu=power10) elseif (EXTRACTED_NUMBER EQUAL 9) list(APPEND ARCH_FLAGS -mcpu=power9) From f3e182816421c648188b5eab269853bf1531d950 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Wed, 17 Jun 2026 22:40:50 +0200 Subject: [PATCH 029/292] mtmd: llava_uhd should no longer use batch dim (#24732) --- tools/mtmd/mtmd-image.cpp | 21 +++++++++++++++++++++ tools/mtmd/mtmd-image.h | 6 ++++++ tools/mtmd/mtmd.cpp | 31 ++++++------------------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index bedf44e07..656d75d80 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -1105,6 +1105,8 @@ bool mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img, cli img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std); output.entries.push_back(std::move(res)); } + output.grid_x = inst.grid_size.width; + output.grid_y = inst.grid_size.height; return true; } @@ -1558,3 +1560,22 @@ bool mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img, clip output.entries.push_back(std::move(img_f32)); return true; } + +bool mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { + // call super class preprocessor + bool ok = mtmd_image_preprocessor_llava_uhd::preprocess(img, output); + if (!ok) { + return false; + } + if (output.entries.size() == 1) { + // Single-tile (overview only): append one newline row. + output.entries[0]->add_newline = true; + } else { + // Multi-tile: overview gets no newline, grid tiles get one. + output.entries[0]->add_newline = false; + for (size_t i = 1; i < output.entries.size(); ++i) { + output.entries[i]->add_newline = true; + } + } + return true; +} diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 91a5bc253..3ddbec2e4 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -197,3 +197,9 @@ struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor { mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; }; + +// similar to llava_uhd, but has add_newline +struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd { + mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} + bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; +}; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index ad709227f..0834e4938 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -639,7 +639,7 @@ struct mtmd_context { { img_beg = ""; img_end = ""; - image_preproc = std::make_unique(ctx_v); + image_preproc = std::make_unique(ctx_v); } break; default: throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj)); @@ -1033,7 +1033,10 @@ struct mtmd_tokenizer { int32_t add_media(std::vector & bitmaps) { GGML_ASSERT(!bitmaps.empty()); - if (!bitmaps[0]->is_audio) { + // note: only one type of media is supported per call, caller should enforce this + const bool is_vision = !bitmaps[0]->is_audio; + + if (is_vision) { // handle image if (!ctx->ctx_v) { @@ -1085,31 +1088,9 @@ struct mtmd_tokenizer { batch_f32.grid_y = tmp_batch.grid_y; } - // Annotate llava-next style tiles so clip_n_output_tokens accounts - // for per-tile newline injection. - if (ctx->proj_type_v() == PROJECTOR_TYPE_GRANITE4_VISION) { - if (batch_f32.entries.size() == 1) { - // Single-tile (overview only): append one newline row. - batch_f32.entries[0]->add_newline = true; - } else { - // Multi-tile: overview gets no newline, grid tiles get one. - batch_f32.entries[0]->add_newline = false; - for (size_t i = 1; i < batch_f32.entries.size(); ++i) { - batch_f32.entries[i]->add_newline = true; - } - } - } - // handle llava-uhd style preprocessing const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0; - if ( - ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_5 - || ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_6 - || ctx->slice_tmpl == MTMD_SLICE_TMPL_LLAMA4 - || ctx->slice_tmpl == MTMD_SLICE_TMPL_IDEFICS3 - || ctx->slice_tmpl == MTMD_SLICE_TMPL_STEP3VL - || (ctx->slice_tmpl == MTMD_SLICE_TMPL_LFM2 && has_tiling_grid) - ) { + if (has_tiling_grid) { // [QWEN_VIDEO] we do not support "frame merging" for llama-uhd style, so no batching for now GGML_ASSERT(bitmaps.size() == 1); From cae0a3b0b074cbb111202c3b7e7884e8de56b23a Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 18 Jun 2026 09:16:06 +0300 Subject: [PATCH 030/292] metal : check for BF16 support in concat kernel (#24747) --- ggml/src/ggml-metal/ggml-metal.metal | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ggml/src/ggml-metal/ggml-metal.metal b/ggml/src/ggml-metal/ggml-metal.metal index 31b5326ec..25e78e100 100644 --- a/ggml/src/ggml-metal/ggml-metal.metal +++ b/ggml/src/ggml-metal/ggml-metal.metal @@ -7557,7 +7557,9 @@ typedef decltype(kernel_concat) kernel_concat_t; template [[host_name("kernel_concat_f32")]] kernel kernel_concat_t kernel_concat; template [[host_name("kernel_concat_f16")]] kernel kernel_concat_t kernel_concat; +#if defined(GGML_METAL_HAS_BF16) template [[host_name("kernel_concat_bf16")]] kernel kernel_concat_t kernel_concat; +#endif template [[host_name("kernel_concat_i8")]] kernel kernel_concat_t kernel_concat; template [[host_name("kernel_concat_i16")]] kernel kernel_concat_t kernel_concat; template [[host_name("kernel_concat_i32")]] kernel kernel_concat_t kernel_concat; From 4a79037b8bd8e6adc7f6379d3054934d4b5df0d0 Mon Sep 17 00:00:00 2001 From: Ravi Panchumarthy Date: Wed, 17 Jun 2026 23:30:08 -0700 Subject: [PATCH 031/292] ci : fix Windows x64 (OpenVINO) release link (#24731) --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d5bcb4d4e..d452e3787 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -542,6 +542,7 @@ jobs: steps: - name: Set OpenVINO version output id: openvino_version + shell: bash run: echo "value=${{ env.OPENVINO_VERSION_MAJOR }}" >> $GITHUB_OUTPUT - name: Clone From 0b73fc79fe7c8968b040e9c96d91610d7d2a8206 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Thu, 18 Jun 2026 08:33:50 +0200 Subject: [PATCH 032/292] ui: Update code formatting command in pre-commit hook (#24685) --- tools/ui/scripts/git-hooks/pre-commit.sh | 2 +- tools/ui/scripts/git-hooks/pre-push.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/ui/scripts/git-hooks/pre-commit.sh b/tools/ui/scripts/git-hooks/pre-commit.sh index 1fa83efde..208ef56fc 100755 --- a/tools/ui/scripts/git-hooks/pre-commit.sh +++ b/tools/ui/scripts/git-hooks/pre-commit.sh @@ -27,7 +27,7 @@ echo "Running pre-commit checks for llama-ui..." # Format only staged files staged_ui=$(git diff --cached --name-only -- tools/ui/) if [ -n "$staged_ui" ]; then - echo "$staged_ui" | xargs npx --no-install prettier --write + echo "$staged_ui" | xargs npm run format format_ok=$? # Re-stage formatted files git add tools/ui/ diff --git a/tools/ui/scripts/git-hooks/pre-push.sh b/tools/ui/scripts/git-hooks/pre-push.sh index 953d3a224..66d79b950 100755 --- a/tools/ui/scripts/git-hooks/pre-push.sh +++ b/tools/ui/scripts/git-hooks/pre-push.sh @@ -57,6 +57,7 @@ if [ $lint_ok -ne 0 ]; then echo "❌ Lint failed" exit 1 fi + if [ $test_ok -ne 0 ]; then echo "❌ Tests failed" exit 1 From 6f1034b32af66d3efa2a572a386cc9bccb01ba55 Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Thu, 18 Jun 2026 14:40:03 +0800 Subject: [PATCH 033/292] [SYCL] support OPs: conv_2d, conv_2d_dw, conv2d_transpose (#24600) * fix conflict * fix format issue, rename * rm debug code * correct the file name --- docs/backend/SYCL.md | 58 + docs/ops.md | 6 +- docs/ops/SYCL.csv | 3164 +++++++++++------------ examples/sycl/update-ops-doc.sh | 9 + examples/sycl/win-update-ops-doc.bat | 8 + ggml/src/ggml-sycl/conv2d-dw.cpp | 158 ++ ggml/src/ggml-sycl/conv2d-dw.hpp | 10 + ggml/src/ggml-sycl/conv2d-transpose.cpp | 125 + ggml/src/ggml-sycl/conv2d-transpose.hpp | 10 + ggml/src/ggml-sycl/conv2d.cpp | 150 ++ ggml/src/ggml-sycl/conv2d.hpp | 10 + ggml/src/ggml-sycl/ggml-sycl.cpp | 20 +- 12 files changed, 2141 insertions(+), 1587 deletions(-) create mode 100755 examples/sycl/update-ops-doc.sh create mode 100644 examples/sycl/win-update-ops-doc.bat create mode 100644 ggml/src/ggml-sycl/conv2d-dw.cpp create mode 100644 ggml/src/ggml-sycl/conv2d-dw.hpp create mode 100644 ggml/src/ggml-sycl/conv2d-transpose.cpp create mode 100644 ggml/src/ggml-sycl/conv2d-transpose.hpp create mode 100644 ggml/src/ggml-sycl/conv2d.cpp create mode 100644 ggml/src/ggml-sycl/conv2d.hpp diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 6617aa2a5..68f6e60e6 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -161,6 +161,64 @@ You could update your test result in it directly. Please refer to [Docker with SYCL](../docker.md#docker-with-sycl) for details. +## Quick Development WOW + +This chapter is for quick development & try with SYCL backend on Intel GPU. + +You need to install following sofeware before development: + - Intel GPU driver + - oneAPI package + - other development tools. + +Please refer to [Linux](#linux) or [Windows](#windows-1) for above installation and resolve the trouble in usage. There are the detailed guide. + +- Linux + +``` +## build from source code +./examples/sycl/build.sh + +## run CONV_2D_DW unit test cases +./build/bin/test-backend-ops -b SYCL0 -o CONV_2D_DW + +## run all unit test cases +./build/bin/test-backend-ops -b SYCL0 + +## run with LLM on the first GPU +./examples/sycl/test.sh -mg 0 -m xxxx.gguf + +## run service with LLM on the first GPU +export ONEAPI_DEVICE_SELECTOR="level_zero:0" +./examples/sycl/start-svr.sh -m xxxx.gguf + +## update the docs/ops.md for new/update OPs +./examples/sycl/update-ops-doc.sh +``` + +- Windows + +``` +## build from source code +examples\sycl\win-build-sycl.bat + +## run CONV_2D_DW unit test cases +build\bin\test-backend-ops.exe -b SYCL0 -o CONV_2D_DW + +## run all unit test cases +build\bin\test-backend-ops.exe -b SYCL0 + +## run LLM on the first GPU +examples\sycl\win-test.bat -mg 0 -m xxxx.gguf + +## run service with LLM on the first GPU +set ONEAPI_DEVICE_SELECTOR="level_zero:0" +examples\sycl\win-start-svr.bat -m xxxx.gguf + +## update the docs/ops.md for new/update OPs +examples\sycl\win-update-ops-doc.bat +``` + + ## Linux ### I. Setup Environment diff --git a/docs/ops.md b/docs/ops.md index 9e9442024..6fc8454c8 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -27,11 +27,11 @@ Legend: | COL2IM_1D | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | CONCAT | ❌ | ✅ | ✅ | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | CONT | ❌ | 🟡 | ✅ | ✅ | ✅ | 🟡 | 🟡 | ✅ | 🟡 | ❌ | ❌ | -| CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | -| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | +| CONV_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | +| CONV_2D_DW | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CONV_3D | ❌ | ❌ | ✅ | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | | CONV_TRANSPOSE_1D | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | -| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | +| CONV_TRANSPOSE_2D | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | COS | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | 🟡 | ✅ | ❌ | ❌ | | COUNT_EQUAL | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | | CPY | ❌ | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | 🟡 | ❌ | ❌ | diff --git a/docs/ops/SYCL.csv b/docs/ops/SYCL.csv index 116197d2f..80a1c29c1 100644 --- a/docs/ops/SYCL.csv +++ b/docs/ops/SYCL.csv @@ -3104,1578 +3104,1578 @@ "SYCL0","IM2COL_3D","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,20,10,3],ne_kernel=[3,3,3,3],IC=1,s0=3,s1=3,s2=3,p0=3,p1=3,p2=3,d0=3,d1=3,d2=3,v=1","support","1","yes","SYCL" "SYCL0","IM2COL_3D","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,20,10,3],ne_kernel=[3,3,3,3],IC=3,s0=3,s1=3,s2=3,p0=3,p1=3,p2=3,d0=3,d1=3,d2=3,v=0","support","1","yes","SYCL" "SYCL0","IM2COL_3D","type_input=f32,type_kernel=f32,dst_type=f32,ne_input=[20,20,10,3],ne_kernel=[3,3,3,3],IC=3,s0=3,s1=3,s2=3,p0=3,p1=3,p2=3,d0=3,d1=3,d2=3,v=1","support","1","yes","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=1","support","0","no","SYCL" -"SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=0","support","0","no","SYCL" -"SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=1","support","0","no","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=1,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=2,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,1,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,2,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,3,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[1,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[2,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[3,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,1,2],ne_kernel=[11,11,1,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,1],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,1,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,2,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,1,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,3,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[1,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[2,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[1,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[3,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f32,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D","ne_input=[141,133,25,2],ne_kernel=[11,11,25,12],type_kernel=f16,stride0=3,stride1=5,padding0=5,padding1=5,dilation0=2,dilation1=4,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D_DW","ne_input=[17,34,9,1],ne_kernel=[3,3,1,9],stride=1,padding=0,dilation=1,cwhn=1","support","1","yes","SYCL" +"SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=0","support","1","yes","SYCL" +"SYCL0","CONV_2D_DW","ne_input=[32,8,64,1],ne_kernel=[3,3,1,64],stride=2,padding=1,dilation=1,cwhn=1","support","1","yes","SYCL" "SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=1,d1=1,d2=1,type_kernel=f32","support","1","yes","SYCL" "SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=1,KW=5,s0=2,s1=1,s2=1,p0=2,p1=0,p2=1,d0=1,d1=1,d2=2,type_kernel=f32","support","1","yes","SYCL" "SYCL0","CONV_3D","N=1,IC=1,ID=18,IH=22,IW=20,OC=1,KD=3,KH=3,KW=3,s0=1,s1=1,s2=1,p0=0,p1=0,p2=0,d0=2,d1=2,d2=2,type_kernel=f32","support","1","yes","SYCL" @@ -5083,12 +5083,12 @@ "SYCL0","COL2IM_1D","type=bf16,K=16,OC=1,T_in=197,s0=8,p0=0","support","0","no","SYCL" "SYCL0","COL2IM_1D","type=bf16,K=1,OC=5,T_in=13,s0=3,p0=0","support","0","no","SYCL" "SYCL0","COL2IM_1D","type=bf16,K=8,OC=2,T_in=3,s0=2,p0=5","support","0","no","SYCL" -"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","0","no","SYCL" -"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","0","no","SYCL" -"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","0","no","SYCL" -"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","0","no","SYCL" -"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","0","no","SYCL" -"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","0","no","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f32,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[3,2,3,1],ne_kernel=[2,2,1,3],stride=1","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[10,10,9,1],ne_kernel=[3,3,1,9],stride=2","support","1","yes","SYCL" +"SYCL0","CONV_TRANSPOSE_2D","kernel_type=f16,ne_input=[129,63,35,1],ne_kernel=[3,3,48,35],stride=1","support","1","yes","SYCL" "SYCL0","COUNT_EQUAL","type=f32,ne=[4,500,1,1]","support","1","yes","SYCL" "SYCL0","COUNT_EQUAL","type=f32,ne=[4,5000,1,1]","support","1","yes","SYCL" "SYCL0","ARGMAX","type=f32,ne=[32,1,1,1]","support","1","yes","SYCL" @@ -11046,8 +11046,8 @@ "SYCL0","ARGSORT","type=f32,ne=[8192,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16383,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16384,1,1,1],order=0","support","1","yes","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","1","yes","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","1","yes","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","0","no","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65535,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65536,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[131071,1,1,1],order=0","support","0","no","SYCL" @@ -11095,8 +11095,8 @@ "SYCL0","ARGSORT","type=f32,ne=[8192,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16383,1,1,1],order=0","support","1","yes","SYCL" "SYCL0","ARGSORT","type=f32,ne=[16384,1,1,1],order=0","support","1","yes","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","1","yes","SYCL" -"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","1","yes","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32767,1,1,1],order=0","support","0","no","SYCL" +"SYCL0","ARGSORT","type=f32,ne=[32768,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65535,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[65536,1,1,1],order=0","support","0","no","SYCL" "SYCL0","ARGSORT","type=f32,ne=[131071,1,1,1],order=0","support","0","no","SYCL" diff --git a/examples/sycl/update-ops-doc.sh b/examples/sycl/update-ops-doc.sh new file mode 100755 index 000000000..6f26fc457 --- /dev/null +++ b/examples/sycl/update-ops-doc.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# MIT license +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: MIT + +./build/bin/test-backend-ops support --output csv > docs/ops/SYCL.csv +./scripts/create_ops_docs.py + diff --git a/examples/sycl/win-update-ops-doc.bat b/examples/sycl/win-update-ops-doc.bat new file mode 100644 index 000000000..b032bcfe1 --- /dev/null +++ b/examples/sycl/win-update-ops-doc.bat @@ -0,0 +1,8 @@ +@echo off + +rem MIT license +rem Copyright (C) 2026 Intel Corporation +rem SPDX-License-Identifier: MIT + +build\bin\test-backend-ops support --output csv > docs\ops\SYCL.csv +python scripts\create_ops_docs.py diff --git a/ggml/src/ggml-sycl/conv2d-dw.cpp b/ggml/src/ggml-sycl/conv2d-dw.cpp new file mode 100644 index 000000000..0a52b7917 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-dw.cpp @@ -0,0 +1,158 @@ +#include "conv2d-dw.hpp" + +struct conv2d_dw_params { + int in_w, in_h; + int out_w, out_h; + int kernel_w, kernel_h; + int stride_x, stride_y; + int padding_x, padding_y; + int dilation_x, dilation_y; + int channels, batches; +}; + +struct conv2d_dw_kernel_bounds { + int y_min, y_max; + int x_min, x_max; +}; + +static inline conv2d_dw_kernel_bounds dw_calculate_kernel_bounds(int out_x, int out_y, + const conv2d_dw_params & p) { + conv2d_dw_kernel_bounds bounds; + bounds.y_min = sycl::max(0, (p.padding_y - out_y * p.stride_y + p.dilation_y - 1) / p.dilation_y); + bounds.y_max = sycl::min(p.kernel_h, + (p.in_h + p.padding_y - out_y * p.stride_y + p.dilation_y - 1) / p.dilation_y); + bounds.x_min = sycl::max(0, (p.padding_x - out_x * p.stride_x + p.dilation_x - 1) / p.dilation_x); + bounds.x_max = sycl::min(p.kernel_w, + (p.in_w + p.padding_x - out_x * p.stride_x + p.dilation_x - 1) / p.dilation_x); + return bounds; +} + +static inline int dw_calculate_input_coord(int out_coord, int kern_coord, int stride, int dilation, int padding) { + return out_coord * stride + kern_coord * dilation - padding; +} + +// whcn layout: input/output stored as [N, C, H, W] +struct dw_whcn_layout { + static int input_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.in_w * p.in_h) + c * p.in_w * p.in_h + y * p.in_w + x; + } + static int kernel_index(int c, int ky, int kx, const conv2d_dw_params & p) { + return c * p.kernel_h * p.kernel_w + ky * p.kernel_w + kx; + } + static int output_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.out_w * p.out_h) + c * p.out_w * p.out_h + y * p.out_w + x; + } + static void unpack_indices(int global_idx, const conv2d_dw_params & p, + int & n, int & c, int & out_y, int & out_x) { + out_x = global_idx % p.out_w; + out_y = (global_idx / p.out_w) % p.out_h; + c = (global_idx / (p.out_w * p.out_h)) % p.channels; + n = global_idx / (p.out_w * p.out_h * p.channels); + } +}; + +// cwhn layout: input/output stored as [N, H, W, C] +struct dw_cwhn_layout { + static int input_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.in_w * p.in_h) + (y * p.in_w + x) * p.channels + c; + } + static int kernel_index(int c, int ky, int kx, const conv2d_dw_params & p) { + return (ky * p.kernel_w + kx) * p.channels + c; + } + static int output_index(int n, int c, int y, int x, const conv2d_dw_params & p) { + return n * (p.channels * p.out_w * p.out_h) + y * (p.out_w * p.channels) + x * p.channels + c; + } + static void unpack_indices(int global_idx, const conv2d_dw_params & p, + int & n, int & c, int & out_y, int & out_x) { + c = global_idx % p.channels; + out_x = (global_idx / p.channels) % p.out_w; + out_y = (global_idx / (p.channels * p.out_w)) % p.out_h; + n = global_idx / (p.channels * p.out_w * p.out_h); + } +}; + +template +static void conv2d_dw_kernel(const float * input, const float * kernel, float * output, + const conv2d_dw_params p, const sycl::nd_item<3> & item_ct1) { + const int global_idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + const int total_elements = p.batches * p.channels * p.out_h * p.out_w; + + if (global_idx >= total_elements) { + return; + } + + int n, c, out_y, out_x; + Layout::unpack_indices(global_idx, p, n, c, out_y, out_x); + + float acc = 0.0f; + const conv2d_dw_kernel_bounds bounds = dw_calculate_kernel_bounds(out_x, out_y, p); + + for (int ky = bounds.y_min; ky < bounds.y_max; ++ky) { + const int in_y = dw_calculate_input_coord(out_y, ky, p.stride_y, p.dilation_y, p.padding_y); + for (int kx = bounds.x_min; kx < bounds.x_max; ++kx) { + const int in_x = dw_calculate_input_coord(out_x, kx, p.stride_x, p.dilation_x, p.padding_x); + acc += input[Layout::input_index(n, c, in_y, in_x, p)] * + kernel[Layout::kernel_index(c, ky, kx, p)]; + } + } + + output[Layout::output_index(n, c, out_y, out_x, p)] = acc; +} + +template +static void conv2d_dw_sycl(const float * x_d, const float * w_d, float * y_d, + const conv2d_dw_params p, const queue_ptr & stream) { + const int total = p.batches * p.channels * p.out_h * p.out_w; + const int num_blocks = (total + SYCL_CONV2D_DW_BLOCK_SIZE - 1) / SYCL_CONV2D_DW_BLOCK_SIZE; + const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_DW_BLOCK_SIZE); + const sycl::range<3> block_nums(1, 1, num_blocks); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + conv2d_dw_kernel(x_d, w_d, y_d, p, item_ct1); + }); +} + +void ggml_sycl_op_conv2d_dw(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + + GGML_ASSERT(kernel->type == GGML_TYPE_F32 && input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + + const float * w_d = (const float *) kernel->data; + const float * x_d = (const float *) input->data; + float * y_d = (float *) dst->data; + + const int32_t * p = (const int32_t *) dst->op_params; + const int stride_x = p[0]; + const int stride_y = p[1]; + const int padding_x = p[2]; + const int padding_y = p[3]; + const int dilation_x = p[4]; + const int dilation_y = p[5]; + + const int in_w = input->ne[0]; + const int in_h = input->ne[1]; + const int kernel_w = kernel->ne[0]; + const int kernel_h = kernel->ne[1]; + const int out_w = dst->ne[0]; + const int out_h = dst->ne[1]; + const int channels = dst->ne[2]; + const int batches = dst->ne[3]; + + const conv2d_dw_params params = { in_w, in_h, out_w, out_h, kernel_w, kernel_h, + stride_x, stride_y, padding_x, padding_y, + dilation_x, dilation_y, channels, batches }; + + const queue_ptr stream = ctx.stream(); + + if (ggml_is_contiguous(input)) { + conv2d_dw_sycl(x_d, w_d, y_d, params, stream); + } else if (ggml_is_contiguous_channels(input)) { + conv2d_dw_sycl(x_d, w_d, y_d, params, stream); + } else { + GGML_ABORT("Unsupported memory layout for conv2d_dw"); + } +} diff --git a/ggml/src/ggml-sycl/conv2d-dw.hpp b/ggml/src/ggml-sycl/conv2d-dw.hpp new file mode 100644 index 000000000..532892221 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-dw.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_CONV2D_DW_HPP +#define GGML_SYCL_CONV2D_DW_HPP + +#include "common.hpp" + +#define SYCL_CONV2D_DW_BLOCK_SIZE 256 + +void ggml_sycl_op_conv2d_dw(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV2D_DW_HPP diff --git a/ggml/src/ggml-sycl/conv2d-transpose.cpp b/ggml/src/ggml-sycl/conv2d-transpose.cpp new file mode 100644 index 000000000..07c325cc6 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-transpose.cpp @@ -0,0 +1,125 @@ +#include "conv2d-transpose.hpp" +#include "convert.hpp" + +template +static void conv2d_transpose_kernel(const float * input, const kernel_t * kernel, float * output, + const int in_w, const int in_h, + const int out_w, const int out_h, + const int kernel_w, const int kernel_h, + const int stride, + const int c_in, const int c_out, const int batches, + const sycl::nd_item<3> & item_ct1) { + const int global_idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + const int total_elements = out_w * out_h * c_out * batches; + + if (global_idx >= total_elements) { + return; + } + + const int out_x = global_idx % out_w; + const int out_y = (global_idx / out_w) % out_h; + const int c_idx = (global_idx / (out_w * out_h)) % c_out; + const int n_idx = global_idx / (out_w * out_h * c_out); + + float acc = 0.0f; + + for (int c_in_idx = 0; c_in_idx < c_in; ++c_in_idx) { + for (int kh = 0; kh < kernel_h; ++kh) { + int in_y = out_y - kh; + if (in_y < 0 || in_y % stride) { + continue; + } + in_y /= stride; + if (in_y >= in_h) { + continue; + } + + for (int kw = 0; kw < kernel_w; ++kw) { + int in_x = out_x - kw; + if (in_x < 0 || in_x % stride) { + continue; + } + in_x /= stride; + if (in_x >= in_w) { + continue; + } + + const int input_idx = (in_w * in_h * c_in) * n_idx + (in_w * in_h) * c_in_idx + in_w * in_y + in_x; + const int kernel_idx = (kernel_h * kernel_w * c_out) * c_in_idx + (kernel_h * kernel_w) * c_idx + + kernel_w * kh + kw; + + acc += input[input_idx] * ggml_sycl_cast(kernel[kernel_idx]); + } + } + } + + output[(out_w * out_h * c_out) * n_idx + (out_w * out_h) * c_idx + out_w * out_y + out_x] = acc; +} + +template +static void conv2d_transpose_sycl(const float * input_d, const kernel_t * kernel_d, float * output_d, + const int in_w, const int in_h, + const int out_w, const int out_h, + const int kernel_w, const int kernel_h, + const int stride, + const int c_in, const int c_out, const int batches, + const queue_ptr & stream) { + const int total = out_w * out_h * c_out * batches; + const int num_blocks = (total + SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE - 1) / SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE; + const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE); + const sycl::range<3> block_nums(1, 1, num_blocks); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + conv2d_transpose_kernel(input_d, kernel_d, output_d, + in_w, in_h, out_w, out_h, kernel_w, kernel_h, + stride, c_in, c_out, batches, item_ct1); + }); +} + +// input: (W, H, C_in, N) +// kernel: (W, H, C_out, C_in) +// output: (W, H, C_out, N) +void ggml_sycl_op_conv2d_transpose(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(input->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32); + + GGML_ASSERT(ggml_is_contiguous(input)); + GGML_ASSERT(ggml_is_contiguous(kernel)); + GGML_ASSERT(ggml_is_contiguous(dst)); + + const float * input_d = (const float *) input->data; + float * output_d = (float *) dst->data; + const void * kernel_d = kernel->data; + + const int input_w = input->ne[0]; + const int input_h = input->ne[1]; + const int channels_in = input->ne[2]; + const int batches = input->ne[3]; + const int output_w = dst->ne[0]; + const int output_h = dst->ne[1]; + const int channels_out = kernel->ne[2]; + const int kernel_w = kernel->ne[0]; + const int kernel_h = kernel->ne[1]; + const int stride = dst->op_params[0]; + + GGML_ASSERT(channels_in == kernel->ne[3]); + GGML_ASSERT(stride > 0); + + const queue_ptr stream = ctx.stream(); + + if (kernel->type == GGML_TYPE_F16) { + conv2d_transpose_sycl(input_d, (const sycl::half *) kernel_d, output_d, + input_w, input_h, output_w, output_h, kernel_w, kernel_h, + stride, channels_in, channels_out, batches, stream); + } else { + conv2d_transpose_sycl(input_d, (const float *) kernel_d, output_d, + input_w, input_h, output_w, output_h, kernel_w, kernel_h, + stride, channels_in, channels_out, batches, stream); + } +} diff --git a/ggml/src/ggml-sycl/conv2d-transpose.hpp b/ggml/src/ggml-sycl/conv2d-transpose.hpp new file mode 100644 index 000000000..ca067318d --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d-transpose.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_CONV2D_TRANSPOSE_HPP +#define GGML_SYCL_CONV2D_TRANSPOSE_HPP + +#include "common.hpp" + +#define SYCL_CONV2D_TRANSPOSE_BLOCK_SIZE 256 + +void ggml_sycl_op_conv2d_transpose(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV2D_TRANSPOSE_HPP diff --git a/ggml/src/ggml-sycl/conv2d.cpp b/ggml/src/ggml-sycl/conv2d.cpp new file mode 100644 index 000000000..3b3b49d05 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d.cpp @@ -0,0 +1,150 @@ +#include "conv2d.hpp" +#include "convert.hpp" + +struct conv2d_params { + const int64_t IW, IH; + const int64_t OW, OH; + const int64_t KW, KH; + const int64_t ST_X, ST_Y; + const int64_t PD_X, PD_Y; + const int64_t DL_X, DL_Y; + const int64_t IC, OC; + const int64_t B; + const int64_t TOTAL; +}; + +struct conv2d_kernel_bounds { + int64_t y_min, y_max; + int64_t x_min, x_max; +}; + +static inline int64_t conv2d_max64(int64_t a, int64_t b) { + return (a > b) ? a : b; +} + +static inline int64_t conv2d_min64(int64_t a, int64_t b) { + return (a < b) ? a : b; +} + +static inline conv2d_kernel_bounds calculate_kernel_bounds(int64_t out_x, int64_t out_y, const conv2d_params & P) { + conv2d_kernel_bounds bounds; + bounds.y_min = conv2d_max64(0, (P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y); + bounds.y_max = conv2d_min64(P.KH, (P.IH + P.PD_Y - out_y * P.ST_Y + P.DL_Y - 1) / P.DL_Y); + bounds.x_min = conv2d_max64(0, (P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X); + bounds.x_max = conv2d_min64(P.KW, (P.IW + P.PD_X - out_x * P.ST_X + P.DL_X - 1) / P.DL_X); + return bounds; +} + +static inline int calculate_input_coord(int64_t out_coord, int64_t kern_coord, int64_t stride, + int64_t dilation, int64_t padding) { + return out_coord * stride + kern_coord * dilation - padding; +} + +// whcn layout helpers (matching ggml tensor memory order) +static inline int64_t whcn_input_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) { + return n * (P.IC * P.IW * P.IH) + c * P.IW * P.IH + y * P.IW + x; +} + +static inline int64_t whcn_kernel_index(int64_t c_out, int64_t c_in, int64_t ky, int64_t kx, const conv2d_params & P) { + return c_out * (P.IC * P.KH * P.KW) + c_in * (P.KH * P.KW) + ky * P.KW + kx; +} + +static inline int64_t whcn_output_index(int64_t n, int64_t c, int64_t y, int64_t x, const conv2d_params & P) { + return n * (P.OC * P.OW * P.OH) + c * P.OW * P.OH + y * P.OW + x; +} + +template +static void conv2d_kernel(const float * input, const T * kernel, float * output, + const conv2d_params P, const sycl::nd_item<3> & item_ct1) { + const int64_t global_idx = item_ct1.get_local_id(2) + + item_ct1.get_group(2) * item_ct1.get_local_range(2); + + if (global_idx >= P.TOTAL) { + return; + } + + const int64_t out_x = global_idx % P.OW; + const int64_t out_y = (global_idx / P.OW) % P.OH; + const int64_t c_out = (global_idx / (P.OW * P.OH)) % P.OC; + const int64_t n = global_idx / (P.OW * P.OH * P.OC); + + float acc = 0.0f; + + const conv2d_kernel_bounds bounds = calculate_kernel_bounds(out_x, out_y, P); + + for (int64_t c_in = 0; c_in < P.IC; ++c_in) { + for (int64_t ky = bounds.y_min; ky < bounds.y_max; ++ky) { + const int64_t in_y = calculate_input_coord(out_y, ky, P.ST_Y, P.DL_Y, P.PD_Y); + for (int64_t kx = bounds.x_min; kx < bounds.x_max; ++kx) { + const int64_t in_x = calculate_input_coord(out_x, kx, P.ST_X, P.DL_X, P.PD_X); + const float input_val = input[whcn_input_index(n, c_in, in_y, in_x, P)]; + const T kernel_val = kernel[whcn_kernel_index(c_out, c_in, ky, kx, P)]; + acc += input_val * ggml_sycl_cast(kernel_val); + } + } + } + + output[whcn_output_index(n, c_out, out_y, out_x, P)] = acc; +} + +template +static void conv2d_sycl(const float * X_D, const T * K_D, float * Y_D, + const conv2d_params P, const queue_ptr & stream) { + const int num_blocks = (P.TOTAL + SYCL_CONV2D_BLOCK_SIZE - 1) / SYCL_CONV2D_BLOCK_SIZE; + const sycl::range<3> block_dims(1, 1, SYCL_CONV2D_BLOCK_SIZE); + const sycl::range<3> block_nums(1, 1, num_blocks); + stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) { + conv2d_kernel(X_D, K_D, Y_D, P, item_ct1); + }); +} + +void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + + const ggml_tensor * kernel = dst->src[0]; + const ggml_tensor * input = dst->src[1]; + const float * K_D = (const float *) kernel->data; + const float * X_D = (const float *) input->data; + float * Y_D = (float *) dst->data; + + GGML_ASSERT(ggml_is_contiguous(kernel)); + GGML_ASSERT(kernel->type == GGML_TYPE_F16 || kernel->type == GGML_TYPE_F32); + GGML_ASSERT(input->type == GGML_TYPE_F32); + GGML_ASSERT(dst->type == GGML_TYPE_F32); + + // same number of input channels + GGML_ASSERT(input->ne[2] == kernel->ne[2]); + + const queue_ptr stream = ctx.stream(); + + const int32_t * p = (const int32_t *) dst->op_params; + const int ST_X = p[0]; + const int ST_Y = p[1]; + const int PD_X = p[2]; + const int PD_Y = p[3]; + const int DL_X = p[4]; + const int DL_Y = p[5]; + + // no cwhn layout support + GGML_ASSERT(p[6] == 0); + + const int IW = input->ne[0]; + const int IH = input->ne[1]; + const int OW = dst->ne[0]; + const int OH = dst->ne[1]; + const int KW = kernel->ne[0]; + const int KH = kernel->ne[1]; + const int IC = input->ne[2]; + const int OC = kernel->ne[3]; + const int B = input->ne[3]; + + const int64_t total = (int64_t) B * OC * OH * OW; + const conv2d_params params = { IW, IH, OW, OH, KW, KH, ST_X, ST_Y, PD_X, PD_Y, DL_X, DL_Y, IC, OC, B, total }; + + if (kernel->type == GGML_TYPE_F16) { + conv2d_sycl(X_D, (const sycl::half *) K_D, Y_D, params, stream); + } else { + conv2d_sycl(X_D, K_D, Y_D, params, stream); + } +} diff --git a/ggml/src/ggml-sycl/conv2d.hpp b/ggml/src/ggml-sycl/conv2d.hpp new file mode 100644 index 000000000..efd25ab42 --- /dev/null +++ b/ggml/src/ggml-sycl/conv2d.hpp @@ -0,0 +1,10 @@ +#ifndef GGML_SYCL_CONV2D_HPP +#define GGML_SYCL_CONV2D_HPP + +#include "common.hpp" + +#define SYCL_CONV2D_BLOCK_SIZE 256 + +void ggml_sycl_op_conv2d(ggml_backend_sycl_context & ctx, ggml_tensor * dst); + +#endif // GGML_SYCL_CONV2D_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 4c0567669..77d145890 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -62,6 +62,9 @@ #include "ggml-sycl/repeat_back.hpp" #include "ggml-sycl/set_rows.hpp" #include "ggml-sycl/set.hpp" +#include "ggml-sycl/conv2d.hpp" +#include "ggml-sycl/conv2d-dw.hpp" +#include "ggml-sycl/conv2d-transpose.hpp" #include "ggml-sycl/ssm_conv.hpp" #include "ggml-sycl/sycl_hw.hpp" #include "ggml-sycl/ssm_scan.hpp" @@ -4664,12 +4667,21 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_ARGMAX: ggml_sycl_argmax(ctx, dst); break; - case GGML_OP_CONV_TRANSPOSE_1D: - ggml_sycl_op_conv_transpose_1d(ctx, dst); + case GGML_OP_CONV_2D: + ggml_sycl_op_conv2d(ctx, dst); + break; + case GGML_OP_CONV_2D_DW: + ggml_sycl_op_conv2d_dw(ctx, dst); break; case GGML_OP_CONV_3D: ggml_sycl_conv_3d(ctx, dst); break; + case GGML_OP_CONV_TRANSPOSE_1D: + ggml_sycl_op_conv_transpose_1d(ctx, dst); + break; + case GGML_OP_CONV_TRANSPOSE_2D: + ggml_sycl_op_conv2d_transpose(ctx, dst); + break; case GGML_OP_REPEAT: ggml_sycl_repeat(ctx, dst); break; @@ -5387,6 +5399,10 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g } return false; } + case GGML_OP_CONV_2D: + case GGML_OP_CONV_2D_DW: + case GGML_OP_CONV_TRANSPOSE_2D: + return true; case GGML_OP_UNARY: switch (ggml_get_unary_op(op)) { case GGML_UNARY_OP_SGN: From 32e806b9c16ab298ea8ea791c4fc0df390015a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= <1629204+CISC@users.noreply.github.com> Date: Thu, 18 Jun 2026 09:32:56 +0200 Subject: [PATCH 034/292] ci : fix check-release message parsing (#24751) --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d452e3787..9789215f2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,11 +46,13 @@ jobs: steps: - id: check + env: + COMMIT_MESSAGE: ${{ github.event.head_commit.message }} run: | if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "should_release=true" >> $GITHUB_OUTPUT elif [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/master" ]]; then - if echo "${{ github.event.head_commit.message }}" | grep -q '\[no release\]'; then + if echo "$COMMIT_MESSAGE" | grep -q '\[no release\]'; then echo "should_release=false" >> $GITHUB_OUTPUT else echo "should_release=true" >> $GITHUB_OUTPUT From 6ec59ddaeab5eb099bcd7c9189f4b9266123bc07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Thu, 18 Jun 2026 09:57:59 +0200 Subject: [PATCH 035/292] app : enable self-update only when built with llama-install.sh (#24754) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- app/llama.cpp | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/app/llama.cpp b/app/llama.cpp index 30b09f9ef..c4578ea53 100644 --- a/app/llama.cpp +++ b/app/llama.cpp @@ -20,16 +20,21 @@ int llama_fit_params(int argc, char ** argv); int llama_quantize(int argc, char ** argv); int llama_perplexity(int argc, char ** argv); -// hands the update over to the install script, which downloads and swaps the binary +// Self-update is only supported for binaries built with llama-install.sh static int llama_update(int argc, char ** argv) { (void) argc; (void) argv; +#ifdef LLAMA_INSTALL_BUILD #if defined(_WIN32) return system("powershell -NoProfile -ExecutionPolicy Bypass -Command \"irm https://llama.app/install.ps1 | iex\""); #else return system("curl -fsSL https://llama.app/install.sh | sh"); #endif +#else + printf("Updates are available only when installed from https://llama.app\n"); + return 1; +#endif } static const char * progname; @@ -46,21 +51,29 @@ struct command { int (*func)(int, char **); }; +#ifdef LLAMA_INSTALL_BUILD +#define UPDATE_HIDDEN false +#else +#define UPDATE_HIDDEN true +#endif + static const command cmds[] = { - {"serve", "HTTP API server", {"server"}, false, llama_server }, - {"cli", "Command-line interactive interface", {"client"}, false, llama_cli }, - {"update", "Update llama to the latest release", {}, false, llama_update }, - {"completion", "Text completion", {"complete"}, true, llama_completion }, - {"bench", "Benchmark prompt processing and text generation", {}, true, llama_bench }, - {"batched-bench", "Benchmark batched decoding performance", {}, true, llama_batched_bench}, - {"fit-params", "Compute parameters to fit a model in device memory", {}, true, llama_fit_params }, - {"quantize", "Quantize a model", {}, true, llama_quantize }, - {"perplexity", "Compute model perplexity and KL divergence", {}, true, llama_perplexity }, - {"version", "Show version", {}, false, version }, - {"licenses", "Show third-party licenses", {"credits"}, false, licenses }, - {"help", "Show available commands", {}, false, help }, + {"serve", "HTTP API server", {"server"}, false, llama_server }, + {"cli", "Command-line interactive interface", {"client"}, false, llama_cli }, + {"update", "Update llama to the latest release", {}, UPDATE_HIDDEN, llama_update }, + {"completion", "Text completion", {"complete"}, true, llama_completion }, + {"bench", "Benchmark prompt processing and text generation", {}, true, llama_bench }, + {"batched-bench", "Benchmark batched decoding performance", {}, true, llama_batched_bench}, + {"fit-params", "Compute parameters to fit a model in device memory", {}, true, llama_fit_params }, + {"quantize", "Quantize a model", {}, true, llama_quantize }, + {"perplexity", "Compute model perplexity and KL divergence", {}, true, llama_perplexity }, + {"version", "Show version", {}, false, version }, + {"licenses", "Show third-party licenses", {"credits"}, false, licenses }, + {"help", "Show available commands", {}, false, help }, }; +#undef UPDATE_HIDDEN + static int version(int argc, char ** argv) { printf("%s\n", llama_build_info()); return 0; From dd69db292465ef698def69499ccb920a67bd613e Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Thu, 18 Jun 2026 16:17:37 +0800 Subject: [PATCH 036/292] sycl : support MUL_MAT and OUT_PROD with Q1_0 (#24721) --- ggml/src/ggml-sycl/convert.cpp | 6 +++ ggml/src/ggml-sycl/dequantize.hpp | 15 +++++++ ggml/src/ggml-sycl/dmmv.cpp | 53 ++++++++++++++++++++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 28 ++++++++---- ggml/src/ggml-sycl/mmvq.cpp | 74 +++++++++++++++++++++++++++++++ ggml/src/ggml-sycl/outprod.cpp | 54 ++++++++++++++++++---- ggml/src/ggml-sycl/vecdotq.hpp | 35 +++++++++++++++ 7 files changed, 247 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index 65593402e..060d0aca2 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -642,6 +642,8 @@ static void convert_unary_sycl(const void * vx, dst_t * y, const int64_t k, dpct to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { switch (type) { + case GGML_TYPE_Q1_0: + return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -724,6 +726,8 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { switch (type) { + case GGML_TYPE_Q1_0: + return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -830,6 +834,8 @@ to_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type) { case GGML_TYPE_BF16: return convert_unary_nc_sycl; #endif + case GGML_TYPE_Q1_0: + return dequantize_block_nc_sycl; case GGML_TYPE_Q4_0: return dequantize_block_nc_sycl; case GGML_TYPE_Q4_1: diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index ca8cd96c0..7b66c73b0 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -70,6 +70,21 @@ static __dpct_inline__ void dequantize_q4_0_reorder(const void *d_ptr, const int #endif // GGML_SYCL_F16 } +static __dpct_inline__ void dequantize_q1_0_reorder(const void *d_ptr, const int64_t ib, const void *qs, + const int iqs, dfloat2 &v) { + // Q1_0 reorder layout: scale values followed by quantized bits + const dfloat d = (const dfloat)*((const sycl::half*)d_ptr+ib); + + const int bit_index_0 = iqs + 0; + const int bit_index_1 = iqs + 1; + + const int bit_0 = (*((const uint8_t *)qs + bit_index_0 / 8) >> (bit_index_0 % 8)) & 1; + const int bit_1 = (*((const uint8_t *)qs + bit_index_1 / 8) >> (bit_index_1 % 8)) & 1; + + v.x() = (2 * bit_0 - 1) * d; + v.y() = (2 * bit_1 - 1) * d; +} + static __dpct_inline__ void dequantize_q4_1(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_1 * x = (const block_q4_1 *) vx; diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index e091e5235..fb8a1757f 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -1423,6 +1423,50 @@ static void dequantize_mul_mat_vec_q4_0_sycl(const void *vx, const dfloat *y, } } +static void dequantize_mul_mat_vec_q1_0_sycl_reorder(const void *vx, const dfloat *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec_reorder( + vx, y, dst, ncols, nrows, item_ct1); + }); + } +} + +static void dequantize_mul_mat_vec_q1_0_sycl(const void *vx, const dfloat *y, + float *dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % GGML_SYCL_DMMV_X == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + // the number of rows may exceed maximum grid size in the y or z dimensions, use the x dimension instead + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + { + dpct::has_capability_or_fail(stream->get_device(), + {sycl::aspect::fp16}); + + stream->parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + dequantize_mul_mat_vec( + vx, y, dst, ncols, nrows, item_ct1); + }); + } +} + static void dequantize_mul_mat_vec_q4_1_sycl(const void *vx, const dfloat *y, float *dst, const int ncols, const int nrows, @@ -1759,6 +1803,7 @@ void ggml_sycl_op_dequantize_mul_mat_vec( sycl::half *src1_dfloat = nullptr; // dfloat == half bool src1_convert_f16 = + src0->type == GGML_TYPE_Q1_0 || src0->type == GGML_TYPE_Q4_0 || src0->type == GGML_TYPE_Q4_1 || src0->type == GGML_TYPE_Q5_0 || src0->type == GGML_TYPE_Q5_1 || src0->type == GGML_TYPE_Q8_0 || src0->type == GGML_TYPE_F16 || @@ -1777,6 +1822,14 @@ void ggml_sycl_op_dequantize_mul_mat_vec( #endif // GGML_SYCL_F16 switch (src0->type) { + case GGML_TYPE_Q1_0: + if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && + ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { + dequantize_mul_mat_vec_q1_0_sycl_reorder(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } else { + dequantize_mul_mat_vec_q1_0_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } + break; case GGML_TYPE_Q4_0: if ((ggml_tensor_extra_gpu*)dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 77d145890..0aebdc441 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -976,6 +976,7 @@ static int64_t get_row_rounding(ggml_type type, const std::array= VER_GEN9 ? 128 : 64; @@ -3507,6 +3508,7 @@ inline bool ggml_sycl_supports_mmq(enum ggml_type type) { inline bool ggml_sycl_supports_reorder_mul_mat_sycl(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: return true; @@ -3522,6 +3524,7 @@ inline bool ggml_sycl_supports_reorder_mul_mat_sycl(enum ggml_type type) { inline bool ggml_sycl_supports_reorder_dmmv(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: return true; @@ -3532,6 +3535,7 @@ inline bool ggml_sycl_supports_reorder_dmmv(enum ggml_type type) { inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: case GGML_TYPE_Q3_K: @@ -3546,6 +3550,7 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { static bool ggml_sycl_supports_dmmv(enum ggml_type type) { switch (type) { + case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: @@ -5385,7 +5390,7 @@ static ggml_backend_buffer_t ggml_backend_sycl_device_buffer_from_host_ptr(ggml_ return nullptr; } -static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { +static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { ggml_backend_sycl_device_context *sycl_ctx = (ggml_backend_sycl_device_context *)dev->context; int device = sycl_ctx->device; @@ -5450,19 +5455,12 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g struct ggml_tensor * a = op->src[0]; struct ggml_tensor * b = op->src[1]; - // disable Q1_0 until implementation - if (a->type == GGML_TYPE_Q1_0 || b->type == GGML_TYPE_Q1_0) { - return false; - } - if (a->ne[3] != b->ne[3]) { return false; } ggml_type src0_type = op->src[0]->type; - - // TODO: The configuration below needs more work to be supported with oneDNN if (ggml_is_permuted(a) && !ggml_is_contiguous(a) && a->ne[2] > 1 && a->ne[3] > 1 && src0_type == GGML_TYPE_F16) { @@ -5472,12 +5470,17 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g // TODO: This specific configuration can fail with oneDNN and needs more debugging if (!ggml_is_permuted(a) && ggml_is_permuted(b) && b->ne[2] > 1 && b->ne[3] > 1 && a->ne[0] > 128 && a->ne[2] == 1 && src0_type == GGML_TYPE_F16) { + printf("zjy 2\n"); return false; } return true; } case GGML_OP_OUT_PROD: - return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && op->src[1]->type == GGML_TYPE_F32 && op->ne[2] == 1 && op->ne[3] == 1; + return op->type == GGML_TYPE_F32 && + (op->src[0]->type == GGML_TYPE_F32 || + (op->src[0]->type == GGML_TYPE_Q1_0 && op->src[0]->ne[2] == op->src[1]->ne[2] && + op->src[0]->ne[3] == op->src[1]->ne[3])) && + op->src[1]->type == GGML_TYPE_F32; case GGML_OP_GET_ROWS: { switch (op->src[0]->type) { @@ -5734,6 +5737,13 @@ static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const g GGML_UNUSED(dev); } +static bool ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor * op) { + bool res = do_ggml_backend_sycl_device_supports_op(dev, op); + GGML_SYCL_DEBUG("[SYCL] call %s op->op=%s op->type=%s -> %s\n", __func__, ggml_op_name(op->op), + ggml_type_name(op->type), res ? "true" : "false"); + return res; +} + static bool ggml_backend_sycl_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { if (buft->iface.get_name != ggml_backend_sycl_buffer_type_get_name) { return false; diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 909c7aeb2..7b1b3d467 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -1194,6 +1194,66 @@ static void mul_mat_vec_q8_0_q8_1_sycl_switch_ncols( } } +static void mul_mat_vec_q1_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK1_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + }); + }); +} + +template +static void mul_mat_vec_q1_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK1_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_q1_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: mul_mat_vec_q1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: mul_mat_vec_q1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 3: mul_mat_vec_q1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 4: mul_mat_vec_q1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 5: mul_mat_vec_q1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 6: mul_mat_vec_q1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 7: mul_mat_vec_q1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 8: mul_mat_vec_q1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for Q1_0 multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2120,6 +2180,20 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_q8_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; + case GGML_TYPE_Q1_0: + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y = src1_padded_col_size / QK8_1; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling mul_mat_vec_q1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + mul_mat_vec_q1_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y, stride_col_dst, stream); + return; + } else if (i == 0 || src1_ncols == 1) { + GGML_SYCL_DEBUG("Calling mul_mat_vec_q1_0_q8_1_sycl\n"); + mul_mat_vec_q1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + break; case GGML_TYPE_Q2_K: if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; diff --git a/ggml/src/ggml-sycl/outprod.cpp b/ggml/src/ggml-sycl/outprod.cpp index f52b11f0d..8d10dad8c 100644 --- a/ggml/src/ggml-sycl/outprod.cpp +++ b/ggml/src/ggml-sycl/outprod.cpp @@ -1,11 +1,12 @@ #include "outprod.hpp" +#include "convert.hpp" void ggml_sycl_op_out_prod(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); const ggml_tensor *src0 = dst->src[0]; const ggml_tensor *src1 = dst->src[1]; - GGML_ASSERT(src0->type == GGML_TYPE_F32); + GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_Q1_0); GGML_ASSERT(src1->type == GGML_TYPE_F32); GGML_ASSERT(dst->type == GGML_TYPE_F32); GGML_ASSERT(ggml_is_contiguous(src0)); @@ -20,11 +21,31 @@ void ggml_sycl_op_out_prod(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { GGML_ASSERT(ne01 == ne11); // Inner dimensions must match GGML_ASSERT(ne0 == ne00); // Output rows match src0 rows GGML_ASSERT(ne1 == ne10); // Output cols match src1 cols + GGML_ASSERT(ne2 == ne12); + GGML_ASSERT(ne3 == ne13); + GGML_ASSERT(ne2 % ne02 == 0); + GGML_ASSERT(ne3 % ne03 == 0); // Get data pointers - const float* src0_d = (const float*)src0->data; - const float* src1_d = (const float*)src1->data; - float* dst_d = (float*)dst->data; + const float * src0_d = (const float *) src0->data; + const float * src1_d = (const float *) src1->data; + float * dst_d = (float *) dst->data; + + ggml_sycl_pool_alloc src0_as_f32(ctx.pool()); + int64_t src0_nb02 = nb02; + int64_t src0_nb03 = nb03; + if (src0->type == GGML_TYPE_Q1_0) { + scope_op_debug_print scope_dbg_print(__func__, "/to_fp32_sycl", dst, /*num_src=*/2, + " : converting src0 Q1_0 to fp32"); + src0_d = src0_as_f32.alloc(ne00 * ne01 * ne02 * ne03); + const to_fp32_sycl_t to_fp32_sycl = ggml_get_to_fp32_sycl(src0->type, dst); + GGML_ASSERT(to_fp32_sycl != nullptr); + to_fp32_sycl(src0->data, const_cast(src0_d), ne00 * ne01 * ne02 * ne03, stream); + + // Dequantized src0 buffer is contiguous fp32 [ne00, ne01, ne02, ne03]. + src0_nb02 = ne00 * ne01 * (int64_t) sizeof(float); + src0_nb03 = ne00 * ne01 * ne02 * (int64_t) sizeof(float); + } // GEMM parameters const float alpha = 1.0f; @@ -35,12 +56,27 @@ void ggml_sycl_op_out_prod(ggml_backend_sycl_context& ctx, ggml_tensor* dst) { const oneapi::mkl::transpose src1_op = src1_T ? oneapi::mkl::transpose::nontrans : oneapi::mkl::transpose::trans; const int64_t ldb = (src1_T ? nb10 : nb11) / sizeof(float); + const int64_t r2 = ne2 / ne02; + const int64_t r3 = ne3 / ne03; + try { - // Perform matrix multiplication using oneMKL GEMM - oneapi::mkl::blas::column_major::gemm(*stream, oneapi::mkl::transpose::nontrans, src1_op, - ne0, ne1, ne01, alpha, src0_d, ne00, src1_d, ldb, beta, dst_d, ne0); - } - catch (sycl::exception const& exc) { + // OUT_PROD applies independently to each (i2, i3) destination plane. + for (int64_t i3 = 0; i3 < ne3; ++i3) { + for (int64_t i2 = 0; i2 < ne2; ++i2) { + const int64_t i03 = i3 / r3; + const int64_t i02 = i2 / r2; + + const float * src0_plane = (const float *) ((const char *) src0_d + i02 * src0_nb02 + i03 * src0_nb03); + const float * src1_plane = (const float *) ((const char *) src1_d + i2 * nb12 + i3 * nb13); + float * dst_plane = (float *) ((char *) dst_d + i2 * nb2 + i3 * nb3); + + // Perform matrix multiplication using oneMKL GEMM + oneapi::mkl::blas::column_major::gemm(*stream, oneapi::mkl::transpose::nontrans, src1_op, + ne0, ne1, ne01, alpha, src0_plane, ne00, + src1_plane, ldb, beta, dst_plane, ne0); + } + } + } catch (sycl::exception const& exc) { std::cerr << exc.what() << std::endl; GGML_ASSERT(false); } diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index 4b58b09ab..765fb7f15 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -309,6 +309,41 @@ vec_dot_q6_K_q8_1_impl_mmvq(const int &vl, const int &vh, vl, vh, u[0], u[1], scales[0], scales[4], d, d8[0], d8[1]); } +#define VDR_Q1_0_Q8_1_MMVQ 1 +#define VDR_Q1_0_Q8_1_MMQ 4 + +static __dpct_inline__ float +vec_dot_q1_0_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs) { + + const block_q1_0 * bq1_0 = (const block_q1_0 *) vbq; + + const block_q8_1 * bq8_1_chunk = bq8_1 + iqs; + const float d1 = bq1_0->d; + const int v = get_int_from_uint8_aligned(bq1_0->qs, iqs); + + int vi_bytes[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int shift = j * 4; + const int bits4 = (v >> shift) & 0x0F; + const int b0 = (bits4 & 0x01) ? 1 : -1; + const int b1 = (bits4 & 0x02) ? 1 : -1; + const int b2 = (bits4 & 0x04) ? 1 : -1; + const int b3 = (bits4 & 0x08) ? 1 : -1; + vi_bytes[j] = (b0 & 0xFF) | ((b1 & 0xFF) << 8) | ((b2 & 0xFF) << 16) | ((b3 & 0xFF) << 24); + } + + int sumi = 0; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int u = get_int_from_int8_aligned(bq8_1_chunk->qs, j); + sumi = ggml_sycl_dp4a(vi_bytes[j], u, sumi); + } + + return d1 * bq8_1_chunk->ds[0] * sumi; +} + // VDR = vec dot ratio, how many contiguous integers each thread processes when the vec dot kernel is called // MMVQ = mul_mat_vec_q, MMQ = mul_mat_q From 9724f664e803e70eb8d046a3fac411122ad42ff7 Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Thu, 18 Jun 2026 16:18:26 +0800 Subject: [PATCH 037/292] [SYCL] rename GGML_SYCL_SUPPORT_LEVEL_ZERO (#24719) * rename GGML_SYCL_SUPPORT_LEVEL_ZERO to GGML_SYCL_SUPPORT_LEVEL_ZERO_API, and GGML_SYCL_ENABLE_LEVEL_ZERO to GGML_SYCL_USE_LEVEL_ZERO_API * fix code format * fix error when rebase --- docs/backend/SYCL.md | 4 ++-- ggml/CMakeLists.txt | 2 +- ggml/src/ggml-sycl/CMakeLists.txt | 10 ++++----- ggml/src/ggml-sycl/common.cpp | 10 ++++----- ggml/src/ggml-sycl/common.hpp | 2 +- ggml/src/ggml-sycl/ggml-sycl.cpp | 36 +++++++++++++++---------------- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 68f6e60e6..d482d8840 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -759,7 +759,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | GGML_SYCL_GRAPH | ON *(default)* \|OFF *(Optional)* | Enable build with [SYCL Graph extension](https://github.com/intel/llvm/blob/sycl/sycl/doc/extensions/experimental/sycl_ext_oneapi_graph.asciidoc). | | GGML_SYCL_DNN | ON *(default)* \|OFF *(Optional)* | Enable build with oneDNN. | | GGML_SYCL_HOST_MEM_FALLBACK | ON *(default)* \|OFF *(Optional)* | Allow host memory fallback when device memory is full during quantized weight reorder. Enables inference to continue at reduced speed (reading over PCIe) instead of failing. Requires Linux kernel 6.8+. | -| GGML_SYCL_SUPPORT_LEVEL_ZERO | ON *(default)* \|OFF *(Optional)* | Enable Level Zero API for device memory allocation. Requires Level Zero headers/library at build time and Intel GPU driver (Level Zero runtime) at run time. Reduces system RAM usage during multi-GPU inference. | +| GGML_SYCL_SUPPORT_LEVEL_ZERO_API | ON *(default)* \|OFF *(Optional)* | Support to use Level Zero API for device memory allocation. Requires Level Zero headers/library at build time and Intel GPU driver (Level Zero runtime) at run time. Reduces system RAM usage during multi-GPU inference. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).| | CMAKE_C_COMPILER | `icx` *(Linux)*, `icx/cl` *(Windows)* | Set `icx` compiler for SYCL code path. | | CMAKE_CXX_COMPILER | `icpx` *(Linux)*, `icx` *(Windows)* | Set `icpx/icx` compiler for SYCL code path. | @@ -774,7 +774,7 @@ use 1 SYCL GPUs: [0] with Max compute units:512 | GGML_SYCL_ENABLE_FLASH_ATTN | 1 (default) or 0| Enable Flash-Attention. It can reduce memory usage. The performance impact depends on the LLM.| | GGML_SYCL_DISABLE_OPT | 0 (default) or 1 | Disable optimize features for Intel GPUs. (Recommended to 1 for Intel devices older than Gen 10) | | GGML_SYCL_DISABLE_GRAPH | 0 or 1 (default) | Disable running computations through SYCL Graphs feature. Disabled by default because SYCL Graph is still on development, no better performance. | -| GGML_SYCL_ENABLE_LEVEL_ZERO | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO=ON at build time. | +| GGML_SYCL_USE_LEVEL_ZERO_API | 1 (default) or 0 | Use Level Zero API for device memory allocation instead of SYCL. Reduces system RAM usage on Intel dGPUs by avoiding DMA-buf/TTM host memory staging. Requires GGML_SYCL_SUPPORT_LEVEL_ZERO_API=ON at build time. SYCL backend always runs on Level Zero running time even if it's set as OFF (The SYCL api will be usage for memory allocation).| | GGML_SYCL_DISABLE_DNN | 0 (default) or 1 | Disable running computations through oneDNN and always use oneMKL. | | GGML_SYCL_ENABLE_VMM | 0 or 1 (default) | Enable the virtual-memory device pool. | | ZES_ENABLE_SYSMAN | 0 (default) or 1 | Support to get free memory of GPU by sycl::aspect::ext_intel_free_memory.
Recommended to use when --split-mode = layer | diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 249ed3da2..0507e0c5a 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -249,7 +249,7 @@ option(GGML_SYCL "ggml: use SYCL" option(GGML_SYCL_F16 "ggml: use 16 bit floats for sycl calculations" OFF) option(GGML_SYCL_GRAPH "ggml: enable graphs in the SYCL backend" ON) option(GGML_SYCL_HOST_MEM_FALLBACK "ggml: allow host memory fallback in SYCL reorder (requires kernel 6.8+)" ON) -option(GGML_SYCL_SUPPORT_LEVEL_ZERO "ggml: use Level Zero API in SYCL backend" ON) +option(GGML_SYCL_SUPPORT_LEVEL_ZERO_API "ggml: use Level Zero API in SYCL backend" ON) option(GGML_SYCL_DNN "ggml: enable oneDNN in the SYCL backend" ON) set (GGML_SYCL_TARGET "INTEL" CACHE STRING "ggml: sycl target device") diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 180de9220..1c17d20df 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -39,8 +39,8 @@ if (WIN32) set(CMAKE_CXX_COMPILER "icx") set(CMAKE_CXX_COMPILER_ID "IntelLLVM") endif() - # Level Zero SDK path for Windows (only when GGML_SYCL_SUPPORT_LEVEL_ZERO is enabled) - if(GGML_SYCL_SUPPORT_LEVEL_ZERO) + # Level Zero SDK path for Windows (only when GGML_SYCL_SUPPORT_LEVEL_ZERO_API is enabled) + if(GGML_SYCL_SUPPORT_LEVEL_ZERO_API) if(DEFINED ENV{LEVEL_ZERO_V1_SDK_PATH}) set(LEVEL_ZERO_V1_SDK_PATH $ENV{LEVEL_ZERO_V1_SDK_PATH}) if(EXISTS "${LEVEL_ZERO_V1_SDK_PATH}") @@ -105,8 +105,8 @@ endif() target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") -message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO ${GGML_SYCL_SUPPORT_LEVEL_ZERO}") -if (GGML_SYCL_SUPPORT_LEVEL_ZERO) +message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}") +if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) # Link against Level Zero loader for direct device memory allocation. # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging # in the xe kernel driver during multi-GPU inference. @@ -114,7 +114,7 @@ if (GGML_SYCL_SUPPORT_LEVEL_ZERO) find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) if(ZE_LOADER_LIB AND LEVEL_ZERO_INCLUDE_DIR) target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) - target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO) + target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") message(STATUS "Level Zero headers found: ${LEVEL_ZERO_INCLUDE_DIR}") else() diff --git a/ggml/src/ggml-sycl/common.cpp b/ggml/src/ggml-sycl/common.cpp index 38ace8bf5..e1b6db13e 100644 --- a/ggml/src/ggml-sycl/common.cpp +++ b/ggml/src/ggml-sycl/common.cpp @@ -12,7 +12,7 @@ #include "common.hpp" #include -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API #include #endif @@ -84,9 +84,9 @@ int64_t downsample_sycl_global_range(int64_t accumulate_block_num, int64_t block return sycl_down_blk_size; } -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) { - return g_ggml_sycl_enable_level_zero && + return g_ggml_sycl_use_level_zero_api && q.get_device().is_gpu() && q.get_backend() == sycl::backend::ext_oneapi_level_zero; } @@ -95,7 +95,7 @@ static bool ggml_sycl_use_level_zero_device_alloc(sycl::queue &q) { // Use Level Zero zeMemAllocDevice to avoid sycl::malloc_device triggering // DMA-buf/TTM system RAM staging in the xe kernel driver during multi-GPU inference. void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) { -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { void *ptr = nullptr; auto ze_ctx = sycl::get_native(q.get_context()); @@ -127,7 +127,7 @@ void * ggml_sycl_malloc_device(size_t size, sycl::queue &q) { void ggml_sycl_free_device(void *ptr, sycl::queue &q) { if (!ptr) return; -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (ggml_sycl_use_level_zero_device_alloc(q)) { auto ze_ctx = sycl::get_native(q.get_context()); zeMemFree(ze_ctx, ptr); diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index c87a4636e..8534bd358 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -324,7 +324,7 @@ struct ggml_tensor_extra_gpu { optimize_feature optimized_feature; }; -extern int g_ggml_sycl_enable_level_zero; +extern int g_ggml_sycl_use_level_zero_api; void * ggml_sycl_malloc_device(size_t size, sycl::queue &q); void ggml_sycl_free_device(void *ptr, sycl::queue &q); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 0aebdc441..d8b83d0e2 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -32,7 +32,7 @@ #include #include -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API #include #endif #if defined(GGML_SYCL_GRAPH) && SYCL_EXT_ONEAPI_ASYNC_MEMORY_ALLOC @@ -87,7 +87,7 @@ int g_ggml_sycl_enable_vmm = 1; int g_ggml_sycl_prioritize_dmmv = 0; int g_ggml_sycl_use_async_mem_op = 0; int g_ggml_sycl_use_async_mem_op_requested = 1; -int g_ggml_sycl_enable_level_zero = 0; +int g_ggml_sycl_use_level_zero_api = 0; int g_ggml_sycl_enable_flash_attention = 1; int g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; int g_ggml_sycl_usm_system = 0; @@ -157,7 +157,7 @@ static ggml_sycl_device_info ggml_sycl_init() { info.ext_oneapi_level_zero = false; } -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (info.ext_oneapi_level_zero && device.is_gpu() && device.default_queue().get_backend() == sycl::backend::ext_oneapi_level_zero) { ze_device_handle_t ze_dev = sycl::get_native(device.default_queue().get_device()); ze_device_properties_t props = {}; @@ -172,13 +172,13 @@ static ggml_sycl_device_info ggml_sycl_init() { info.default_tensor_split[id] /= total_vram; } -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API // Large buffers can be allocated before ggml_check_sycl() initializes other // g_ggml_sycl_enable_* globals, so initialize this one as early as we can. - g_ggml_sycl_enable_level_zero = - info.ext_oneapi_level_zero && ggml_sycl_get_env("GGML_SYCL_ENABLE_LEVEL_ZERO", 1); + g_ggml_sycl_use_level_zero_api = + info.ext_oneapi_level_zero && ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); #else - g_ggml_sycl_enable_level_zero = 0; + g_ggml_sycl_use_level_zero_api = 0; #endif return info; @@ -277,7 +277,7 @@ static void ggml_check_sycl() try { g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); g_ggml_sycl_dev2dev_memcpy = ggml_sycl_get_env("GGML_SYCL_DEV2DEV_MEMCPY", DEV2DEV_MEMCPY_SYCL); - if (g_ggml_sycl_enable_level_zero == 0) { + if (g_ggml_sycl_use_level_zero_api == 0) { g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; } @@ -312,10 +312,10 @@ static void ggml_check_sycl() try { #else GGML_LOG_INFO(" GGML_SYCL_DNNL: no\n"); #endif -#if defined(GGML_SYCL_SUPPORT_LEVEL_ZERO) - GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO: yes\n"); +#if defined(GGML_SYCL_SUPPORT_LEVEL_ZERO_API) + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO_API: yes\n"); #else - GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO: no\n"); + GGML_LOG_INFO(" GGML_SYCL_SUPPORT_LEVEL_ZERO_API: no\n"); #endif #if defined(GGML_SYCL_USE_VMM) GGML_LOG_INFO(" GGML_SYCL_USE_VMM: yes\n"); @@ -331,12 +331,12 @@ static void ggml_check_sycl() try { #else GGML_LOG_INFO(" GGML_SYCL_DISABLE_GRAPH: graph disabled by compile flag\n"); #endif -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO - GGML_LOG_INFO(" GGML_SYCL_ENABLE_LEVEL_ZERO: %d\n", g_ggml_sycl_enable_level_zero); +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: %d\n", g_ggml_sycl_use_level_zero_api); GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d\n", g_ggml_sycl_dev2dev_memcpy); #else - GGML_LOG_INFO(" GGML_SYCL_ENABLE_LEVEL_ZERO: Level Zero disabled by compile flag\n"); - GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d, enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO\n", + GGML_LOG_INFO(" GGML_SYCL_USE_LEVEL_ZERO_API: Disable Level Zero API usage by compile flag\n"); + GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d, enable to SYCL API since missing GGML_SYCL_SUPPORT_LEVEL_ZERO_API\n", g_ggml_sycl_dev2dev_memcpy); #endif #if GGML_SYCL_DNNL @@ -602,7 +602,7 @@ catch (sycl::exception const &exc) { std::exit(1); } -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API static bool ggml_sycl_is_l0_discrete_gpu(int device) { return ggml_sycl_info().devices[device].l0_discrete_gpu; } @@ -611,12 +611,12 @@ static bool ggml_sycl_is_l0_discrete_gpu(int device) { static void dev2dev_memcpy(int device_dst, sycl::queue &q_dst, int device_src, sycl::queue &q_src, void *ptr_dst, const void *ptr_src, size_t size) { -#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API if (g_ggml_sycl_dev2dev_memcpy == DEV2DEV_MEMCPY_L0) { // Use Level Zero direct copy for dGPU-to-dGPU transfers. const bool l0_copy_supported = ggml_sycl_is_l0_discrete_gpu(device_dst) && ggml_sycl_is_l0_discrete_gpu(device_src); - if (g_ggml_sycl_enable_level_zero && l0_copy_supported) { + if (g_ggml_sycl_use_level_zero_api && l0_copy_supported) { auto ze_ctx = sycl::get_native(q_dst.get_context()); auto ze_dev = sycl::get_native(q_dst.get_device()); ze_command_queue_desc_t cq_desc = {ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, nullptr, 0, 0, From 24bba7b98ea1544cc89352c7a573baedcb831a64 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 18 Jun 2026 12:04:39 +0200 Subject: [PATCH 038/292] mtmd: refactor preprocessor, add mtmd_image_preproc_out (#24736) * add mtmd_image_preproc_out * add dev docs * remove unused clip API * rm unused clip_image_f32_batch::grid * change preprocess() call signature --- docs/multimodal.md | 5 +- tools/mtmd/README-dev.md | 35 +++++++ tools/mtmd/clip-impl.h | 133 ++++++++++---------------- tools/mtmd/clip.cpp | 119 +++++------------------ tools/mtmd/clip.h | 31 +----- tools/mtmd/mtmd-image.cpp | 195 +++++++++++++++++--------------------- tools/mtmd/mtmd-image.h | 39 ++++---- tools/mtmd/mtmd.cpp | 92 +++++++++--------- 8 files changed, 279 insertions(+), 370 deletions(-) create mode 100644 tools/mtmd/README-dev.md diff --git a/docs/multimodal.md b/docs/multimodal.md index 33d1df33c..76065aac4 100644 --- a/docs/multimodal.md +++ b/docs/multimodal.md @@ -1,10 +1,11 @@ # Multimodal llama.cpp supports multimodal input via `libmtmd`. Currently, there are 2 tools support this feature: -- [llama-mtmd-cli](../tools/mtmd/README.md) +- [llama-cli](../tools/cli/README.md) - [llama-server](../tools/server/README.md) via OpenAI-compatible `/chat/completions` API +- [llama-mtmd-cli](../tools/mtmd/README.md), for testing and development -Currently, we support **image** and **audio** input. Audio is highly experimental and may have reduced quality. +Currently, we support **image**, **audio** and **video** input. To enable it, you can use one of the 2 methods below: diff --git a/tools/mtmd/README-dev.md b/tools/mtmd/README-dev.md new file mode 100644 index 000000000..3a0891587 --- /dev/null +++ b/tools/mtmd/README-dev.md @@ -0,0 +1,35 @@ +# libmtmd dev guide + +## History + +Please refer to [multimodal.md](../../docs/multimodal.md) for a broader context. + +In short: +- `libmtmd` started as a wrapper around `libllava` / `clip.cpp` +- Various components that used to be in `clip.cpp` are moved progressively to mtmd. For example, preprocessor is now part of mtmd + +## Terminologies + +- mtmd: **M**ul**T**i**M**o**D**al +- bitmap: representing a raw input data, for example: RGB image, PCM audio +- tiles / slices: for llava-uhd-style models, the preprocessor breaks a large input into smaller square images called tiles or slices +- chunk: a mtmd_input_chunk represents a preprocessed input that can then be passed through `mtmd_encode()` + +## Pipeline + +A typical pipeline of the core libmtmd is as follows: +- A bitmap (RGB image or PCM audio) is created +- Bitmap and the text prompt is provided to `mtmd_tokenize()` that breaks the input into chunks + - The tokenizer function first expands a "lazy" bitmap if it finds one. Typically, this is used by video, so that one media token corresponds to one input bitmap + - For models that support "fused" temporal frames like Qwen-VL, the tokenizer tries to merge pair of consecutive frames into one batch + - The preprocessor will then be called, which produces a list of chunks + - Depending on the model itself, special tokens will be injected to separate image chunks (i.e. llava-uhd-style models) +- Multiple bitmaps may be batched together to form a larger `mtmd_batch()` +- Single image or batch is encoded, via `mtmd_encode()` or `mtmd_batch_encode()` +- Get the output embeddings + +## Helper + +We provide a set of helper functions via `mtmd_helper` to make using libmtmd easier. The helper provides: +- Image, audio and video file decoding (for example, decode raw JPEG into RGB bitmap) +- Manage `llama_batch` and calls to `llama_decode` diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index b104f3736..f232b68e5 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -367,56 +367,56 @@ enum projector_type { }; static std::map PROJECTOR_TYPE_NAMES = { - { PROJECTOR_TYPE_MLP, "mlp" }, - { PROJECTOR_TYPE_LDP, "ldp" }, - { PROJECTOR_TYPE_LDPV2, "ldpv2"}, - { PROJECTOR_TYPE_MINICPMV, "resampler"}, - { PROJECTOR_TYPE_GLM_EDGE, "adapter"}, - { PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"}, - { PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"}, - { PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"}, - { PROJECTOR_TYPE_STEP3VL, "step3vl"}, - { PROJECTOR_TYPE_GEMMA3, "gemma3"}, - { PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"}, - { PROJECTOR_TYPE_GEMMA3NA, "gemma3na"}, - { PROJECTOR_TYPE_GEMMA4V, "gemma4v"}, - { PROJECTOR_TYPE_GEMMA4A, "gemma4a"}, - { PROJECTOR_TYPE_GEMMA4UV, "gemma4uv"}, - { PROJECTOR_TYPE_GEMMA4UA, "gemma4ua"}, - { PROJECTOR_TYPE_PHI4, "phi4"}, - { PROJECTOR_TYPE_IDEFICS3, "idefics3"}, - { PROJECTOR_TYPE_PIXTRAL, "pixtral"}, - { PROJECTOR_TYPE_ULTRAVOX, "ultravox"}, - { PROJECTOR_TYPE_INTERNVL, "internvl"}, - { PROJECTOR_TYPE_LLAMA4, "llama4"}, - { PROJECTOR_TYPE_QWEN2A, "qwen2a"}, - { PROJECTOR_TYPE_QWEN3A, "qwen3a"}, - { PROJECTOR_TYPE_GLMA, "glma"}, - { PROJECTOR_TYPE_QWEN25O, "qwen2.5o"}, - { PROJECTOR_TYPE_VOXTRAL, "voxtral"}, - { PROJECTOR_TYPE_MERALION, "meralion"}, - { PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"}, - { PROJECTOR_TYPE_LFM2, "lfm2"}, - { PROJECTOR_TYPE_KIMIVL, "kimivl"}, - { PROJECTOR_TYPE_PADDLEOCR, "paddleocr"}, - { PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"}, - { PROJECTOR_TYPE_COGVLM, "cogvlm"}, - { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"}, - { PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"}, - { PROJECTOR_TYPE_DEEPSEEKOCR,"deepseekocr"}, - { PROJECTOR_TYPE_DEEPSEEKOCR2,"deepseekocr2"}, - { PROJECTOR_TYPE_LFM2A, "lfm2a"}, - { PROJECTOR_TYPE_GLM4V, "glm4v"}, - { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, - { PROJECTOR_TYPE_YASA2, "yasa2"}, - { PROJECTOR_TYPE_KIMIK25, "kimik25"}, - { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, - { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, - { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, - { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, - { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, - { PROJECTOR_TYPE_MIMOVL, "mimovl"}, - { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, + { PROJECTOR_TYPE_MLP, "mlp" }, + { PROJECTOR_TYPE_LDP, "ldp" }, + { PROJECTOR_TYPE_LDPV2, "ldpv2"}, + { PROJECTOR_TYPE_MINICPMV, "resampler"}, + { PROJECTOR_TYPE_GLM_EDGE, "adapter"}, + { PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"}, + { PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"}, + { PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"}, + { PROJECTOR_TYPE_STEP3VL, "step3vl"}, + { PROJECTOR_TYPE_GEMMA3, "gemma3"}, + { PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"}, + { PROJECTOR_TYPE_GEMMA3NA, "gemma3na"}, + { PROJECTOR_TYPE_GEMMA4V, "gemma4v"}, + { PROJECTOR_TYPE_GEMMA4A, "gemma4a"}, + { PROJECTOR_TYPE_GEMMA4UV, "gemma4uv"}, + { PROJECTOR_TYPE_GEMMA4UA, "gemma4ua"}, + { PROJECTOR_TYPE_PHI4, "phi4"}, + { PROJECTOR_TYPE_IDEFICS3, "idefics3"}, + { PROJECTOR_TYPE_PIXTRAL, "pixtral"}, + { PROJECTOR_TYPE_ULTRAVOX, "ultravox"}, + { PROJECTOR_TYPE_INTERNVL, "internvl"}, + { PROJECTOR_TYPE_LLAMA4, "llama4"}, + { PROJECTOR_TYPE_QWEN2A, "qwen2a"}, + { PROJECTOR_TYPE_QWEN3A, "qwen3a"}, + { PROJECTOR_TYPE_GLMA, "glma"}, + { PROJECTOR_TYPE_QWEN25O, "qwen2.5o"}, + { PROJECTOR_TYPE_VOXTRAL, "voxtral"}, + { PROJECTOR_TYPE_MERALION, "meralion"}, + { PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"}, + { PROJECTOR_TYPE_LFM2, "lfm2"}, + { PROJECTOR_TYPE_KIMIVL, "kimivl"}, + { PROJECTOR_TYPE_PADDLEOCR, "paddleocr"}, + { PROJECTOR_TYPE_LIGHTONOCR, "lightonocr"}, + { PROJECTOR_TYPE_COGVLM, "cogvlm"}, + { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"}, + { PROJECTOR_TYPE_DOTS_OCR, "dots_ocr"}, + { PROJECTOR_TYPE_DEEPSEEKOCR, "deepseekocr"}, + { PROJECTOR_TYPE_DEEPSEEKOCR2, "deepseekocr2"}, + { PROJECTOR_TYPE_LFM2A, "lfm2a"}, + { PROJECTOR_TYPE_GLM4V, "glm4v"}, + { PROJECTOR_TYPE_YOUTUVL, "youtuvl"}, + { PROJECTOR_TYPE_YASA2, "yasa2"}, + { PROJECTOR_TYPE_KIMIK25, "kimik25"}, + { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"}, + { PROJECTOR_TYPE_EXAONE4_5, "exaone4_5"}, + { PROJECTOR_TYPE_HUNYUANVL, "hunyuanvl"}, + { PROJECTOR_TYPE_MINICPMV4_6, "minicpmv4_6"}, + { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, + { PROJECTOR_TYPE_MIMOVL, "mimovl"}, + { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { @@ -640,47 +640,18 @@ static void clip_log_internal(enum ggml_log_level level, const char * format, .. // cpp wrappers // -// wrapper for clip_image_size -struct clip_image_size_deleter { - void operator()(clip_image_size * val) { clip_image_size_free(val); } -}; -typedef std::unique_ptr clip_image_size_ptr; - -// wrapper for clip_image_u8 -struct clip_image_u8_deleter { - void operator()(clip_image_u8 * val) { clip_image_u8_free(val); } -}; -typedef std::unique_ptr clip_image_u8_ptr; - -// wrapper for clip_image_f32 -struct clip_image_f32_deleter { - void operator()(clip_image_f32 * val) { clip_image_f32_free(val); } -}; -typedef std::unique_ptr clip_image_f32_ptr; - -struct clip_image_u8_batch { - std::vector entries; -}; - struct clip_image_f32_batch { - std::vector entries; + std::vector entries; bool is_audio = false; - // for llava-uhd style models, we need to know the grid size - // note: entries.size() == grid_x * grid_y + 1 (one overview image) - int grid_x = 0; - int grid_y = 0; - clip_image_f32_batch clone() const { clip_image_f32_batch new_batch{ /* entries */ {}, /* is_audio */ is_audio, - /* grid_x */ grid_x, - /* grid_y */ grid_y, }; new_batch.entries.reserve(entries.size()); for (const auto & entry : entries) { - new_batch.entries.emplace_back(new clip_image_f32(*entry)); + new_batch.entries.emplace_back(entry); // copy } return new_batch; } diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 208486fd1..dc6223295 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -865,7 +865,7 @@ ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale } static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const clip_image_f32_batch & imgs) { - const clip_image_f32 & img = *imgs.entries[0]; + const clip_image_f32 & img = imgs.entries[0]; std::unique_ptr builder; switch (ctx->proj_type()) { @@ -2825,16 +2825,16 @@ struct clip_model_loader { // create a fake batch const auto & hparams = ctx_clip.model.hparams; clip_image_f32_batch batch; - clip_image_f32_ptr img(clip_image_f32_init()); + clip_image_f32 img; if (ctx_clip.model.modality == CLIP_MODALITY_VISION) { const int sz = hparams.warmup_image_size; - img->set_size({sz, sz}, false, false); + img.set_size({sz, sz}, false, false); LOG_INF("%s: warmup with image size = %d x %d\n", __func__, sz, sz); } else { - img->set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); + img.set_size({hparams.warmup_audio_size, hparams.n_mel_bins}, false, false); LOG_INF("%s: warmup with audio size = %d\n", __func__, hparams.warmup_audio_size); } - batch.entries.push_back(std::move(img)); + batch.entries.push_back(img); return batch; } @@ -3124,64 +3124,6 @@ struct clip_cap clip_get_cap(const char * fname) { return res; } -struct clip_image_size * clip_image_size_init() { - struct clip_image_size * load_image_size = new struct clip_image_size(); - load_image_size->width = 448; - load_image_size->height = 448; - return load_image_size; -} - -struct clip_image_u8 * clip_image_u8_init() { - return new clip_image_u8(); -} - -struct clip_image_f32 * clip_image_f32_init() { - return new clip_image_f32(); -} - -struct clip_image_f32_batch * clip_image_f32_batch_init() { - return new clip_image_f32_batch(); -} - -void clip_image_size_free(struct clip_image_size * load_image_size) { - if (load_image_size == nullptr) { - return; - } - delete load_image_size; -} -void clip_image_u8_free(struct clip_image_u8 * img) { delete img; } -void clip_image_f32_free(struct clip_image_f32 * img) { delete img; } -void clip_image_u8_batch_free(struct clip_image_u8_batch * batch) { delete batch; } -void clip_image_f32_batch_free(struct clip_image_f32_batch * batch) { delete batch; } - -size_t clip_image_f32_batch_n_images(const struct clip_image_f32_batch * batch) { - return batch->entries.size(); -} - -size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int idx) { - if (idx < 0 || idx >= (int)batch->entries.size()) { - LOG_ERR("%s: invalid index %d\n", __func__, idx); - return 0; - } - return batch->entries[idx]->nx(); -} - -size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx) { - if (idx < 0 || idx >= (int)batch->entries.size()) { - LOG_ERR("%s: invalid index %d\n", __func__, idx); - return 0; - } - return batch->entries[idx]->ny(); -} - -clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx) { - if (idx < 0 || idx >= (int)batch->entries.size()) { - LOG_ERR("%s: invalid index %d\n", __func__, idx); - return nullptr; - } - return batch->entries[idx].get(); -} - void clip_free(clip_ctx * ctx) { if (ctx == nullptr) { return; @@ -3189,23 +3131,11 @@ void clip_free(clip_ctx * ctx) { delete ctx; } -int32_t clip_get_image_size(const struct clip_ctx * ctx) { - return ctx->model.hparams.image_size; -} - -int32_t clip_get_patch_size(const struct clip_ctx * ctx) { - return ctx->model.hparams.patch_size; -} - -int32_t clip_get_hidden_size(const struct clip_ctx * ctx) { - return ctx->model.hparams.n_embd; -} - const char * clip_patch_merge_type(const struct clip_ctx * ctx) { return ctx->model.hparams.mm_patch_merge_type == PATCH_MERGE_SPATIAL_UNPAD ? "spatial_unpad" : "flat"; } -int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img) { +int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) { const auto & params = ctx->model.hparams; const int n_total = clip_n_output_tokens(ctx, img); const auto & proj = ctx->proj_type(); @@ -3228,7 +3158,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * return n_total; } -int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img) { +int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) { const auto & params = ctx->model.hparams; const auto & proj = ctx->proj_type(); switch (proj) { @@ -3250,7 +3180,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * return 1; } -int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * img) { +int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { const auto & params = ctx->model.hparams; // for models with fixed size image, the input image is already pre-processed and resized to square @@ -3500,16 +3430,15 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im return n_patches; } -bool clip_image_encode(struct clip_ctx * ctx, const int n_threads, clip_image_f32 * img, std::vector & out_vec) { +bool clip_image_encode(struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec) { clip_image_f32_batch imgs; - clip_image_f32_ptr img_copy(clip_image_f32_init()); - *img_copy = *img; + clip_image_f32 img_copy = *img; imgs.entries.push_back(std::move(img_copy)); return clip_image_batch_encode(ctx, n_threads, &imgs, out_vec); } -bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { +bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32_batch * imgs_c_ptr, std::vector & out_batch_embd) { const clip_image_f32_batch & imgs = *imgs_c_ptr; int n_batch_cur = imgs.entries.size(); @@ -3533,8 +3462,8 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima const auto & model = ctx->model; const auto & hparams = model.hparams; - const int image_size_width = imgs.entries[0]->nx(); - const int image_size_height = imgs.entries[0]->ny(); + const int image_size_width = imgs.entries[0].nx(); + const int image_size_height = imgs.entries[0].ny(); const int patch_size = hparams.patch_size; const int num_patches = ((image_size_width / patch_size) * (image_size_height / patch_size)); @@ -3572,7 +3501,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima if (!imgs.is_audio) { size_t nelem = 0; for (const auto & img : imgs.entries) { - nelem += img->nx() * img->ny() * 3; + nelem += img.nx() * img.ny() * 3; } std::vector inp_raw(nelem); @@ -3590,13 +3519,13 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // IMPORTANT: [QWEN_VIDEO] the batch dim is currently used for temporal dim in Qwen-VL models // All entries must have the same spatial size (enforced by can_batch_with() during merging) { - const int nx = imgs.entries[0]->nx(); - const int ny = imgs.entries[0]->ny(); + const int nx = imgs.entries[0].nx(); + const int ny = imgs.entries[0].ny(); const int n = nx * ny; for (int b = 0; b < n_batch_cur; b++) { LOG_DBG("%s: copying image %d/%d to input buffer (nx=%d, ny=%d)\n", __func__, b+1, n_batch_cur, nx, ny); - const auto & buf = imgs.entries[b]->get_ro_buf(); + const auto & buf = imgs.entries[b].get_ro_buf(); float * batch_entry = inp_raw.data() + b * (3*n); for (int y = 0; y < ny; y++) { for (int x = 0; x < nx; x++) { @@ -3616,9 +3545,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima GGML_ASSERT(imgs.entries.size() == 1); const auto & mel_inp = imgs.entries[0]; - const auto & buf = mel_inp->get_ro_buf(); - const int n_step = mel_inp->nx(); - const int n_mel = mel_inp->ny(); + const auto & buf = mel_inp.get_ro_buf(); + const int n_step = mel_inp.nx(); + const int n_mel = mel_inp.ny(); GGML_ASSERT((size_t)n_step * n_mel == buf.size()); set_input_f32("inp_raw", buf); @@ -4232,7 +4161,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima GGML_ASSERT(imgs.entries.size() == 1); const auto & img0 = imgs.entries.front(); // Compute n_pos matching SSCP output: two stride-2 convs - int n_pos = img0->nx(); + int n_pos = img0.nx(); for (int i = 0; i < 2; i++) { n_pos = (n_pos - 1) / 2 + 1; } // Chunked local attention: blocked causal mask and RPE @@ -4280,7 +4209,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima case PROJECTOR_TYPE_LFM2A: { GGML_ASSERT(imgs.entries.size() == 1); - const auto n_frames = clip_n_output_tokens(ctx, imgs.entries.front().get()); + const auto n_frames = clip_n_output_tokens(ctx, &imgs.entries.front()); auto d_model = 512; auto seq_len = n_frames * 2 - 1; @@ -4338,7 +4267,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // reshapes as ggml_get_rows gathers. The names are set // by g4v_gather() in models/granite4-vision.cpp. const int patch_size = model.hparams.patch_size; - const int image_side = imgs.entries.front()->nx() / patch_size; + const int image_side = imgs.entries.front().nx() / patch_size; const int window_side = hparams.downsample_window_side; const int query_side = hparams.downsample_query_side; const int n = image_side / window_side; @@ -4432,7 +4361,7 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one) const int n_tokens_out = embeddings->ne[1]; - const int expected_n_tokens_out = clip_n_output_tokens(ctx, imgs.entries[0].get()); + const int expected_n_tokens_out = clip_n_output_tokens(ctx, &imgs.entries[0]); if (n_tokens_out != expected_n_tokens_out) { LOG_ERR("%s: expected output %d tokens, got %d\n", __func__, expected_n_tokens_out, n_tokens_out); GGML_ABORT("Invalid number of output tokens"); diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index 7197af856..f66f4bc3b 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -29,7 +29,6 @@ struct clip_image_size { }; struct clip_image_f32; -struct clip_image_u8_batch; struct clip_image_f32_batch; enum clip_modality { @@ -63,41 +62,21 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params void clip_free(struct clip_ctx * ctx); -int32_t clip_get_image_size (const struct clip_ctx * ctx); -int32_t clip_get_patch_size (const struct clip_ctx * ctx); -int32_t clip_get_hidden_size(const struct clip_ctx * ctx); - // TODO: should be enum, not string const char * clip_patch_merge_type(const struct clip_ctx * ctx); -int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * img); +int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img); // for M-RoPE, this will be the number of token positions in X and Y directions // for other models, X will be the total number of tokens and Y will be 1 -int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img); -int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img); +int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img); +int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img); // this should be equal to the embedding dimension of the text model int clip_n_mmproj_embd(const struct clip_ctx * ctx); -struct clip_image_size * clip_image_size_init(void); -struct clip_image_u8 * clip_image_u8_init (void); -struct clip_image_f32 * clip_image_f32_init(void); -struct clip_image_f32_batch * clip_image_f32_batch_init(void); // only used by libllava - -void clip_image_size_free (struct clip_image_size * img_size); -void clip_image_u8_free (struct clip_image_u8 * img); -void clip_image_f32_free(struct clip_image_f32 * img); -void clip_image_u8_batch_free (struct clip_image_u8_batch * batch); -void clip_image_f32_batch_free(struct clip_image_f32_batch * batch); - -// use for accessing underlay data of clip_image_f32_batch -size_t clip_image_f32_batch_n_images(const struct clip_image_f32_batch * batch); // equivalent to batch->size() -size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->nx -size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny -struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data - -bool clip_image_encode (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, std::vector & out_vec); +// TODO: remove clip_image_encode() and always use batched version +bool clip_image_encode (struct clip_ctx * ctx, int n_threads, const clip_image_f32 * img, std::vector & out_vec); bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, std::vector & out_batch_embd); bool clip_is_llava(const struct clip_ctx * ctx); diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 656d75d80..a807fefda 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -4,17 +4,26 @@ #include #include -// -// base implementation -// - -void mtmd_image_preprocessor::img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst, const float mean[3], const float std[3]) { - dst.from_u8(src); - dst.normalize(mean, std); +void mtmd_image_preproc_out::append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized) { + clip_image_f32 dst; + dst.from_u8(img); + if (normalized) { + dst.normalize(hparams.image_mean, hparams.image_std); + } + entries.push_back(std::move(dst)); } -void mtmd_image_preprocessor::img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst) { - dst.from_u8(src); +void mtmd_image_preproc_out::append(const clip_hparams & hparams, const std::vector & imgs, bool normalized) { + for (const auto & img : imgs) { + append(hparams, img, normalized); + } +} + +void mtmd_image_preproc_out::append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized) { + if (normalized) { + img.normalize(hparams.image_mean, hparams.image_std); + } + entries.push_back(std::move(img)); } // set of tools to manipulate images @@ -595,21 +604,17 @@ private: // mtmd_image_preprocessor_llava_uhd // -bool mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img) { const clip_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); - std::vector imgs = slice_image(img, inst); - - for (size_t i = 0; i < imgs.size(); ++i) { - // clip_image_save_to_bmp(*imgs[i], "slice_" + std::to_string(i) + ".bmp"); - clip_image_f32_ptr res(clip_image_f32_init()); - img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(res)); - } + std::vector imgs = slice_image(img, inst); + mtmd_image_preproc_out output; + output.append(hparams, imgs, true); output.grid_x = inst.grid_size.width; output.grid_y = inst.grid_size.height; - return true; + + return output; } mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) { @@ -717,28 +722,28 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_ll return res; } -std::vector mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst, bool overview_first) { - std::vector output; +std::vector mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst, bool overview_first) { + std::vector output; // resize to overview size - clip_image_u8_ptr resized_img(clip_image_u8_init()); - img_tool::resize(img, *resized_img, inst.overview_size, hparams.image_resize_algo_ov, + clip_image_u8 resized_img; + img_tool::resize(img, resized_img, inst.overview_size, hparams.image_resize_algo_ov, hparams.image_pad_ov, hparams.image_pad_color_ov); if (overview_first) { - output.push_back(std::move(resized_img)); + output.push_back(resized_img); } if (inst.slices.empty()) { // no slices, just return the resized image if (!overview_first) { - output.push_back(std::move(resized_img)); + output.push_back(resized_img); } return output; } // resize to refined size - clip_image_u8_ptr refined_img(clip_image_u8_init()); - img_tool::resize(img, *refined_img, inst.refined_size, hparams.image_resize_algo_rf, + clip_image_u8 refined_img; + img_tool::resize(img, refined_img, inst.refined_size, hparams.image_resize_algo_rf, hparams.image_pad_rf, hparams.image_pad_color_rf); // create slices @@ -748,13 +753,13 @@ std::vector mtmd_image_preprocessor_llava_uhd::slice_image(co int w = slice.size.width; int h = slice.size.height; - clip_image_u8_ptr img_slice(clip_image_u8_init()); - img_tool::crop(*refined_img, *img_slice, x, y, w, h); + clip_image_u8 img_slice; + img_tool::crop(refined_img, img_slice, x, y, w, h); output.push_back(std::move(img_slice)); } if (!overview_first) { - output.push_back(std::move(resized_img)); + output.push_back(resized_img); } return output; @@ -871,24 +876,23 @@ clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_s // mtmd_image_preprocessor_fixed_size // -bool mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img) { clip_image_u8 resized_image; int sz = hparams.image_size; img_tool::resize(img, resized_image, {sz, sz}, hparams.image_resize_algo, hparams.image_resize_pad, hparams.image_pad_color); - clip_image_f32_ptr img_f32(clip_image_f32_init()); - img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(img_f32)); - return true; + mtmd_image_preproc_out output; + output.append(hparams, resized_image, true); + return output; } // // mtmd_image_preprocessor_dyn_size // -bool mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) { GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0); clip_image_u8 resized_image; const clip_image_size original_size = img.get_size(); @@ -903,17 +907,16 @@ bool mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img, cli hparams.image_resize_algo, hparams.image_resize_pad, hparams.image_pad_color); - clip_image_f32_ptr img_f32(clip_image_f32_init()); - img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(img_f32)); - return true; + mtmd_image_preproc_out output; + output.append(hparams, resized_image, true); + return output; } // // mtmd_image_preprocessor_longest_edge // -bool mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img) { GGML_ASSERT(hparams.image_longest_edge > 0); clip_image_u8 resized_image; const clip_image_size original_size = img.get_size(); @@ -927,10 +930,9 @@ bool mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img, hparams.image_resize_algo, hparams.image_resize_pad, hparams.image_pad_color); - clip_image_f32_ptr img_f32(clip_image_f32_init()); - img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(img_f32)); - return true; + mtmd_image_preproc_out output; + output.append(hparams, resized_image, true); + return output; } // @@ -1040,7 +1042,7 @@ clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int wi // mtmd_image_preprocessor_idefics3 // -bool mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img) { // The refined size has two steps: // 1. Resize w/ aspect-ratio preserving such that the longer side is // the preprocessor longest size @@ -1077,44 +1079,35 @@ bool mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img, cli } auto imgs = slice_image(img, instructions); - // cast and normalize to f32 - for (size_t i = 0; i < imgs.size(); ++i) { - // clip_image_save_to_bmp(*imgs[i], "slice_" + std::to_string(i) + ".bmp"); - clip_image_f32_ptr res(clip_image_f32_init()); - img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(res)); - } - + mtmd_image_preproc_out output; + output.append(hparams, imgs, true); output.grid_x = instructions.grid_size.width; output.grid_y = instructions.grid_size.height; - return true; + return output; } // // mtmd_image_preprocessor_internvl // -bool mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img) { GGML_ASSERT(!hparams.image_res_candidates.empty()); const clip_image_size original_size = img.get_size(); auto const inst = get_slice_instructions(original_size); - std::vector imgs = slice_image(img, inst, false); + std::vector imgs = slice_image(img, inst, false); - for (size_t i = 0; i < imgs.size(); ++i) { - clip_image_f32_ptr res(clip_image_f32_init()); - img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(res)); - } + mtmd_image_preproc_out output; + output.append(hparams, imgs, true); output.grid_x = inst.grid_size.width; output.grid_y = inst.grid_size.height; - return true; + return output; } // // mtmd_image_preprocessor_deepseekocr // -bool mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img) { static constexpr int native_resolutions[] = { 1024 /* base */, 1280 /* large */ }; // TODO: support 512 (tiny) and 640 (small) once we have eval data for them @@ -1137,14 +1130,11 @@ bool mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img, clip_image_u8 padded; img_tool::resize(img, padded, {image_size, image_size}, RESIZE_ALGO_BICUBIC_PILLOW, PAD_NEAREST, hparams.image_pad_color); - - clip_image_f32_ptr res(clip_image_f32_init()); - img_u8_to_f32(padded, *res, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(res)); - + mtmd_image_preproc_out output; + output.append(hparams, padded, true); output.grid_x = 1; output.grid_y = 1; - return true; + return output; } // @@ -1207,10 +1197,11 @@ clip_image_size mtmd_image_preprocessor_deepseekocr2::find_closest_aspect_ratio( return best_ratio; } -bool mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img) { // emit 768x768 local tiles when the image is larger than a tile in either // dimension, then always a 1024x1024 global view. order: [tiles..., global]. + mtmd_image_preproc_out output; const auto img_size = img.get_size(); if (img_size.width > tile_size || img_size.height > tile_size) { const float aspect_ratio = static_cast(img_size.width) / img_size.height; @@ -1226,9 +1217,7 @@ bool mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img, for (int col = 0; col < grid.width; col++) { clip_image_u8 tile; img_tool::crop(refined, tile, col * tile_size, row * tile_size, tile_size, tile_size); - clip_image_f32_ptr res(clip_image_f32_init()); - img_u8_to_f32(tile, *res, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(res)); + output.append(hparams, tile, true); } } } @@ -1237,14 +1226,11 @@ bool mtmd_image_preprocessor_deepseekocr2::preprocess(const clip_image_u8 & img, clip_image_u8 padded; img_tool::resize(img, padded, { base_size, base_size }, RESIZE_ALGO_BICUBIC_PILLOW, PAD_NEAREST, hparams.image_pad_color); - clip_image_f32_ptr global(clip_image_f32_init()); - img_u8_to_f32(padded, *global, hparams.image_mean, hparams.image_std); - global->add_viewsep = true; - output.entries.push_back(std::move(global)); - + output.append(hparams, padded, true); + output.entries.back().add_viewsep = true; output.grid_x = 1; output.grid_y = 1; - return true; + return output; } // @@ -1260,7 +1246,8 @@ void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32( const float std[3]) { const auto src_size = src.get_size(); if (src_size.width == target_width && src_size.height == target_height) { - img_u8_to_f32(src, dst, mean, std); + dst.from_u8(src); + dst.normalize(mean, std); return; } @@ -1455,24 +1442,25 @@ mtmd_image_preprocessor_step3vl::slice_instructions mtmd_image_preprocessor_step return instructions; } -bool mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img) { clip_image_u8 prepared = prepare_image(img, hparams); const auto instructions = build_slice_instructions(hparams, prepared.get_size()); - clip_image_f32_ptr overview_f32(clip_image_f32_init()); + mtmd_image_preproc_out output; + clip_image_f32 overview_f32; img_u8_resize_bilinear_to_f32( prepared, - *overview_f32, + overview_f32, hparams.image_size, hparams.image_size, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(overview_f32)); + output.append(hparams, overview_f32, false); if (instructions.slices.empty()) { output.grid_x = 0; output.grid_y = 0; - return true; + return output; } clip_image_u8 img_for_crop = prepared; @@ -1488,28 +1476,28 @@ bool mtmd_image_preprocessor_step3vl::preprocess(const clip_image_u8 & img, clip // If the requested patch extends past the source image, pad the out-of-bounds area with black. clip_image_u8 patch = crop_with_black_padding(img_for_crop, slice.x, slice.y, slice.size.width, slice.size.height); - clip_image_f32_ptr patch_f32(clip_image_f32_init()); + clip_image_f32 patch_f32; img_u8_resize_bilinear_to_f32( patch, - *patch_f32, + patch_f32, crop_size, crop_size, hparams.image_mean, hparams.image_std); - output.entries.push_back(std::move(patch_f32)); + output.append(hparams, patch_f32, false); } output.grid_x = instructions.grid_size.width; output.grid_y = instructions.grid_size.height; - return true; + return output; } // // mtmd_image_preprocessor_youtuvl // -bool mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { +mtmd_image_preproc_out mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img) { const int patch_size = hparams.patch_size; // typically 16 const int merge_size = hparams.n_merge; // typically 2 const int align_size = patch_size * merge_size; // 32 @@ -1553,29 +1541,22 @@ bool mtmd_image_preprocessor_youtuvl::preprocess(const clip_image_u8 & img, clip clip_image_u8 resized; img_tool::resize(img, resized, new_size, hparams.image_resize_algo, hparams.image_resize_pad); - // Normalize to float32 - clip_image_f32_ptr img_f32(clip_image_f32_init()); - img_u8_to_f32(resized, *img_f32, hparams.image_mean, hparams.image_std); - // Add to results - output.entries.push_back(std::move(img_f32)); - return true; + mtmd_image_preproc_out output; + output.append(hparams, resized, true); + return output; } -bool mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) { - // call super class preprocessor - bool ok = mtmd_image_preprocessor_llava_uhd::preprocess(img, output); - if (!ok) { - return false; - } +mtmd_image_preproc_out mtmd_image_preprocessor_granite::preprocess(const clip_image_u8 & img) { + auto output = mtmd_image_preprocessor_llava_uhd::preprocess(img); if (output.entries.size() == 1) { // Single-tile (overview only): append one newline row. - output.entries[0]->add_newline = true; + output.entries[0].add_newline = true; } else { // Multi-tile: overview gets no newline, grid tiles get one. - output.entries[0]->add_newline = false; + output.entries[0].add_newline = false; for (size_t i = 1; i < output.entries.size(); ++i) { - output.entries[i]->add_newline = true; + output.entries[i].add_newline = true; } } - return true; + return output; } diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 3ddbec2e4..8819a135a 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -8,6 +8,16 @@ #define MTMD_INTERNAL_HEADER +struct mtmd_image_preproc_out { + std::vector entries; + // grid size is required for llava-uhd style models + int grid_x = 0; + int grid_y = 0; + void append(const clip_hparams & hparams, const clip_image_u8 & img, bool normalized = true); + void append(const clip_hparams & hparams, const std::vector & imgs, bool normalized = true); + void append(const clip_hparams & hparams, clip_image_f32 & img, bool normalized = true); +}; + // base class, models must inherit from this class struct mtmd_image_preprocessor { const clip_hparams & hparams; @@ -15,10 +25,7 @@ struct mtmd_image_preprocessor { mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {} virtual ~mtmd_image_preprocessor() = default; - virtual bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) = 0; - - void img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst, const float mean[3], const float std[3]); - void img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst); + virtual mtmd_image_preproc_out preprocess(const clip_image_u8 & img) = 0; }; /** @@ -42,7 +49,7 @@ struct mtmd_image_preprocessor { */ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; struct slice_coordinates { int x; @@ -60,7 +67,7 @@ struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor { // LFM2 override this function to implement its custom slicing logic virtual slice_instructions get_slice_instructions(const clip_image_size & original_size); - std::vector slice_image(const clip_image_u8 & img, const slice_instructions & inst, bool overview_first = true); + std::vector slice_image(const clip_image_u8 & img, const slice_instructions & inst, bool overview_first = true); private: clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false); @@ -91,7 +98,7 @@ private: // downscale or upscale the input image to fixed size struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; // resize image to multiple of patch_size*n_merge, while preserving aspect ratio @@ -99,13 +106,13 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { // this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; // similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor { mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; // custom llava-uhd slicing logic for LFM2 @@ -131,17 +138,17 @@ private: struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor { mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; // DeepSeek-OCR-2: a 1024x1024 global view, plus InternVL-style 768x768 local @@ -153,7 +160,7 @@ struct mtmd_image_preprocessor_deepseekocr2 : mtmd_image_preprocessor { static constexpr int max_tiles = 6; mtmd_image_preprocessor_deepseekocr2(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; private: static std::vector get_target_ratios(); @@ -168,7 +175,7 @@ private: // ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size); private: @@ -195,11 +202,11 @@ private: struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor { mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; // similar to llava_uhd, but has add_newline struct mtmd_image_preprocessor_granite : mtmd_image_preprocessor_llava_uhd { mtmd_image_preprocessor_granite(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {} - bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; }; diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 0834e4938..f9ee021dd 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -114,7 +114,7 @@ struct mtmd_image_tokens { // true if one of entries in batch_f32 is a placeholder bool is_placeholder() const { for (const auto & entry : batch_f32.entries) { - if (entry->is_placeholder()) { + if (entry.is_placeholder()) { return true; } } @@ -147,7 +147,7 @@ struct mtmd_audio_tokens { // true if one of entries in batch_f32 is a placeholder bool is_placeholder() const { for (const auto & entry : batch_f32.entries) { - if (entry->is_placeholder()) { + if (entry.is_placeholder()) { return true; } } @@ -1050,7 +1050,7 @@ struct mtmd_tokenizer { // TODO @ngxson : this is quite hacky because preprocessor only support batch with one single element, that need to be fixed in the future (e.g. by changing the preprocessor interface always take single input) - clip_image_f32_batch batch_f32; + mtmd_image_preproc_out preproc_out; for (const auto * bmp : bitmaps) { // sanity check @@ -1063,42 +1063,40 @@ struct mtmd_tokenizer { } // convert mtmd_bitmap to clip_image_u8 - clip_image_u8_ptr img_u8(clip_image_u8_init()); - img_u8->set_size( + clip_image_u8 img_u8; + img_u8.set_size( {(int)bmp->nx, (int)bmp->ny}, bmp->is_placeholder()); - img_u8->cpy_buf(bmp->get_ro_buf()); + img_u8.cpy_buf(bmp->get_ro_buf()); // preprocess image - clip_image_f32_batch tmp_batch; - bool ok = ctx->image_preproc->preprocess(*img_u8, tmp_batch); - if (!ok) { - LOG_ERR("Unable to preprocess image\n"); - return 2; - } + mtmd_image_preproc_out tmp_preproc_out = ctx->image_preproc->preprocess(img_u8); - // move entries and grid dimensions to the "global" batch_f32 - for (auto & entry : tmp_batch.entries) { - batch_f32.entries.emplace_back(std::move(entry)); + // move entries and grid dimensions to the "global" preproc_out + for (auto & entry : tmp_preproc_out.entries) { + preproc_out.entries.emplace_back(std::move(entry)); } // for llava-uhd style, we need to handle grid too - // we don't care about overwriting these values for now because llama-uhd doesn't support batching anyway - batch_f32.grid_x = tmp_batch.grid_x; - batch_f32.grid_y = tmp_batch.grid_y; + // we don't care about overwriting these values for now because the case where bitmaps.size() > 1 is only for frame merging (qwen-vl), not supported by llava-uhd + if (tmp_preproc_out.grid_x > 0 && tmp_preproc_out.grid_y > 0) { + GGML_ASSERT(bitmaps.size() == 1); + preproc_out.grid_x = tmp_preproc_out.grid_x; + preproc_out.grid_y = tmp_preproc_out.grid_y; + } } // handle llava-uhd style preprocessing - const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0; + const bool has_tiling_grid = preproc_out.grid_x > 0 && preproc_out.grid_y > 0; if (has_tiling_grid) { // [QWEN_VIDEO] we do not support "frame merging" for llama-uhd style, so no batching for now GGML_ASSERT(bitmaps.size() == 1); - const int n_col = batch_f32.grid_x; - const int n_row = batch_f32.grid_y; + const int n_col = preproc_out.grid_x; + const int n_row = preproc_out.grid_y; // split batch into chunks of single images - // NOTE: batch_f32 will be invalidated after this call - auto chunks = split_batch_to_chunk(std::move(batch_f32), bitmaps[0]->id); + // NOTE: preproc_out will be invalidated after this call + auto chunks = split_batch_to_chunk(std::move(preproc_out), bitmaps[0]->id); GGML_ASSERT(chunks.size() > 0); auto ov_chunk = std::move(chunks.front()); @@ -1150,8 +1148,8 @@ struct mtmd_tokenizer { } else { size_t n_tokens = 0; - for (const auto & e : batch_f32.entries) { - n_tokens += clip_n_output_tokens(ctx->ctx_v, e.get()); + for (auto & e : preproc_out.entries) { + n_tokens += clip_n_output_tokens(ctx->ctx_v, &e); if (clip_model_n_temporal_merge(ctx->ctx_v) == 2) { // [QWEN_VIDEO] pair input is merged to the same embd, so only count as one image break; @@ -1165,8 +1163,8 @@ struct mtmd_tokenizer { if (mtmd_decode_use_mrope(ctx)) { // for Qwen2VL, we need this information for M-RoPE decoding positions - image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, batch_f32.entries[0].get()); - image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, batch_f32.entries[0].get()); + image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, &preproc_out.entries[0]); + image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, &preproc_out.entries[0]); } else { // other models, we only need the total number of tokens image_tokens->nx = n_tokens; @@ -1181,6 +1179,12 @@ struct mtmd_tokenizer { image_tokens->image_idx = n_images_added; GGML_ASSERT(n_tokens == (size_t)image_tokens->n_tokens()); } + + clip_image_f32_batch batch_f32; + batch_f32.is_audio = false; + batch_f32.entries = std::move(preproc_out.entries); + // do NOT use preproc_out from this point on, it's moved + image_tokens->batch_f32 = std::move(batch_f32); image_tokens->id = bitmaps[0]->id; // optional @@ -1260,13 +1264,13 @@ struct mtmd_tokenizer { for (auto & mel_spec : mel_spec_chunks) { const bool is_placeholder = mel_spec.data.empty(); - clip_image_f32_ptr mel_f32(clip_image_f32_init()); - mel_f32->set_size( + clip_image_f32 mel_f32; + mel_f32.set_size( {mel_spec.n_len, mel_spec.n_mel}, is_placeholder, /* is_audio */ true); - mel_f32->cpy_buf(mel_spec.data); + mel_f32.cpy_buf(mel_spec.data); - size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, mel_f32.get()); + size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, &mel_f32); clip_image_f32_batch batch_f32; batch_f32.is_audio = true; @@ -1296,12 +1300,12 @@ struct mtmd_tokenizer { return 0; } - std::vector split_batch_to_chunk(clip_image_f32_batch && batch_f32, const std::string & id) { + std::vector split_batch_to_chunk(mtmd_image_preproc_out && preproc_out, const std::string & id) { std::vector chunks; - for (auto & entry : batch_f32.entries) { + for (auto & entry : preproc_out.entries) { mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens); - image_tokens->nx = clip_n_output_tokens(ctx->ctx_v, entry.get()); + image_tokens->nx = clip_n_output_tokens(ctx->ctx_v, &entry); image_tokens->ny = 1; image_tokens->batch_f32.entries.push_back(std::move(entry)); image_tokens->id = id; @@ -1406,16 +1410,16 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im // e.g., DeepSeek-OCR-2: 144 per tile views, 257 for the global view size_t offset = 0; for (size_t i = 0; i < entries.size(); i++) { - if (entries[i]->is_placeholder()) { + if (entries[i].is_placeholder()) { LOG_ERR("%s: image tokens batch entry %zu is placeholder\n", __func__, i); return 1; } - int n_tokens_per_image = clip_n_output_tokens(ctx_clip, entries[i].get()); + int n_tokens_per_image = clip_n_output_tokens(ctx_clip, &entries[i]); std::vector tmp_embd((size_t)n_tokens_per_image * n_embd_out); bool ok_i = clip_image_encode( ctx_clip, ctx->n_threads, - entries[i].get(), + &entries[i], tmp_embd); if (!ok_i) { LOG_ERR("%s: failed to encode image %zu\n", __func__, i); @@ -2063,16 +2067,18 @@ void mtmd_debug_preprocess_image(mtmd_context * ctx, const std::vector clip_image_u8 img_u8; img_u8.set_size({nx, ny}, false); img_u8.cpy_buf(rgb_values); - clip_image_f32_batch batch_f32; GGML_ASSERT(ctx->image_preproc != nullptr); - bool ok = ctx->image_preproc->preprocess(img_u8, batch_f32); - if (!ok) { - LOG_ERR("%s: failed to preprocess image\n", __func__); - return; + mtmd_image_preproc_out preproc_out = ctx->image_preproc->preprocess(img_u8); + + clip_image_f32_batch batch_f32; + batch_f32.is_audio = false; + for (auto & entry : preproc_out.entries) { + batch_f32.entries.push_back(std::move(entry)); } + LOG_INF("%s: preprocessed image to batch_f32 with %d entries\n", __func__, (int)batch_f32.entries.size()); for (size_t i = 0; i < batch_f32.entries.size(); i++) { - LOG_INF("%s: entry %zu has nx=%d, ny=%d\n", __func__, i, batch_f32.entries[i]->nx(), batch_f32.entries[i]->ny()); + LOG_INF("%s: entry %zu has nx=%d, ny=%d\n", __func__, i, batch_f32.entries[i].nx(), batch_f32.entries[i].ny()); // TODO: better way to dump entry content? } } From 968c43891abb1def4db76f25713b558629fb92b1 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 18 Jun 2026 12:15:46 +0200 Subject: [PATCH 039/292] server: fix router args not being forwarded to child instances (#24760) --- tools/server/server-models.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index ff9a0df12..cc7184165 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -351,6 +351,12 @@ void server_models::load_models() { source_map[name] = SERVER_MODEL_SOURCE_PRESET; } + // overlay router's own CLI args on top of every model preset so that + // e.g. `llama-server --temp 0` is honoured by all child processes + for (auto & [name, preset] : final_presets) { + preset.merge(base_preset); + } + auto get_source = [&](const std::string & name) { return source_map.count(name) ? source_map.at(name) : SERVER_MODEL_SOURCE_PRESET; }; From 552258c5350dcf86c6a1e9a2dcd45f06076a4667 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Thu, 18 Jun 2026 12:45:23 +0200 Subject: [PATCH 040/292] server: (router) rework -hf preset repo (#24739) * server: temporary remove HF remote preset * rework remove preset.ini support * rm unused get_remote_preset_whitelist() * print warning * add docs * rm stray file --- common/arg.cpp | 112 ++++++++++++---------------------------- common/common.h | 9 ++-- common/download.cpp | 53 +++++++++++++------ common/download.h | 1 + common/preset.cpp | 50 +----------------- common/preset.h | 2 +- docs/preset.md | 74 +++++++++++++------------- tools/server/server.cpp | 6 +++ 8 files changed, 120 insertions(+), 187 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 8382c1d85..bd4b113d1 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -285,58 +285,15 @@ static std::string clean_file_name(const std::string & fname) { return clean_fname; } -static bool common_params_handle_remote_preset(common_params & params, llama_example ex) { - GGML_ASSERT(!params.model.hf_repo.empty()); - - // the returned hf_repo is without tag - auto [hf_repo, hf_tag] = common_download_split_repo_tag(params.model.hf_repo); - - // "latest" tag (default if not specified) is translated to "default" preset - if (hf_tag == "latest") { - hf_tag = "default"; - } - - std::string model_endpoint = common_get_model_endpoint(); - auto preset_url = model_endpoint + hf_repo + "/resolve/main/preset.ini"; - - // prepare local path for caching - auto preset_fname = clean_file_name(hf_repo + "_preset.ini"); - auto preset_path = fs_get_cache_file(preset_fname); - common_download_opts opts; - opts.bearer_token = params.hf_token; - opts.offline = params.offline; - - LOG_TRC("%s: looking for remote preset at %s\n", __func__, preset_url.c_str()); - const int status = common_download_file_single(preset_url, preset_path, opts); - const bool has_preset = status >= 200 && status < 400; - - // remote preset is optional, so we don't error out if not found - if (has_preset) { - LOG_TRC("%s: applying remote preset from %s\n", __func__, preset_url.c_str()); - common_preset_context ctx(ex, /* only_remote_allowed */ true); - common_preset global; - auto remote_presets = ctx.load_from_ini(preset_path, global); - remote_presets = ctx.cascade(global, remote_presets); - if (remote_presets.find(hf_tag) != remote_presets.end()) { - common_preset preset = remote_presets.at(hf_tag); - LOG_INF("\n%s", preset.to_ini().c_str()); // to_ini already added trailing newline - preset.apply_to_params(params); - } else { - throw std::runtime_error("Remote preset.ini does not contain [" + std::string(hf_tag) + "] section"); - } - } else { - LOG_TRC("%s: no remote preset found, skipping\n", __func__); - } - - return has_preset; -} - struct handle_model_result { bool found_mmproj = false; common_params_model mmproj; bool found_mtp = false; common_params_model mtp; + + bool found_preset = false; + std::string preset_path; }; static handle_model_result common_params_handle_model(struct common_params_model & model, @@ -355,6 +312,12 @@ static handle_model_result common_params_handle_model(struct common_params_model common_download_opts hf_opts = opts; auto download_result = common_download_model(model, hf_opts); + if (!download_result.preset_path.empty()) { + result.found_preset = true; + result.preset_path = download_result.preset_path; + return result; // skip everything else if preset.ini is used + } + if (download_result.model_path.empty()) { throw std::runtime_error("failed to download model from Hugging Face"); } @@ -454,6 +417,17 @@ bool common_params_handle_models(common_params & params, llama_example curr_ex) try { auto res = common_params_handle_model(params.model, opts); + if (res.found_preset) { + if (!params.models_preset.empty()) { + throw std::invalid_argument("cannot use both --models-preset and -hf with a preset.ini file"); + } + // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file + params.models_preset_hf = params.model.hf_repo; // only for showing a warning + params.models_preset = res.preset_path; + params.model = common_params_model{}; // make sure to clear model, so server starts in router mode + return true; + } + if (params.no_mmproj) { params.mmproj = {}; } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) { @@ -601,30 +575,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context // parse the first time to get -hf option (used for remote preset) parse_cli_args(); - // export_graph_ops loads only metadata - const bool skip_model_download = ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; - - // maybe handle remote preset - if (!params.model.hf_repo.empty() && !skip_model_download) { - std::string cli_hf_repo = params.model.hf_repo; - bool has_preset = common_params_handle_remote_preset(params, ctx_arg.ex); - - // special case: if hf_repo explicitly set by preset, we need to preserve it (ignore CLI value) - // this is useful when we have one HF repo pointing to other HF repos (one model - multiple GGUFs) - std::string preset_hf_repo = params.model.hf_repo; - bool preset_has_hf_repo = preset_hf_repo != cli_hf_repo; - - if (has_preset) { - // re-parse CLI args to override preset values - parse_cli_args(); - } - - // preserve hf_repo from preset if needed - if (preset_has_hf_repo) { - params.model.hf_repo = preset_hf_repo; - } - } - postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); @@ -635,15 +585,21 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); } - // handle model and download - if (!skip_model_download) { - common_params_handle_models(params, ctx_arg.ex); - } + // export_graph_ops loads only metadata + const bool skip_model_download = ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; - // model is required (except for server) - // TODO @ngxson : maybe show a list of available models in CLI in this case - if (params.model.path.empty() && ctx_arg.ex != LLAMA_EXAMPLE_SERVER && !skip_model_download && !params.usage && !params.completion) { - throw std::invalid_argument("error: --model is required\n"); + if (!skip_model_download) { + // handle model and download + common_params_handle_models(params, ctx_arg.ex); + + // model is required (except for server) + // TODO @ngxson : maybe show a list of available models in CLI in this case + if (params.model.path.empty() + && ctx_arg.ex != LLAMA_EXAMPLE_SERVER + && !params.usage + && !params.completion) { + throw std::invalid_argument("error: --model is required\n"); + } } if (params.escape) { diff --git a/common/common.h b/common/common.h index 0b284cbb3..040b9cf23 100644 --- a/common/common.h +++ b/common/common.h @@ -642,10 +642,11 @@ struct common_params { std::vector server_tools; // router server configs - std::string models_dir = ""; // directory containing models for the router server - std::string models_preset = ""; // directory containing model presets for the router server - int models_max = 4; // maximum number of models to load simultaneously - bool models_autoload = true; // automatically load models when requested via the router server + std::string models_dir = ""; // directory containing models for the router server + std::string models_preset = ""; // directory containing model presets for the router server + int models_max = 4; // maximum number of models to load simultaneously + bool models_autoload = true; // automatically load models when requested via the router server + std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty) bool log_json = false; diff --git a/common/download.cpp b/common/download.cpp index c3c8ff49b..f32046275 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -696,6 +696,7 @@ struct hf_plan { hf_cache::hf_files model_files; hf_cache::hf_file mmproj; hf_cache::hf_file mtp; + hf_cache::hf_file preset; // if set, only this file is downloaded }; static hf_plan get_hf_plan(const common_params_model & model, @@ -717,6 +718,14 @@ static hf_plan get_hf_plan(const common_params_model & model, return plan; } + // if preset.ini exists in the repo root, download only that file + for (const auto & f : all) { + if (f.path == "preset.ini") { + plan.preset = f; + return plan; + } + } + hf_cache::hf_file primary; if (!model.hf_file.empty()) { @@ -794,14 +803,19 @@ common_download_model_result common_download_model(const common_params_model & if (is_hf) { hf = get_hf_plan(model, opts, download_mmproj, download_mtp); - for (const auto & f : hf.model_files) { - tasks.push_back({f.url, f.local_path}); - } - if (!hf.mmproj.path.empty()) { - tasks.push_back({hf.mmproj.url, hf.mmproj.local_path}); - } - if (!hf.mtp.path.empty()) { - tasks.push_back({hf.mtp.url, hf.mtp.local_path}); + if (!hf.preset.path.empty()) { + // if preset.ini exists, only download that file alone + tasks.push_back({hf.preset.url, hf.preset.local_path}); + } else { + for (const auto & f : hf.model_files) { + tasks.push_back({f.url, f.local_path}); + } + if (!hf.mmproj.path.empty()) { + tasks.push_back({hf.mmproj.url, hf.mmproj.local_path}); + } + if (!hf.mtp.path.empty()) { + tasks.push_back({hf.mtp.url, hf.mtp.local_path}); + } } } else if (!model.url.empty()) { tasks = get_url_tasks(model); @@ -835,17 +849,22 @@ common_download_model_result common_download_model(const common_params_model & } if (is_hf) { - for (const auto & f : hf.model_files) { - hf_cache::finalize_file(f); - } - result.model_path = hf.primary.final_path; + if (!hf.preset.path.empty()) { + // if preset.ini is used, do not set other paths + result.preset_path = hf_cache::finalize_file(hf.preset); + } else { + for (const auto & f : hf.model_files) { + hf_cache::finalize_file(f); + } + result.model_path = hf.primary.final_path; - if (!hf.mmproj.path.empty()) { - result.mmproj_path = hf_cache::finalize_file(hf.mmproj); - } + if (!hf.mmproj.path.empty()) { + result.mmproj_path = hf_cache::finalize_file(hf.mmproj); + } - if (!hf.mtp.path.empty()) { - result.mtp_path = hf_cache::finalize_file(hf.mtp); + if (!hf.mtp.path.empty()) { + result.mtp_path = hf_cache::finalize_file(hf.mtp); + } } } else { result.model_path = model.path; diff --git a/common/download.h b/common/download.h index 237179764..8dbf07836 100644 --- a/common/download.h +++ b/common/download.h @@ -63,6 +63,7 @@ struct common_download_model_result { std::string model_path; std::string mmproj_path; std::string mtp_path; + std::string preset_path; }; // throw if the file is missing or invalid (e.g. ETag check failed) diff --git a/common/preset.cpp b/common/preset.cpp index 51ea984d8..f0cc1fa1a 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -16,48 +16,6 @@ static std::string rm_leading_dashes(const std::string & str) { return str.substr(pos); } -// only allow a subset of args for remote presets for security reasons -// do not add more args unless absolutely necessary -// args that output to files are strictly prohibited -static std::set get_remote_preset_whitelist(const std::map & key_to_opt) { - static const std::set allowed_options = { - "model-url", - "hf-repo", - "hf-repo-draft", - "hf-repo-v", // vocoder - "hf-file-v", // vocoder - "mmproj-url", - "pooling", - "jinja", - "batch-size", - "ubatch-size", - "cache-reuse", - "chat-template-kwargs", - "mmap", - // note: sampling params are automatically allowed by default - // negated args will be added automatically if the positive arg is specified above - }; - - std::set allowed_keys; - - for (const auto & it : key_to_opt) { - const std::string & key = it.first; - const common_arg & opt = it.second; - if (allowed_options.find(key) != allowed_options.end() || opt.is_sampling) { - allowed_keys.insert(key); - // also add variant keys (args without leading dashes and env vars) - for (const auto & arg : opt.get_args()) { - allowed_keys.insert(rm_leading_dashes(arg)); - } - for (const auto & env : opt.get_env()) { - allowed_keys.insert(env); - } - } - } - - return allowed_keys; -} - std::vector common_preset::to_args(const std::string & bin_path) const { std::vector args; @@ -300,16 +258,10 @@ static std::string parse_bool_arg(const common_arg & arg, const std::string & ke return value; } -common_preset_context::common_preset_context(llama_example ex, bool only_remote_allowed) +common_preset_context::common_preset_context(llama_example ex) : ctx_params(common_params_parser_init(default_params, ex)) { common_params_add_preset_options(ctx_params.options); key_to_opt = get_map_key_opt(ctx_params); - - // setup allowed keys if only_remote_allowed is true - if (only_remote_allowed) { - filter_allowed_keys = true; - allowed_keys = get_remote_preset_whitelist(key_to_opt); - } } common_presets common_preset_context::load_from_ini(const std::string & path, common_preset & global) const { diff --git a/common/preset.h b/common/preset.h index 06f829c3e..52935ebde 100644 --- a/common/preset.h +++ b/common/preset.h @@ -60,7 +60,7 @@ struct common_preset_context { std::set allowed_keys; // if only_remote_allowed is true, only accept whitelisted keys - common_preset_context(llama_example ex, bool only_remote_allowed = false); + common_preset_context(llama_example ex); // load presets from INI file common_presets load_from_ini(const std::string & path, common_preset & global) const; diff --git a/docs/preset.md b/docs/preset.md index d49fb0a1a..85762a420 100644 --- a/docs/preset.md +++ b/docs/preset.md @@ -8,55 +8,53 @@ The INI preset feature, introduced in [PR#17859](https://github.com/ggml-org/lla When running multiple models on the server (router mode), INI preset files can be used to configure model-specific parameters. Please refer to the [server documentation](../tools/server/README.md) for more details. -### Using a Remote Preset +### Using a Hugging Face Preset -> [!NOTE] +> [!IMPORTANT] > -> This feature is currently only supported via the `-hf` option. +> Please only use presets that you can trust! Unknown presets may be unsafe -For GGUF models hosted on Hugging Face, you can include a `preset.ini` file in the root directory of the repository to define specific configurations for that model. +You can push your preset to Hugging Face Hub and share with other users by: +1. Creating an empty model repository on Hugging Face +2. Creating a `preset.ini` file in the root directory of the repository -Example: +Example of a `preset.ini`: ```ini -hf-repo-draft = username/my-draft-model-GGUF -temp = 0.5 -top-k = 20 -top-p = 0.95 +[*] +ctx-size = 0 +mmap = 1 +kv-unified = 1 +parallel = 4 +spec-default = 1 + +[Qwen3.5-4B] +hf = unsloth/Qwen3.5-4B-GGUF:Q4_K_M +ctx-size = 262144 +batch-size = 2048 +ubatch-size = 2048 +top-p = 1.0 +top-k = 0 +min-p = 0.01 +temp = 1.0 + +[gpt-oss-120b-hf] +hf = ggml-org/gpt-oss-120b-GGUF +ctx-size = 262144 +batch-size = 2048 +ubatch-size = 2048 +top-p = 1.0 +top-k = 0 +min-p = 0.01 +temp = 1.0 +chat-template-kwargs = {"reasoning_effort": "high"} ``` -For security reasons, only certain options are allowed. Please refer to [preset.cpp](../common/preset.cpp) for the complete list of permitted options. - -Example usage: - -Assuming your repository `username/my-model-with-preset` contains a `preset.ini` with the configuration above: - -```sh -llama-cli -hf username/my-model-with-preset - -# This is equivalent to: -llama-cli -hf username/my-model-with-preset \ - --hf-repo-draft username/my-draft-model-GGUF \ - --temp 0.5 \ - --top-k 20 \ - --top-p 0.95 -``` - -You can also override preset arguments by specifying them on the command line: +The preset will be loaded similarly to the `--models-preset` option. Therefore, you can also override certain params via CLI arguments: ```sh # Force temp = 0.1, overriding the preset value -llama-cli -hf username/my-model-with-preset --temp 0.1 -``` - -If you want to define multiple preset configurations for one or more GGUF models, you can create a blank HF repo for each preset. Each HF repo should contain a `preset.ini` file that references the actual model(s): - -```ini -hf-repo = user/my-model-main -hf-repo-draft = user/my-model-draft -temp = 0.8 -ctx-size = 1024 -; (and other configurations) +llama-cli -hf username/my-preset --temp 0.1 ``` ### Named presets diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 0364d7d3b..78ab0318c 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -349,6 +349,12 @@ int llama_server(int argc, char ** argv) { SRV_INF("router server is listening on %s\n", ctx_http.listening_address.c_str()); SRV_WRN("%s", "NOTE: router mode is experimental\n"); SRV_WRN("%s", " it is not recommended to use this mode in untrusted environments\n"); + + if (!params.models_preset_hf.empty()) { + SRV_WRN( "NOTE: using preset.ini from HF repo '%s'\n", params.models_preset_hf.c_str()); + SRV_WRN("%s", " please only use presets that you can trust! Unknown presets may be unsafe\n"); + } + if (ctx_http.thread.joinable()) { ctx_http.thread.join(); // keep the main thread alive } From 10786217e9d40c848ac0133cbe9c5f22a52421bb Mon Sep 17 00:00:00 2001 From: Anuj Attri Date: Thu, 18 Jun 2026 06:49:14 -0400 Subject: [PATCH 041/292] server : return HTTP 400 on invalid grammar (#24144) (#24154) Throw on grammar parse failure so the server returns HTTP 400 instead of silently dropping the constraint. Add a regression test for the invalid-grammar response. Fixes #24144 --- common/sampling.cpp | 3 +++ tools/server/tests/unit/test_chat_completion.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/common/sampling.cpp b/common/sampling.cpp index c537f3350..75a299e23 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -259,6 +259,9 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st } } } + if (!grmr && !grammar_str.empty()) { + throw std::runtime_error("failed to parse grammar"); + } // Compute prefill tokens from the generation prompt std::vector prefill_tokens; diff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.py index fe55dc5ab..b00aac649 100644 --- a/tools/server/tests/unit/test_chat_completion.py +++ b/tools/server/tests/unit/test_chat_completion.py @@ -307,6 +307,20 @@ def test_completion_with_grammar(jinja: bool, grammar: str, n_predicted: int, re assert match_regex(re_content, choice["message"]["content"]), choice["message"]["content"] +def test_completion_with_invalid_grammar(): + global server + server.start() + res = server.make_request("POST", "/chat/completions", data={ + "max_tokens": 8, + "messages": [ + {"role": "user", "content": "Does not matter what I say, does it?"}, + ], + "grammar": "root ::= this is (not valid GBNF", + }) + assert res.status_code == 400, res.body + assert "error" in res.body + + @pytest.mark.parametrize("messages", [ None, "string", From 20832179e20c3d29ea79cd106dd16578cefe93ec Mon Sep 17 00:00:00 2001 From: Amos Wong <8733840+amoshydra@users.noreply.github.com> Date: Thu, 18 Jun 2026 19:14:20 +0800 Subject: [PATCH 042/292] ui: provide touch accessible model selection UI (#24604) * ui : add model selector storybook stories Covers list, favorites, single-model, all status states (loading/loaded/sleeping/failed/idle), and selection states. * ui : improve model selector mobile UX with hover media queries Use @media (hover:none) to show action buttons directly on touch devices and color-code them by model status (amber=sleeping, green=loaded, muted=idle). Status dots hidden on touch. Desktop hover behavior unchanged. --- .../app/models/ModelsSelectorOption.svelte | 44 +-- .../app/models/ModelsSelectorSheet.svelte | 2 +- .../stories/ModelsSelector.stories.svelte | 269 ++++++++++++++++++ 3 files changed, 297 insertions(+), 18 deletions(-) create mode 100644 tools/ui/tests/stories/ModelsSelector.stories.svelte diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte index d103d4b67..fef1490f3 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte @@ -79,7 +79,7 @@
e.stopPropagation()} > {#if isFav} @@ -113,12 +113,16 @@
{#if isLoading} - +
+ +
{:else if isFailed} -
- +
+ -