mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-04-28 11:40:43 +00:00
5d2b52d80d
* hexagon: restore HTP_OPMASK_QUEUE * hexagon: honor OPMASK_SKIP_COMPUTE in hmx-matmul * hex-prof: restore op profiling * hex-prof: enable PMU * hexagon: simplify and improve op-queuing with full profiling support Add separate profile descriptors. * hexagon: remove opsync and rename opmask into opstage opsync is no longer needed since the profiler is fully async now. opmask name was confusing and opstage is more accurate. * hexagon: refactor opbatch queue handling * hexagon: add iface hooks for enabling profiler from the host Also move all the PMU setup stuff out of the hex-utils since it's not inteded for normal use. * hexagon: make profiler mode configurable On older devices getting PMU counters is expensive so it's now optional. * hexagon: add support for setting profiler pmu events from env * hexagon: simplify profiler output (no need to print buffs, etc) * hexagon: simplify pmu counter formating * hexagon: add a simple profile post-proc tool * hex-prof: add support for reading logs from stdin * hexagon: document GGML_HEXAGON_PROFILE * hex-prof: update default width for dims field * hex-prof: fix linter warnings and errors * Update ggml/src/ggml-hexagon/htp/htp-ops.h Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com> * Update scripts/snapdragon/ggml-hexagon-profile.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com> --------- Co-authored-by: Trivikram Reddy <tamarnat@qti.qualcomm.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
68 lines
1.3 KiB
PowerShell
68 lines
1.3 KiB
PowerShell
#!/usr/bin/env pwsh
|
|
|
|
# Basedir on device
|
|
$basedir=".\pkg-snapdragon"
|
|
|
|
$cli_opts=$args
|
|
|
|
$model="gemma-3-4b-it-Q4_0.gguf"
|
|
if ($null -ne $env:M) {
|
|
$model=$env:M
|
|
}
|
|
|
|
$mmproj="mmproj-F16.gguf"
|
|
if ($null -ne $env:MMPROJ) {
|
|
$mmproj=$env:MMPROJ
|
|
}
|
|
|
|
$image=""
|
|
if ($null -ne $env:IMG) {
|
|
$image=$env:IMG
|
|
}
|
|
|
|
$device="HTP0"
|
|
if ($null -ne $env:D) {
|
|
$device=$env:D
|
|
}
|
|
|
|
if ($null -ne $env:V) {
|
|
$env:GGML_HEXAGON_VERBOSE=$env:V
|
|
}
|
|
|
|
if ($null -ne $env:SCHED) {
|
|
$env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
|
|
}
|
|
|
|
if ($null -ne $env:PROF) {
|
|
$env:GGML_HEXAGON_PROFILE=$env:PROF
|
|
}
|
|
|
|
if ($null -ne $env:OPSTAGE) {
|
|
$env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
|
|
}
|
|
|
|
if ($null -ne $env:NHVX) {
|
|
$env:GGML_HEXAGON_NHVX=$env:NHVX
|
|
}
|
|
|
|
if ($null -ne $env:NDEV) {
|
|
$env:GGML_HEXAGON_NDEV=$env:NDEV
|
|
}
|
|
|
|
if ($null -ne $env:HB) {
|
|
$env:GGML_HEXAGON_HOSTBUF=$env:HB
|
|
}
|
|
|
|
if ($null -ne $env:MTMD_DEVICE) {
|
|
$env:MTMD_BACKEND_DEVICE=$env:MTMD_DEVICE
|
|
}
|
|
|
|
$env:ADSP_LIBRARY_PATH="$basedir\lib"
|
|
|
|
& "$basedir\bin\llama-mtmd-cli.exe" `
|
|
--no-mmap -m $basedir\..\..\gguf\$model `
|
|
--mmproj $basedir\..\..\gguf\$mmproj `
|
|
--image $basedir\..\..\gguf\$image `
|
|
--poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
|
|
--ctx-size 8192 --ubatch-size 256 -fa on `
|
|
-ngl 99 --device $device -v $cli_opts
|