mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-05-23 21:13:36 +00:00
c9872a2575
* hmx-mm: update debug logging in hmx-mm * hmx-mm: update dequant logic to use HVX_vector_x2/4 * hmx-mm: remove non-pipelined version of the quantize matmul It seems that we don't reall need non-pipelined version * hmx-mm: use activation depth mode and update naming Co-authored-by: Kim-Chyan Gan <kgan@qti.qualcomm.com> * hex-mm: minor hmx matmul naming updates * hmx-mm: remove unused vars * snapdragon: scripts bump default ubatch-size to 1K * hexagon: combine HMX and power and clock settings into a single set_power call * hmx-mm: remove leftover of the scale repl helper * hexagon: fix editconf error --------- Co-authored-by: Kim-Chyan Gan <kgan@qti.qualcomm.com>
53 lines
998 B
PowerShell
53 lines
998 B
PowerShell
|
|
#!/usr/bin/env pwsh
|
|
|
|
# Basedir on device
|
|
$basedir=".\pkg-snapdragon"
|
|
|
|
$cli_opts=$args
|
|
|
|
$model="Llama-3.2-3B-Instruct-Q4_0.gguf"
|
|
if ($null -ne $env:M) {
|
|
$model=$env:M
|
|
}
|
|
|
|
$device="HTP0"
|
|
if ($null -ne $env:D) {
|
|
$device=$env:D
|
|
}
|
|
|
|
if ($null -ne $env:V) {
|
|
$env:GGML_HEXAGON_VERBOSE=$env:V
|
|
}
|
|
|
|
if ($null -ne $env:SCHED) {
|
|
$env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
|
|
}
|
|
|
|
if ($null -ne $env:PROF) {
|
|
$env:GGML_HEXAGON_PROFILE=$env:PROF
|
|
}
|
|
|
|
if ($null -ne $env:OPSTAGE) {
|
|
$env:GGML_HEXAGON_OPSTAGE=$env:OPSTAGE
|
|
}
|
|
|
|
if ($null -ne $env:NHVX) {
|
|
$env:GGML_HEXAGON_NHVX=$env:NHVX
|
|
}
|
|
|
|
if ($null -ne $env:NDEV) {
|
|
$env:GGML_HEXAGON_NDEV=$env:NDEV
|
|
}
|
|
|
|
if ($null -ne $env:HB) {
|
|
$env:GGML_HEXAGON_HOSTBUF=$env:HB
|
|
}
|
|
|
|
$env:ADSP_LIBRARY_PATH="$basedir\lib"
|
|
|
|
& "$basedir\bin\llama-cli.exe" `
|
|
--no-mmap -m $basedir\..\..\gguf\$model `
|
|
--poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
|
|
--ctx-size 8192 --ubatch-size 1024 -fa on `
|
|
-ngl 99 --device $device $cli_opts
|