vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-13 02:19:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

not working correctly

Browse source
This commit is contained in:
Concedo 2025-06-02 22:12:10 +08:00
commit 6ce85c54d6
30 changed files with 1603 additions and 724 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
tools/server/public/index.html.gz
View file

Binary file not shown.

29
tools/server/server.cpp
View file

@ -2016,11 +2016,6 @@ struct server_context {
params_base.n_cache_reuse = 0;
SRV_WRN("%s\n", "cache_reuse is not supported by this context, it will be disabled");
}
if (!params_base.speculative.model.path.empty()) {
SRV_ERR("%s\n", "err: speculative decode is not supported by this context");
return false;
}
}
return true;
@ -3214,9 +3209,18 @@ struct server_context {
}
if (slot.n_past > 0 && slot.n_past < (int) slot.cache_tokens.size()) {
if (llama_kv_self_seq_pos_min(ctx, slot.id) > 0) {
const auto pos_min = llama_kv_self_seq_pos_min(ctx, slot.id);
if (pos_min == -1) {
SLT_ERR(slot, "n_past = %d, cache_tokens.size() = %d, seq_id = %d, pos_min = %d\n", slot.n_past, (int) slot.cache_tokens.size(), slot.id, pos_min);
GGML_ABORT("pos_min == -1, but n_past > 0 - should not happen: https://github.com/ggml-org/llama.cpp/pull/13833#discussion_r2116181237");
}
const auto n_swa = llama_model_n_swa(model);
if (pos_min > slot.n_past - n_swa) {
SLT_WRN(slot, "n_past = %d, cache_tokens.size() = %d, seq_id = %d, pos_min = %d, n_swa = %d\n", slot.n_past, (int) slot.cache_tokens.size(), slot.id, pos_min, n_swa);
SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA, see %s)\n",
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
llama_kv_self_seq_rm(ctx, slot.id, 0, -1);
slot.n_past = 0;
}
}
@ -3379,8 +3383,10 @@ struct server_context {
}
}
int32_t i_next = 0;
// process the created batch of tokens
for (int32_t i = 0; i < batch.n_tokens; i += n_batch) {
for (int32_t i = 0; i < batch.n_tokens; i = i_next) {
const int32_t n_tokens = std::min(n_batch, batch.n_tokens - i);
llama_batch batch_view = {
@ -3425,13 +3431,18 @@ struct server_context {
// retry with half the batch size to try to find a free slot in the KV cache
n_batch /= 2;
i -= n_batch;
SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size - try increasing it via the context size or enable defragmentation, i = %d, n_batch = %d, ret = %d\n", i, n_batch, ret);
SRV_WRN("failed to find free space in the KV cache, retrying with smaller batch size, i = %d, n_batch = %d, ret = %d\n", i, n_batch, ret);
continue; // continue loop of n_batch
}
// move the head of the batch forward with the number of tokens we just processed
i_next = i + n_tokens;
// on successful decode, restore the original batch size
n_batch = llama_n_batch(ctx);
for (auto & slot : slots) {
if (slot.i_batch < (int) i || slot.i_batch >= (int) (i + n_tokens)) {
continue; // continue loop of slots

27
tools/server/webui/src/App.tsx
View file

@ -5,21 +5,24 @@ import { AppContextProvider, useAppContext } from './utils/app.context';
import ChatScreen from './components/ChatScreen';
import SettingDialog from './components/SettingDialog';
import { Toaster } from 'react-hot-toast';
import { ModalProvider } from './components/ModalProvider';
function App() {
return (
<HashRouter>
<div className="flex flex-row drawer lg:drawer-open">
<AppContextProvider>
<Routes>
<Route element={<AppLayout />}>
<Route path="/chat/:convId" element={<ChatScreen />} />
<Route path="*" element={<ChatScreen />} />
</Route>
</Routes>
</AppContextProvider>
</div>
</HashRouter>
<ModalProvider>
<HashRouter>
<div className="flex flex-row drawer lg:drawer-open">
<AppContextProvider>
<Routes>
<Route element={<AppLayout />}>
<Route path="/chat/:convId" element={<ChatScreen />} />
<Route path="*" element={<ChatScreen />} />
</Route>
</Routes>
</AppContextProvider>
</div>
</HashRouter>
</ModalProvider>
);
}

151
tools/server/webui/src/components/ModalProvider.tsx Normal file
View file

@ -0,0 +1,151 @@
import React, { createContext, useState, useContext } from 'react';
type ModalContextType = {
showConfirm: (message: string) => Promise<boolean>;
showPrompt: (
message: string,
defaultValue?: string
) => Promise<string | undefined>;
showAlert: (message: string) => Promise<void>;
};
const ModalContext = createContext<ModalContextType>(null!);
interface ModalState<T> {
isOpen: boolean;
message: string;
defaultValue?: string;
resolve: ((value: T) => void) | null;
}
export function ModalProvider({ children }: { children: React.ReactNode }) {
const [confirmState, setConfirmState] = useState<ModalState<boolean>>({
isOpen: false,
message: '',
resolve: null,
});
const [promptState, setPromptState] = useState<
ModalState<string | undefined>
>({ isOpen: false, message: '', resolve: null });
const [alertState, setAlertState] = useState<ModalState<void>>({
isOpen: false,
message: '',
resolve: null,
});
const inputRef = React.useRef<HTMLInputElement>(null);
const showConfirm = (message: string): Promise<boolean> => {
return new Promise((resolve) => {
setConfirmState({ isOpen: true, message, resolve });
});
};
const showPrompt = (
message: string,
defaultValue?: string
): Promise<string | undefined> => {
return new Promise((resolve) => {
setPromptState({ isOpen: true, message, defaultValue, resolve });
});
};
const showAlert = (message: string): Promise<void> => {
return new Promise((resolve) => {
setAlertState({ isOpen: true, message, resolve });
});
};
const handleConfirm = (result: boolean) => {
confirmState.resolve?.(result);
setConfirmState({ isOpen: false, message: '', resolve: null });
};
const handlePrompt = (result?: string) => {
promptState.resolve?.(result);
setPromptState({ isOpen: false, message: '', resolve: null });
};
const handleAlertClose = () => {
alertState.resolve?.();
setAlertState({ isOpen: false, message: '', resolve: null });
};
return (
<ModalContext.Provider value={{ showConfirm, showPrompt, showAlert }}>
{children}
{/* Confirm Modal */}
{confirmState.isOpen && (
<dialog className="modal modal-open z-[1100]">
<div className="modal-box">
<h3 className="font-bold text-lg">{confirmState.message}</h3>
<div className="modal-action">
<button
className="btn btn-ghost"
onClick={() => handleConfirm(false)}
>
Cancel
</button>
<button
className="btn btn-error"
onClick={() => handleConfirm(true)}
>
Confirm
</button>
</div>
</div>
</dialog>
)}
{/* Prompt Modal */}
{promptState.isOpen && (
<dialog className="modal modal-open z-[1100]">
<div className="modal-box">
<h3 className="font-bold text-lg">{promptState.message}</h3>
<input
type="text"
className="input input-bordered w-full mt-2"
defaultValue={promptState.defaultValue}
ref={inputRef}
onKeyDown={(e) => {
if (e.key === 'Enter') {
handlePrompt((e.target as HTMLInputElement).value);
}
}}
/>
<div className="modal-action">
<button className="btn btn-ghost" onClick={() => handlePrompt()}>
Cancel
</button>
<button
className="btn btn-primary"
onClick={() => handlePrompt(inputRef.current?.value)}
>
Submit
</button>
</div>
</div>
</dialog>
)}
{/* Alert Modal */}
{alertState.isOpen && (
<dialog className="modal modal-open z-[1100]">
<div className="modal-box">
<h3 className="font-bold text-lg">{alertState.message}</h3>
<div className="modal-action">
<button className="btn" onClick={handleAlertClose}>
OK
</button>
</div>
</div>
</dialog>
)}
</ModalContext.Provider>
);
}
export function useModals() {
const context = useContext(ModalContext);
if (!context) throw new Error('useModals must be used within ModalProvider');
return context;
}

14
tools/server/webui/src/components/SettingDialog.tsx
View file

@ -13,6 +13,7 @@ import {
SquaresPlusIcon,
} from '@heroicons/react/24/outline';
import { OpenInNewTab } from '../utils/common';
import { useModals } from './ModalProvider';
type SettKey = keyof typeof CONFIG_DEFAULT;
@ -282,14 +283,15 @@ export default function SettingDialog({
const [localConfig, setLocalConfig] = useState<typeof CONFIG_DEFAULT>(
JSON.parse(JSON.stringify(config))
);
const { showConfirm, showAlert } = useModals();
const resetConfig = () => {
if (window.confirm('Are you sure you want to reset all settings?')) {
const resetConfig = async () => {
if (await showConfirm('Are you sure you want to reset all settings?')) {
setLocalConfig(CONFIG_DEFAULT);
}
};
const handleSave = () => {
const handleSave = async () => {
// copy the local config to prevent direct mutation
const newConfig: typeof CONFIG_DEFAULT = JSON.parse(
JSON.stringify(localConfig)
@ -302,14 +304,14 @@ export default function SettingDialog({
const mustBeNumeric = isNumeric(CONFIG_DEFAULT[key as SettKey]);
if (mustBeString) {
if (!isString(value)) {
alert(`Value for ${key} must be string`);
await showAlert(`Value for ${key} must be string`);
return;
}
} else if (mustBeNumeric) {
const trimmedValue = value.toString().trim();
const numVal = Number(trimmedValue);
if (isNaN(numVal) || !isNumeric(numVal) || trimmedValue.length === 0) {
alert(`Value for ${key} must be numeric`);
await showAlert(`Value for ${key} must be numeric`);
return;
}
// force conversion to number
@ -317,7 +319,7 @@ export default function SettingDialog({
newConfig[key] = numVal;
} else if (mustBeBoolean) {
if (!isBoolean(value)) {
alert(`Value for ${key} must be boolean`);
await showAlert(`Value for ${key} must be boolean`);
return;
}
} else {

10
tools/server/webui/src/components/Sidebar.tsx
View file

@ -14,6 +14,7 @@ import {
import { BtnWithTooltips } from '../utils/common';
import { useAppContext } from '../utils/app.context';
import toast from 'react-hot-toast';
import { useModals } from './ModalProvider';
export default function Sidebar() {
const params = useParams();
@ -38,6 +39,7 @@ export default function Sidebar() {
StorageUtils.offConversationChanged(handleConversationChange);
};
}, []);
const { showConfirm, showPrompt } = useModals();
const groupedConv = useMemo(
() => groupConversationsByDate(conversations),
@ -130,7 +132,7 @@ export default function Sidebar() {
onSelect={() => {
navigate(`/chat/${conv.id}`);
}}
onDelete={() => {
onDelete={async () => {
if (isGenerating(conv.id)) {
toast.error(
'Cannot delete conversation while generating'
@ -138,7 +140,7 @@ export default function Sidebar() {
return;
}
if (
window.confirm(
await showConfirm(
'Are you sure to delete this conversation?'
)
) {
@ -167,14 +169,14 @@ export default function Sidebar() {
document.body.removeChild(a);
URL.revokeObjectURL(url);
}}
onRename={() => {
onRename={async () => {
if (isGenerating(conv.id)) {
toast.error(
'Cannot rename conversation while generating'
);
return;
}
const newName = window.prompt(
const newName = await showPrompt(
'Enter new name for the conversation',
conv.name
);