mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-08-16 04:05:15 +00:00
* feat(web-shell): stream-highlight code blocks and fix fence-language aliases - Highlight assistant code fences live as they stream in via @shikijs/stream, replacing the per-token whole-block re-tokenize that flickered between plain grey and colored and only settled at the end of the turn. - Fix fence-language aliases (`ts`/`js`/`py`/... and uppercase like `SQL`) that previously rendered unhighlighted because the supported-language gate only matched canonical Shiki ids. - Consolidate the static and streaming paths onto a single lazily-created JavaScript-regex-engine highlighter (no WASM, on-demand language loading) and hand off without a grey flash; bump shiki ^1 -> ^4 and add @shikijs/stream. * fix(web-shell): address review feedback on streaming code highlight - Stabilize the `code` renderer via an IsStreamingContext so the CodeBlock/StreamingCodeBlock instances are reused (not remounted) across the streaming→settled transition, keeping the no-flash hold effective; drop the now-redundant COMPONENTS_STREAMING split. - Log a warning when the streaming highlighter fails to load, instead of failing silently to plain text. - Export `enqueueSuffix` and add unit tests for its no-op/extend/diverge/throw branches, plus a streaming→settled hand-off render test. * fix(web-shell): harden code highlighter (review round 2) - Guard highlightToHtmlSync with try/catch so a synchronous tokenization error falls back to the async path instead of crashing the render tree. - Dedupe concurrent loadLanguage calls via a pending-languages map (two callers could previously both pass the `has` check and double-load). - Add unit tests for codeHighlighter: highlightToHtml output, the sync cold→warm transition, the 30K-char sync bail-out, and concurrent-load dedup. * fix(web-shell): guard oversized streaming fences + review cleanups - Fall back to the plain renderer while streaming once the block or its trailing line exceeds a threshold (`exceedsStreamingLimit`), so a large single-line / minified fence can't pin the UI via @shikijs/stream re-tokenizing the unstable current line on every chunk. - Drop `shell`/`zsh` from SUPPORTED_LANGUAGES — they were dead entries shadowed by the alias map (→ bash). - Tests: add `__resetForTesting` to drop cross-test ordering coupling; add highlighter retry / pending-language cleanup tests, `exceedsStreamingLimit` unit tests, an oversized-fence fallback test, and a highlighter-load-failure fallback test for StreamingCodeBlock. * refactor(web-shell): stream-highlight code blocks via synchronous per-chunk re-render (drop @shikijs/stream) --------- Co-authored-by: 易良 <1204183885@qq.com>
97 lines
3.4 KiB
TypeScript
97 lines
3.4 KiB
TypeScript
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
|
|
const mocks = vi.hoisted(() => ({
|
|
createHighlighter: vi.fn(),
|
|
loadLanguage: vi.fn(),
|
|
codeToHtml: vi.fn(() => '<pre class="shiki"></pre>'),
|
|
}));
|
|
|
|
vi.mock('shiki', () => ({
|
|
createHighlighter: mocks.createHighlighter,
|
|
createJavaScriptRegexEngine: () => ({}),
|
|
}));
|
|
|
|
const {
|
|
__resetForTesting,
|
|
getCachedHtml,
|
|
getCodeHighlighter,
|
|
highlightToHtmlSync,
|
|
SHIKI_CACHE_MAX,
|
|
} = await import('./codeHighlighter');
|
|
|
|
const THEME = 'github-dark-default';
|
|
|
|
beforeEach(() => {
|
|
__resetForTesting();
|
|
mocks.createHighlighter.mockReset();
|
|
mocks.loadLanguage.mockReset().mockResolvedValue(undefined);
|
|
mocks.codeToHtml.mockReset().mockReturnValue('<pre class="shiki"></pre>');
|
|
});
|
|
|
|
describe('codeHighlighter retry/cleanup contracts', () => {
|
|
it('does not cache a rejected highlighter promise — the next call retries', async () => {
|
|
mocks.createHighlighter.mockRejectedValueOnce(new Error('boom'));
|
|
mocks.createHighlighter.mockResolvedValue({
|
|
loadLanguage: mocks.loadLanguage,
|
|
codeToHtml: mocks.codeToHtml,
|
|
});
|
|
|
|
await expect(getCodeHighlighter('typescript')).rejects.toThrow('boom');
|
|
await expect(getCodeHighlighter('typescript')).resolves.toBeDefined();
|
|
expect(mocks.createHighlighter).toHaveBeenCalledTimes(2);
|
|
});
|
|
|
|
it('records a failed language and does not retry the load on the next call', async () => {
|
|
mocks.createHighlighter.mockResolvedValue({
|
|
loadLanguage: mocks.loadLanguage,
|
|
codeToHtml: mocks.codeToHtml,
|
|
});
|
|
mocks.loadLanguage.mockRejectedValue(new Error('lang fail'));
|
|
|
|
await expect(getCodeHighlighter('python')).rejects.toThrow('lang fail');
|
|
// The second call is short-circuited (no re-request) and still rejects.
|
|
await expect(getCodeHighlighter('python')).rejects.toThrow(
|
|
/previously failed/,
|
|
);
|
|
expect(mocks.loadLanguage).toHaveBeenCalledTimes(1);
|
|
// pendingLanguages was cleaned up (the failure didn't leave it stuck).
|
|
expect(mocks.loadLanguage).toHaveBeenCalledWith('python');
|
|
});
|
|
|
|
it('highlightToHtmlSync returns null when codeToHtml throws (warm but failing)', async () => {
|
|
mocks.createHighlighter.mockResolvedValue({
|
|
loadLanguage: mocks.loadLanguage,
|
|
codeToHtml: mocks.codeToHtml,
|
|
});
|
|
await getCodeHighlighter('typescript'); // warm the language
|
|
mocks.codeToHtml.mockImplementation(() => {
|
|
throw new Error('tokenize boom');
|
|
});
|
|
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
|
|
expect(
|
|
highlightToHtmlSync('const x = 1;', 'typescript', 'github-dark-default'),
|
|
).toBeNull();
|
|
expect(warn).toHaveBeenCalled();
|
|
warn.mockRestore();
|
|
});
|
|
|
|
it('evicts the oldest cache entry once past SHIKI_CACHE_MAX', async () => {
|
|
mocks.createHighlighter.mockResolvedValue({
|
|
loadLanguage: mocks.loadLanguage,
|
|
codeToHtml: mocks.codeToHtml,
|
|
});
|
|
await getCodeHighlighter('typescript'); // warm the language
|
|
|
|
// Cache one entry beyond the limit (each distinct code is a distinct key).
|
|
for (let i = 0; i <= SHIKI_CACHE_MAX; i++) {
|
|
highlightToHtmlSync(`code-${i}`, 'typescript', THEME);
|
|
}
|
|
|
|
// The first (oldest) entry was evicted; the most recent is still cached.
|
|
expect(getCachedHtml('code-0', 'typescript', THEME)).toBeNull();
|
|
expect(
|
|
getCachedHtml(`code-${SHIKI_CACHE_MAX}`, 'typescript', THEME),
|
|
).not.toBeNull();
|
|
});
|
|
});
|