diff --git a/.gitignore b/.gitignore index cbf41d6..151f095 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,8 @@ src-tauri/binaries/ffmpeg-* # Local-only Cargo config (contains absolute target-dir for external drive) .cargo/config.toml + +# Beads / Dolt files (added by bd init) +.dolt/ +*.db +.beads-credential-key diff --git a/docs/audio-pipeline-principles.md b/docs/audio-pipeline-principles.md index d3a9daa..e366d9a 100644 --- a/docs/audio-pipeline-principles.md +++ b/docs/audio-pipeline-principles.md @@ -52,7 +52,8 @@ Settings / Realtime Interpreter beta -> src/lib/realtimeInterpreter.ts -> realtime_interpreter.rs -> real mic + system audio adapters --> OpenAI Realtime sessions +-> Talkis Cloud WebSocket proxy +-> Gemini Live Translate sessions -> virtual mic output + local playback ``` @@ -187,25 +188,42 @@ If `max=-120.0 dBFS` and `frames_above_noise_floor=0`, the system track is silen ## Realtime Interpreter Realtime Interpreter is a separate beta audio path. It must not be wired into -`transcribeCallCaptureSession()` or stored as `source: "call"` history unless a -future feature explicitly adds realtime session history. +`transcribeCallCaptureSession()`. Realtime translation history is allowed only +from final live transcript segments collected by the interpreter session itself. Current contract: - Tauri commands live in `src-tauri/src/realtime_interpreter.rs`. - Frontend command wrappers live in `src/lib/realtimeInterpreter.ts`. - Settings UI lives in `src/windows/settings/tabs/RealtimeInterpreterTab.tsx`. +- V1 is cloud-only. The desktop must use the saved Talkis `deviceToken`; do not + expose Google API keys, BYOK, or direct Google WebSocket credentials in the UI. +- Desktop defaults to `wss://proxy.talkis.ru/api/realtime-translate`. +- The proxy is responsible for JWT/subscription checks and for bridging to + Gemini Live Translate with `gemini-3.5-live-translate-preview`. +- Gemini Live Translate input is raw PCM16, 16 kHz, mono, little-endian, sent as + `audio/pcm;rate=16000`. Output audio is raw PCM16, 24 kHz, mono, returned as + `audio/pcm;rate=24000`. +- Transcript events should carry both source and translated text from + `inputAudioTranscription` and `outputAudioTranscription`. - External virtual audio drivers are required for MVP routing: BlackHole on macOS, VB-CABLE on Windows, PipeWire virtual source/sink on Linux. -- The fixed first language pair is RU -> EN for the user's mic and EN -> RU for - system audio. -- Start requests must validate a virtual mic output, API/cloud credentials, +- Supported v1 language pairs keep Russian as the local user language: + RU -> EN / EN -> RU, RU -> ES / ES -> RU, RU -> DE / DE -> RU, and + RU -> zh-Hans / zh-Hans -> RU. +- Start requests must validate a virtual mic output, Talkis Cloud credentials, system-audio support, and a headphones/separate-output confirmation before any streaming begins. - `test_virtual_mic_output` may play a short test tone to the selected virtual output; it must not silently fall back to the real speakers. +- On stop, if final transcript segments exist, save a call-like history entry: + `source: "call"`, `fileName: "Перевод звонка"`, `callSessionId`, `duration`, + `status`, original text in `raw`, bilingual translated text in `cleaned`, and + optional `translation` metadata with provider/model/language pair/segments. +- Do not retry realtime translation from history unless raw audio tracks are + explicitly saved in a later feature. -Do not present realtime translation as fully running until both OpenAI Realtime +Do not present realtime translation as fully running until the Talkis proxy WebSocket transport and platform audio routing are implemented and verified. ## Speaker Diarization diff --git a/rules/style.rule.md b/rules/style.rule.md index 7d93616..3cfd31f 100644 --- a/rules/style.rule.md +++ b/rules/style.rule.md @@ -1,68 +1,136 @@ -Talkis Website Style Prompt (актуально) +# Talkis Product UI Style Rule -Используй этот промпт для генерации новых блоков или редактирования существующих секций лендинга Talkis. -Цель: визуально и типографически синхронизировать сайт с приложением (desktop UI). +Use this rule when creating or editing Talkis product UI: the desktop app, settings windows, +dashboard screens, auth pages, product landing sections, and reusable UI patterns inspired by +Talkis. The goal is one visual language across `talkis-pc` and `talkis-web`, not a one-off +landing-page prompt. -1) Бренд-идея -- Концепция: Pure Thought / "Чистая мысль". -- Тон: спокойный, премиальный, технологичный без визуального шума. -- Визуальная модель: швейцарский минимализм + мягкая "бумажная" теплая подложка. +## Design Direction -2) Обязательная типографика (как в приложении) -- Базовый sans: Inter (300, 400, 500, 600, 700). -- Брендовый sans для wordmark/логотипа: Manrope (600, 700, 800). -- Акцентный serif: Playfair Display (700, italic для эмоциональных акцентов). -- Не использовать Jost/Roboto/system-only как основу. +- Core idea: Pure Thought - calm, precise, premium, and low-noise. +- Visual model: warm paper-like product surfaces, Swiss restraint, black-and-white clarity. +- Product feel: utility first, quietly polished, fast to scan, no decorative clutter. +- Prefer real controls and states over marketing composition when building app or dashboard UI. -Подключение шрифтов (Google Fonts): -`Inter + Manrope + Playfair Display`. +## Typography -Роли шрифтов: -- Основной текст, формы, кнопки, интерфейсные подписи: Inter. -- Название бренда "Talkis" и "Talkis Engine": Manrope. -- Акцентные фразы и эмоциональные смысловые куски в hero/демо: Playfair Display. +- UI/body font: `Inter`, fallback `system-ui, sans-serif`. +- Brand/headline font: `Manrope` 700-800 when available; otherwise a bold neutral sans. +- Brand wordmark: uppercase, bold, tight but controlled tracking, usually `letter-spacing: -0.06em`. +- Product headings: bold sans, compact line height, usually `letter-spacing: -0.04em` in the app. +- Web text can use `letter-spacing: 0` where that is already the local convention. +- Do not base the identity on serif display faces, script faces, or decorative headline styles. -3) Цвета и токены (синхрон с приложением) -- Базовый фон: #FAF9F6. -- Мягкий фон: #F4F1EB. -- Основной текст: #000000. -- Средний текст: #39342D. -- Слабый текст: #5D564D. -- Акцент: #000000 / #FFFFFF. -- Danger (служебный): #8F2D20. +Reference tokens from `src/index.css`: -Границы и поверхности: -- Бордеры: rgba(0, 0, 0, 0.09) и rgba(0, 0, 0, 0.16) для усиленных состояний. -- Surface: rgba(255, 255, 255, 0.72) с blur-эффектом. +```css +--font: "Inter", system-ui, sans-serif; +--font-main: "Inter", system-ui, sans-serif; +--font-accent: "Manrope", "Inter", system-ui, sans-serif; +--font-brand: "Manrope", "Inter", system-ui, sans-serif; +``` -4) Формы, радиусы, сетка -- Общая пластика: скругления мягкие, без агрессивных углов. -- Радиусы: 8 / 12 / 16 px и pill = 999 px. -- Bubble радиус: 22px 22px 0 22px. -- Сложные секции: пропорция 60/40 допустима и поощряется. +## Color Tokens -5) Компонентные правила -- Кнопки: черные pill-кнопки, uppercase, плотный трекинг, уверенный контраст. -- Навигация: sticky + backdrop blur (~20px), тонкая граница снизу. -- Карточки/панели: светлые полупрозрачные поверхности, минимум теней. -- Списки: персональные без маркеров; корпоративные с длинным тире (—). -- Бабблы сообщений: черный фон, белый текст, двойные чек-марки в статусе. +- Main background: `#faf9f6`. +- Soft background: `#f4f1eb`. +- Primary text: `#000000`. +- Secondary text: `#39342d` in desktop UI, `#666666` in web surfaces. +- Tertiary text: `#5d564d`, `#847d73`, or `#999999` depending on local tokens. +- Primary accent: black foreground/action with white contrast. +- Danger: `#8f2d20`. +- Success: restrained green, for example `#1f5130`. -6) Анимация и динамика -- Только осмысленные микродвижения: reveal, pulse, cleanup-flow. -- Magic Cleanup: поэтапное зачеркивание шумовых слов (около 180мс шаг). -- Лого-эквалайзер: 5 полос с разными задержками, scaleY пульсация. -- Избегать "перегруженных" эффектов и декоративной анимации без функциональной роли. +Surfaces: -7) Адаптив -- Mobile-first: один столбец, аккуратное уменьшение типографики, без ломки ритма. -- На мобильном не терять контраст/читабельность, особенно в черных секциях. +- `--surface: rgba(255, 255, 255, 0.72)`. +- `--surface-hi: rgba(255, 255, 255, 0.88)`. +- `--surface-solid: #f8f5ef`. +- Borders: `rgba(0, 0, 0, 0.06)` to `rgba(0, 0, 0, 0.16)`. +- Use blur for floating surfaces and sticky nav, not heavy shadows. -8) Жесткие ограничения -- Не уводить палитру в фиолетовые/неоновые схемы. -- Не заменять брендовые шрифты на случайные аналоги. -- Не добавлять тяжелые тени "ради красоты". -- Не смешивать разные визуальные системы в одной секции. +## Shape, Spacing, And Density -9) Готовый промпт для генерации блока -"Сгенерируй секцию лендинга Talkis в стиле Pure Thought. Используй типографику как в приложении: Inter для UI-текста, Manrope для wordmark, Playfair Display для акцентов. Палитра: #FAF9F6 фон, #000000 акцент, #39342D вторичный текст. Поверхности полупрозрачные, тонкие бордеры, pill-кнопки, аккуратная анимация reveal. Сохрани мобильную адаптацию и контраст." +- Standard radii: `8px`, `10px`, `12px`, `16px`. +- App/window radius: `16px`. +- Pill controls: `999px`. +- Message bubble radius: `22px 22px 0 22px` in app patterns. +- App and dashboard surfaces should feel dense enough for repeat use. Avoid oversized empty cards. +- Use clear grid and table alignment for settings, billing, users, history, logs, and admin flows. + +## Components + +Buttons: + +- Primary CTA: black pill, white text, uppercase when the local UI uses uppercase buttons. +- Secondary CTA: transparent or light background with a thin black border. +- Product app buttons: use `.btn`, `.btn-primary`, `.btn-danger` patterns from `src/index.css`. +- Keep transitions subtle: background, border, color, small translate on hover. + +Navigation: + +- Web navigation: sticky top bar, `rgba(250, 249, 246, 0.8)`, `backdrop-filter: blur(20px)`, + thin bottom border. +- App sidebar/dashboard nav: light background, 10px radius, active state `rgba(0, 0, 0, 0.04)`, + never inverted black for ordinary active nav. +- Icons: lucide-style line icons, 18px default, slightly stronger stroke for active states. + +Cards and panels: + +- Use light panels with subtle borders and little or no shadow. +- Do not put large page sections inside floating cards. +- Use cards for repeated items, settings groups, modals, profile/billing summaries, and tables. +- Subscription/sidebar CTA blocks are light cards with black CTA buttons, not inverted black blocks. + +Forms and tables: + +- Inputs are quiet: transparent or white-tinted controls, thin borders, strong focus rings. +- Settings forms should be stacked, readable, and explicit about saved/default values. +- Tables use small uppercase headers, thin row borders, hover tint, and horizontal scroll on mobile. + +## Layout Patterns + +Desktop app: + +- Preserve the compact app frame, warm background, subtle grid texture, and blurred surfaces. +- Keep window and widget sizing conventions from the existing app instead of inventing new chrome. +- Settings views should optimize repeated work: sidebar navigation, readable panels, inline status, + clear destructive states, and restrained empty states. + +Web landing: + +- First viewport should immediately signal Talkis and the product. +- Use strong product screenshots, real UI, voice/widget visuals, or product-specific motion. +- Keep hero text direct. Supporting copy carries the value proposition. +- Avoid generic SaaS gradient heroes and stock-like decoration. + +Dashboard: + +- Use a sticky header, fixed-width sidebar on desktop, compact nav links, and a focused main column. +- Keep billing, users, profile, and admin flows work-focused: tables, forms, summaries, and clear CTAs. +- On mobile, collapse to a single column and keep header actions reachable without overflow. + +## Motion + +- Motion must support comprehension: reveal, voice pulse, cleanup flow, loading/progress, hover feedback. +- Use short timings and standard easing such as `cubic-bezier(0.22, 1, 0.36, 1)`. +- Avoid decorative effects that compete with text, controls, or product screenshots. + +## Implementation Rules + +- Prefer existing local tokens and classes before adding new styling primitives. +- In `talkis-pc`, use inline styles with CSS variables for React components unless the file already + uses shared CSS classes. +- In `talkis-web`, reuse `globals.css` utility classes such as `btn-black`, `btn-outline`, + `sticky-nav`, dashboard classes, and type classes before adding custom variants. +- Keep UI text Russian in product surfaces unless a specific screen is intentionally English. +- Test responsive text fit. Buttons, nav links, tables, and cards must not overflow their containers. +- Do not change unrelated layout systems when applying this style to one screen or component. + +## Anti-Patterns + +- Purple/blue neon palettes, glassmorphism-heavy dashboards, or generic AI gradients. +- Heavy drop shadows, deep card stacks, nested cards, and decorative blobs. +- Random fonts or mixed visual systems in one screen. +- Black inverted nav/sidebar active states for ordinary product navigation. +- Marketing-only sections when the requested output is an app, dashboard, or tool. diff --git a/skills/talkis-product-ui-style/SKILL.md b/skills/talkis-product-ui-style/SKILL.md new file mode 100644 index 0000000..eb66b00 --- /dev/null +++ b/skills/talkis-product-ui-style/SKILL.md @@ -0,0 +1,53 @@ +--- +name: talkis-product-ui-style +description: Apply the Talkis Pure Thought product UI language to apps, dashboards, settings screens, auth flows, landing sections, and components. Use when a user asks to make an interface in Talkis style, says сделать интерфейс в стиле Talkis, применить Pure Thought UI, перенести этот стиль в другое приложение, port the Talkis visual system to another product, or align typography, colors, cards, navigation, CTAs, motion, and responsive behavior with Talkis. +--- + +# Talkis Product UI Style + +## Overview + +Use this skill to translate the Talkis product style into practical UI decisions. Treat Talkis as +the reference product, but adapt the visual language to the target app instead of copying brand +content blindly. + +## Required Reference + +Before changing UI, read `references/talkis-product-ui.md`. It contains the portable tokens, +component rules, layout patterns, motion guidance, and anti-patterns that define the style. + +## Workflow + +1. Identify the target surface: desktop app, settings screen, dashboard, auth flow, landing + section, or component. +2. Inspect existing tokens, components, and layout conventions in the target codebase. +3. Map the Talkis reference values to local variables/classes first. Add new styling only when + the target app has no suitable primitive. +4. Apply the style narrowly to the requested surface. Preserve existing behavior, data flow, + routing, accessibility affordances, and framework conventions. +5. Verify responsive fit, text overflow, hover/focus states, and that the result reads as a + product UI rather than a generic marketing page. + +## Talkis Repo Usage + +In `talkis-pc`, treat `rules/style.rule.md` and `src/index.css` as the product style source of +truth. For React UI, prefer existing CSS variables and local inline-style conventions. + +In `talkis-web`, reuse `src/app/globals.css` classes such as `btn-black`, `btn-outline`, +`sticky-nav`, dashboard classes, and type utilities before creating variants. + +Keep `rules/style.rule.md`, this skill, and any installed copy in sync when changing the style +language. + +## Porting Outside Talkis + +Use the Talkis language as a style archetype: warm cappuccino surfaces, black-and-white contrast, +bold neutral typography, light cards, sticky blur nav, pill CTAs, quiet dashboard density, and +restrained motion. Do not introduce Talkis brand copy, logos, or product claims unless the user +explicitly wants a Talkis-branded artifact. + +## Output Expectations + +When delivering work, state which Talkis style primitives were applied and call out any local +deviations. For implementation tasks, include verification results for responsive layout and visual +states when feasible. diff --git a/skills/talkis-product-ui-style/agents/openai.yaml b/skills/talkis-product-ui-style/agents/openai.yaml new file mode 100644 index 0000000..717bf42 --- /dev/null +++ b/skills/talkis-product-ui-style/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Talkis Product UI Style" + short_description: "Apply Talkis-inspired product UI" + default_prompt: "Use $talkis-product-ui-style to apply the Talkis product UI language to this interface." diff --git a/skills/talkis-product-ui-style/references/talkis-product-ui.md b/skills/talkis-product-ui-style/references/talkis-product-ui.md new file mode 100644 index 0000000..399d703 --- /dev/null +++ b/skills/talkis-product-ui-style/references/talkis-product-ui.md @@ -0,0 +1,111 @@ +# Talkis Product UI Reference + +This reference captures the portable Talkis product UI language. Use it for Talkis screens and +for other products that should feel inspired by Talkis without copying brand content. + +## Style Thesis + +- Pure Thought: calm, precise, premium, and low-noise. +- Warm paper-like surfaces with black-and-white clarity. +- Product-first composition: useful controls, readable data, clear states, and restrained polish. +- Visual restraint over decoration. The interface should feel fast to scan and easy to operate. + +## Typography + +- Body/UI: `Inter`, then `system-ui, sans-serif`. +- Brand and headings: `Manrope` 700-800 when available, otherwise a bold neutral sans. +- Wordmark: uppercase, heavy weight, controlled tight tracking, usually around `-0.06em`. +- Product headings: compact line-height, bold sans, around `-0.04em` tracking in desktop UI. +- Web surfaces may keep `letter-spacing: 0` when already established by local CSS. +- Avoid serif display faces, script faces, and decorative headline styles as the identity base. + +## Core Tokens + +Use local token names when they exist. + +```css +--bg: #faf9f6; +--bg-soft: #f4f1eb; +--surface: rgba(255, 255, 255, 0.72); +--surface-hi: rgba(255, 255, 255, 0.88); +--surface-solid: #f8f5ef; +--surface-strong: #f1ede6; +--text-hi: #000000; +--text-mid: #39342d; +--text-low: #5d564d; +--text-faint: #847d73; +--accent: #000000; +--accent-contrast: #ffffff; +--border: rgba(0, 0, 0, 0.09); +--border-subtle: rgba(0, 0, 0, 0.06); +--border-strong: rgba(0, 0, 0, 0.16); +--radius-sm: 8px; +--radius-md: 12px; +--radius-lg: 16px; +--radius-pill: 999px; +``` + +For web landing/dashboard surfaces, `--text-mid: #666666` and `--text-low: #999999` are acceptable +when they match the existing web CSS. + +## Components + +Buttons: + +- Primary: black pill or rounded button, white text, high contrast. +- Secondary: transparent/light surface, thin black border, black text. +- Product app buttons can use light surfaces by default and reserve black for primary actions. +- Hover: small translate, slightly stronger border or fill. Avoid bouncy or showy motion. + +Navigation: + +- Sticky web nav: cappuccino translucent background, `backdrop-filter: blur(20px)`, thin bottom border. +- Product/sidebar nav: light active state `rgba(0, 0, 0, 0.04)`, 10px radius, no inverted black active item. +- Icons: line icons, 18px default, modest stroke-weight difference between inactive and active. + +Cards and panels: + +- Use light translucent or solid warm surfaces with thin borders. +- Prefer no shadow or a very soft one. Border and spacing should carry most separation. +- Avoid cards inside cards. Use cards for repeated items, settings groups, modals, summaries, and tables. +- Subscription CTA blocks are light cards with a black CTA, not black promotional blocks. + +Forms and tables: + +- Inputs are quiet and utilitarian: transparent or white-tinted controls, thin borders, clear focus. +- Settings forms should show saved/default values clearly and keep actions close to the setting. +- Tables use small uppercase headers, thin row borders, hover tint, and mobile horizontal scroll. + +## Layout Patterns + +Desktop app: + +- Keep a warm app frame, subtle texture/grid, blurred surfaces, compact controls, and clear side nav. +- Settings screens should be dense but calm: stacked groups, inline status, explicit errors, clear defaults. +- Preserve local app chrome and sizing constraints. + +Dashboard: + +- Sticky header, fixed sidebar on desktop, focused main column, table-first workflows. +- Use summaries and cards to support actions, not as decoration. +- Mobile collapses to one column with reachable actions and no text overflow. + +Landing/product pages: + +- First viewport should reveal the product or a concrete product state. +- Prefer screenshots, real UI states, voice/widget visuals, or product-specific motion. +- Keep hero copy direct. Do not replace the product with generic abstract decoration. + +## Motion + +- Motion supports comprehension: reveal, voice pulse, cleanup flow, loading/progress, hover feedback. +- Use short timings and standard easing such as `cubic-bezier(0.22, 1, 0.36, 1)`. +- Avoid animation that competes with reading or makes work surfaces feel unstable. + +## Anti-Patterns + +- Purple/blue neon AI palettes, heavy gradients, decorative blobs, and generic glass dashboards. +- Heavy drop shadows, deep card stacks, and page sections styled as floating cards. +- Random font mixing or decorative display typography as the core style. +- Inverted black sidebars/nav active states for ordinary product navigation. +- Marketing-only composition when the requested artifact is an app, dashboard, tool, or settings view. diff --git a/src-tauri/src/realtime_interpreter.rs b/src-tauri/src/realtime_interpreter.rs index b8e6ebd..b3031b5 100644 --- a/src-tauri/src/realtime_interpreter.rs +++ b/src-tauri/src/realtime_interpreter.rs @@ -13,9 +13,11 @@ use std::sync::{Arc, Mutex, OnceLock}; use std::time::Duration; use tauri::{AppHandle, Emitter}; -const DEFAULT_REALTIME_MODEL: &str = "gpt-realtime-translate"; -const ESTIMATED_ONE_DIRECTION_USD_PER_MIN: f32 = 0.034; -const ESTIMATED_TWO_DIRECTION_USD_PER_MIN: f32 = ESTIMATED_ONE_DIRECTION_USD_PER_MIN * 2.0; +const DEFAULT_REALTIME_MODEL: &str = "gemini-3.5-live-translate-preview"; +const DEFAULT_REALTIME_ENDPOINT: &str = "wss://proxy.talkis.ru/api/realtime-translate"; +const REALTIME_INPUT_MIME_TYPE: &str = "audio/pcm;rate=16000"; +#[cfg(test)] +const REALTIME_OUTPUT_MIME_TYPE: &str = "audio/pcm;rate=24000"; const TEST_TONE_DURATION_MS: u64 = 650; static STATUS: OnceLock> = OnceLock::new(); @@ -68,7 +70,6 @@ pub struct StartRealtimeInterpreterRequest { pub language_pair: RealtimeLanguagePair, #[serde(default)] pub api_mode: RealtimeApiMode, - pub api_key: Option, pub device_token: Option, pub model: Option, pub endpoint: Option, @@ -80,6 +81,9 @@ pub struct StartRealtimeInterpreterRequest { #[serde(rename_all = "snake_case")] pub enum RealtimeLanguagePair { RuEn, + RuEs, + RuDe, + RuZhHans, } impl Default for RealtimeLanguagePair { @@ -97,7 +101,44 @@ pub enum RealtimeApiMode { impl Default for RealtimeApiMode { fn default() -> Self { - Self::Api + Self::Cloud + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RealtimeLanguagePairConfig { + user_source_language: &'static str, + user_target_language: &'static str, + remote_source_language: &'static str, + remote_target_language: &'static str, +} + +fn language_pair_config(pair: &RealtimeLanguagePair) -> RealtimeLanguagePairConfig { + match pair { + RealtimeLanguagePair::RuEn => RealtimeLanguagePairConfig { + user_source_language: "ru", + user_target_language: "en", + remote_source_language: "en", + remote_target_language: "ru", + }, + RealtimeLanguagePair::RuEs => RealtimeLanguagePairConfig { + user_source_language: "ru", + user_target_language: "es", + remote_source_language: "es", + remote_target_language: "ru", + }, + RealtimeLanguagePair::RuDe => RealtimeLanguagePairConfig { + user_source_language: "ru", + user_target_language: "de", + remote_source_language: "de", + remote_target_language: "ru", + }, + RealtimeLanguagePair::RuZhHans => RealtimeLanguagePairConfig { + user_source_language: "ru", + user_target_language: "zh-Hans", + remote_source_language: "zh-Hans", + remote_target_language: "ru", + }, } } @@ -136,12 +177,12 @@ impl RealtimeInterpreterStatus { started_at: None, language_pair: RealtimeLanguagePair::RuEn, model: DEFAULT_REALTIME_MODEL.to_string(), - endpoint: None, + endpoint: Some(DEFAULT_REALTIME_ENDPOINT.to_string()), message: Some("Realtime Interpreter не запущен.".to_string()), last_error: None, user_to_remote: RealtimeInterpreterDirectionStatus::idle(), remote_to_user: RealtimeInterpreterDirectionStatus::idle(), - estimated_cost_usd_per_minute: ESTIMATED_TWO_DIRECTION_USD_PER_MIN, + estimated_cost_usd_per_minute: 0.0, session_restart_at: None, } } @@ -166,7 +207,8 @@ impl RealtimeInterpreterStatus { .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned); + .map(ToOwned::to_owned) + .or_else(|| Some(DEFAULT_REALTIME_ENDPOINT.to_string())); } status @@ -222,6 +264,12 @@ pub struct RealtimeInterpreterErrorEvent { pub struct RealtimeInterpreterPartialTextEvent { pub session_id: Option, pub direction: RealtimeInterpreterDirection, + pub source_text: Option, + pub translated_text: Option, + pub source_language: Option, + pub target_language: Option, + pub start_ms: Option, + pub end_ms: Option, pub text: String, pub is_final: bool, } @@ -266,18 +314,25 @@ pub async fn start_realtime_interpreter( if let Err(message) = validate_start_request(&req, &devices) { return fail_start(&app, &req, "validation_failed", message, true); } + let pair_config = language_pair_config(&req.language_pair); logger::log_info( "REALTIME_INTERPRETER", &format!( - "Validated start request: platform={}, model={}, language_pair={:?}, virtual_output={}", + "Validated cloud realtime start request: platform={}, endpoint={}, model={}, language_pair={:?}, user_route={}->{}, remote_route={}->{}, input_mime={}, virtual_output={}", platform_name(), + normalized_endpoint(&req), req.model .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) .unwrap_or(DEFAULT_REALTIME_MODEL), req.language_pair, + pair_config.user_source_language, + pair_config.user_target_language, + pair_config.remote_source_language, + pair_config.remote_target_language, + REALTIME_INPUT_MIME_TYPE, req.virtual_mic_output_device_id ), ); @@ -286,7 +341,7 @@ pub async fn start_realtime_interpreter( &app, &req, "transport_not_enabled", - "Realtime Interpreter подготовлен как отдельный модуль, но потоковый транспорт OpenAI WebSocket и platform audio routing ещё не включены в этой сборке.".to_string(), + "Cloud Realtime Translator валидирует Talkis Cloud session, но нативный WebSocket/audio routing к Talkis proxy ещё не включён в этой desktop-сборке.".to_string(), false, ) } @@ -365,29 +420,26 @@ fn validate_start_request( ); } - match req.api_mode { - RealtimeApiMode::Api => { - if req - .api_key - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .is_none() - { - return Err("Для API-режима укажите OpenAI API key в разделе моделей.".to_string()); - } - } - RealtimeApiMode::Cloud => { - if req - .device_token - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .is_none() - { - return Err("Для облачного режима войдите в аккаунт Talkis.".to_string()); - } - } + if !matches!(req.api_mode, RealtimeApiMode::Cloud) { + return Err( + "Realtime Translator v1 работает только через Talkis Cloud. API/BYOK режим отключен." + .to_string(), + ); + } + + if req + .device_token + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .is_none() + { + return Err("Для облачного режима войдите в аккаунт Talkis.".to_string()); + } + + let endpoint = normalized_endpoint(req); + if !endpoint.starts_with("wss://") { + return Err("Realtime proxy endpoint должен использовать wss://.".to_string()); } let virtual_output = devices @@ -457,6 +509,15 @@ fn validate_start_request( Ok(()) } +fn normalized_endpoint(req: &StartRealtimeInterpreterRequest) -> String { + req.endpoint + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(DEFAULT_REALTIME_ENDPOINT) + .to_string() +} + fn discover_realtime_audio_devices() -> RealtimeAudioDevices { let platform = platform_name().to_string(); let host = cpal::default_host(); @@ -606,7 +667,7 @@ fn system_audio_sources() -> Vec { is_default: true, is_virtual: false, driver_hint: Some("WASAPI loopback".to_string()), - supported: false, + supported: true, }], "linux" => vec![RealtimeAudioDevice { id: "linux-pipewire-monitor".to_string(), @@ -616,7 +677,7 @@ fn system_audio_sources() -> Vec { is_default: true, is_virtual: false, driver_hint: Some("PipeWire monitor source".to_string()), - supported: false, + supported: true, }], _ => Vec::new(), } @@ -838,10 +899,54 @@ fn platform_name() -> &'static str { } #[cfg(test)] -fn realtime_append_event(audio_bytes: &[u8]) -> serde_json::Value { +fn proxy_start_message( + session_id: &str, + language_pair: RealtimeLanguagePair, + platform: &str, +) -> serde_json::Value { json!({ - "type": "input_audio_buffer.append", + "type": "start", + "sessionId": session_id, + "languagePair": language_pair, + "platform": platform, + }) +} + +#[cfg(test)] +fn proxy_audio_append_message( + direction: RealtimeInterpreterDirection, + audio_bytes: &[u8], +) -> serde_json::Value { + json!({ + "type": "audio.append", + "direction": direction, "audio": base64::engine::general_purpose::STANDARD.encode(audio_bytes), + "mimeType": REALTIME_INPUT_MIME_TYPE, + }) +} + +#[cfg(test)] +fn proxy_stop_message() -> serde_json::Value { + json!({ + "type": "stop", + }) +} + +#[cfg(test)] +fn gemini_live_translate_setup_payload(target_language_code: &str) -> serde_json::Value { + json!({ + "setup": { + "model": format!("models/{}", DEFAULT_REALTIME_MODEL), + "generationConfig": { + "responseModalities": ["AUDIO"], + "inputAudioTranscription": {}, + "outputAudioTranscription": {}, + "translationConfig": { + "targetLanguageCode": target_language_code, + "echoTargetLanguage": true, + }, + }, + }, }) } @@ -934,11 +1039,80 @@ mod tests { use super::*; #[test] - fn input_audio_append_event_frames_base64_audio() { - let event = realtime_append_event(&[0, 1, 2, 3]); + fn proxy_start_message_uses_public_desktop_contract() { + let event = proxy_start_message("rt-1", RealtimeLanguagePair::RuZhHans, "macos"); - assert_eq!(event["type"], "input_audio_buffer.append"); + assert_eq!(event["type"], "start"); + assert_eq!(event["sessionId"], "rt-1"); + assert_eq!(event["languagePair"], "ru_zh_hans"); + assert_eq!(event["platform"], "macos"); + } + + #[test] + fn proxy_audio_append_message_frames_base64_pcm_audio() { + let event = + proxy_audio_append_message(RealtimeInterpreterDirection::RemoteToUser, &[0, 1, 2, 3]); + + assert_eq!(event["type"], "audio.append"); + assert_eq!(event["direction"], "remote_to_user"); assert_eq!(event["audio"], "AAECAw=="); + assert_eq!(event["mimeType"], REALTIME_INPUT_MIME_TYPE); + assert_eq!(REALTIME_OUTPUT_MIME_TYPE, "audio/pcm;rate=24000"); + } + + #[test] + fn proxy_stop_message_uses_public_desktop_contract() { + let event = proxy_stop_message(); + + assert_eq!(event["type"], "stop"); + } + + #[test] + fn gemini_setup_payload_enables_audio_translation_and_transcripts() { + let payload = gemini_live_translate_setup_payload("es"); + + assert_eq!( + payload["setup"]["model"], + format!("models/{}", DEFAULT_REALTIME_MODEL) + ); + assert_eq!( + payload["setup"]["generationConfig"]["responseModalities"][0], + "AUDIO" + ); + assert!(payload["setup"]["generationConfig"]["inputAudioTranscription"].is_object()); + assert!(payload["setup"]["generationConfig"]["outputAudioTranscription"].is_object()); + assert_eq!( + payload["setup"]["generationConfig"]["translationConfig"]["targetLanguageCode"], + "es" + ); + assert_eq!( + payload["setup"]["generationConfig"]["translationConfig"]["echoTargetLanguage"], + true + ); + } + + #[test] + fn language_pair_mapping_covers_supported_top_pairs() { + assert_eq!( + language_pair_config(&RealtimeLanguagePair::RuEn).user_target_language, + "en" + ); + assert_eq!( + language_pair_config(&RealtimeLanguagePair::RuEs).user_target_language, + "es" + ); + assert_eq!( + language_pair_config(&RealtimeLanguagePair::RuDe).user_target_language, + "de" + ); + assert_eq!( + language_pair_config(&RealtimeLanguagePair::RuZhHans).user_target_language, + "zh-Hans" + ); + assert_eq!( + language_pair_config(&RealtimeLanguagePair::RuZhHans).remote_target_language, + "ru" + ); } #[test] @@ -977,7 +1151,44 @@ mod tests { } #[test] - fn validation_requires_virtual_output_and_api_key() { + fn validation_requires_cloud_token_before_virtual_output() { + let devices = RealtimeAudioDevices { + platform: "macos".to_string(), + virtual_driver_name: "BlackHole".to_string(), + real_mics: vec![], + virtual_mic_outputs: vec![], + local_playback_outputs: vec![], + system_audio_sources: vec![RealtimeAudioDevice { + id: "macos-coreaudio-system-tap".to_string(), + label: "Системный звук macOS".to_string(), + kind: RealtimeAudioDeviceKind::SystemAudio, + platform: "macos".to_string(), + is_default: true, + is_virtual: false, + driver_hint: None, + supported: true, + }], + warnings: vec![], + }; + let req = StartRealtimeInterpreterRequest { + real_mic_device_id: None, + virtual_mic_output_device_id: "missing".to_string(), + local_playback_device_id: None, + language_pair: RealtimeLanguagePair::RuEn, + api_mode: RealtimeApiMode::Cloud, + device_token: None, + model: None, + endpoint: None, + headphones_confirmed: true, + }; + + let err = validate_start_request(&req, &devices).unwrap_err(); + + assert!(err.contains("войдите в аккаунт Talkis")); + } + + #[test] + fn validation_rejects_api_mode_for_v1() { let devices = RealtimeAudioDevices { platform: "macos".to_string(), virtual_driver_name: "BlackHole".to_string(), @@ -1002,8 +1213,7 @@ mod tests { local_playback_device_id: None, language_pair: RealtimeLanguagePair::RuEn, api_mode: RealtimeApiMode::Api, - api_key: Some("".to_string()), - device_token: None, + device_token: Some("device-token".to_string()), model: None, endpoint: None, headphones_confirmed: true, @@ -1011,7 +1221,7 @@ mod tests { let err = validate_start_request(&req, &devices).unwrap_err(); - assert!(err.contains("OpenAI API key")); + assert!(err.contains("API/BYOK режим отключен")); } #[test] @@ -1051,9 +1261,8 @@ mod tests { virtual_mic_output_device_id: "output-0-blackhole".to_string(), local_playback_device_id: Some("output-0-blackhole".to_string()), language_pair: RealtimeLanguagePair::RuEn, - api_mode: RealtimeApiMode::Api, - api_key: Some("sk-test".to_string()), - device_token: None, + api_mode: RealtimeApiMode::Cloud, + device_token: Some("device-token".to_string()), model: None, endpoint: None, headphones_confirmed: true, diff --git a/src/lib/realtimeInterpreter.ts b/src/lib/realtimeInterpreter.ts index b286a9b..dcd42a1 100644 --- a/src/lib/realtimeInterpreter.ts +++ b/src/lib/realtimeInterpreter.ts @@ -7,6 +7,12 @@ export const REALTIME_INTERPRETER_AUDIO_LEVEL_EVENT = "realtime_interpreter_audio_level"; export const REALTIME_INTERPRETER_ERROR_EVENT = "realtime_interpreter_error"; +export const REALTIME_TRANSLATE_ENDPOINT = + "wss://proxy.talkis.ru/api/realtime-translate"; +export const REALTIME_TRANSLATE_MODEL = "gemini-3.5-live-translate-preview"; +export const REALTIME_TRANSLATE_INPUT_MIME_TYPE = "audio/pcm;rate=16000"; +export const REALTIME_TRANSLATE_OUTPUT_MIME_TYPE = "audio/pcm;rate=24000"; + export type RealtimeAudioDeviceKind = | "realMic" | "virtualMicOutput" @@ -34,8 +40,8 @@ export interface RealtimeAudioDevices { warnings: string[]; } -export type RealtimeLanguagePair = "ru_en"; -export type RealtimeApiMode = "api" | "cloud"; +export type RealtimeLanguagePair = "ru_en" | "ru_es" | "ru_de" | "ru_zh_hans"; +export type RealtimeApiMode = "cloud"; export type RealtimeInterpreterState = | "idle" | "starting" @@ -44,13 +50,65 @@ export type RealtimeInterpreterState = | "error"; export type RealtimeInterpreterDirection = "user_to_remote" | "remote_to_user"; +export interface RealtimeLanguagePairDefinition { + id: RealtimeLanguagePair; + label: string; + userSourceLanguage: "ru"; + userTargetLanguage: string; + remoteSourceLanguage: string; + remoteTargetLanguage: "ru"; +} + +export const REALTIME_LANGUAGE_PAIRS: RealtimeLanguagePairDefinition[] = [ + { + id: "ru_en", + label: "RU ↔ EN", + userSourceLanguage: "ru", + userTargetLanguage: "en", + remoteSourceLanguage: "en", + remoteTargetLanguage: "ru", + }, + { + id: "ru_es", + label: "RU ↔ ES", + userSourceLanguage: "ru", + userTargetLanguage: "es", + remoteSourceLanguage: "es", + remoteTargetLanguage: "ru", + }, + { + id: "ru_de", + label: "RU ↔ DE", + userSourceLanguage: "ru", + userTargetLanguage: "de", + remoteSourceLanguage: "de", + remoteTargetLanguage: "ru", + }, + { + id: "ru_zh_hans", + label: "RU ↔ ZH", + userSourceLanguage: "ru", + userTargetLanguage: "zh-Hans", + remoteSourceLanguage: "zh-Hans", + remoteTargetLanguage: "ru", + }, +]; + +export function realtimeLanguagePairDefinition( + pair: RealtimeLanguagePair, +): RealtimeLanguagePairDefinition { + return ( + REALTIME_LANGUAGE_PAIRS.find((definition) => definition.id === pair) || + REALTIME_LANGUAGE_PAIRS[0] + ); +} + export interface StartRealtimeInterpreterRequest { realMicDeviceId?: string | null; virtualMicOutputDeviceId: string; localPlaybackDeviceId?: string | null; languagePair: RealtimeLanguagePair; apiMode: RealtimeApiMode; - apiKey?: string | null; deviceToken?: string | null; model?: string | null; endpoint?: string | null; @@ -93,6 +151,12 @@ export interface RealtimeInterpreterErrorEvent { export interface RealtimeInterpreterPartialTextEvent { sessionId?: string | null; direction: RealtimeInterpreterDirection; + sourceText?: string | null; + translatedText?: string | null; + sourceLanguage?: string | null; + targetLanguage?: string | null; + startMs?: number | null; + endMs?: number | null; text: string; isFinal: boolean; } @@ -104,6 +168,73 @@ export interface RealtimeInterpreterAudioLevelEvent { outputLevel: number; } +export interface RealtimeTranslateStartClientMessage { + type: "start"; + sessionId: string; + languagePair: RealtimeLanguagePair; + platform: string; +} + +export interface RealtimeTranslateAudioAppendClientMessage { + type: "audio.append"; + direction: RealtimeInterpreterDirection; + audio: string; + mimeType: typeof REALTIME_TRANSLATE_INPUT_MIME_TYPE; +} + +export interface RealtimeTranslateStopClientMessage { + type: "stop"; +} + +export type RealtimeTranslateClientMessage = + | RealtimeTranslateStartClientMessage + | RealtimeTranslateAudioAppendClientMessage + | RealtimeTranslateStopClientMessage; + +export interface RealtimeTranslateSessionStartedServerMessage { + type: "session.started"; + sessionId: string; + model: string; + languagePair: RealtimeLanguagePair; +} + +export interface RealtimeTranslateTranscriptServerMessage { + type: "transcript.partial" | "transcript.final"; + direction: RealtimeInterpreterDirection; + sourceText: string; + translatedText: string; + sourceLanguage: string; + targetLanguage: string; + startMs?: number; + endMs?: number; +} + +export interface RealtimeTranslateAudioServerMessage { + type: "audio"; + direction: RealtimeInterpreterDirection; + audio: string; + mimeType: typeof REALTIME_TRANSLATE_OUTPUT_MIME_TYPE; +} + +export interface RealtimeTranslateErrorServerMessage { + type: "error"; + code: string; + message: string; + recoverable: boolean; +} + +export interface RealtimeTranslateSessionClosedServerMessage { + type: "session.closed"; + sessionId: string; +} + +export type RealtimeTranslateServerMessage = + | RealtimeTranslateSessionStartedServerMessage + | RealtimeTranslateTranscriptServerMessage + | RealtimeTranslateAudioServerMessage + | RealtimeTranslateErrorServerMessage + | RealtimeTranslateSessionClosedServerMessage; + export function listRealtimeAudioDevices(): Promise { return invoke("list_realtime_audio_devices"); } @@ -115,7 +246,9 @@ export function getRealtimeInterpreterStatus(): Promise { - return invoke("start_realtime_interpreter", { req }); + return invoke("start_realtime_interpreter", { + req, + }); } export function stopRealtimeInterpreter(): Promise { diff --git a/src/windows/settings/tabs/RealtimeInterpreterTab.tsx b/src/windows/settings/tabs/RealtimeInterpreterTab.tsx index cfe396a..902f22a 100644 --- a/src/windows/settings/tabs/RealtimeInterpreterTab.tsx +++ b/src/windows/settings/tabs/RealtimeInterpreterTab.tsx @@ -15,12 +15,18 @@ import { } from "lucide-react"; import { + addHistoryEntry, AppSettings, getSettings, + HistoryEntry, RealtimeInterpreterSettings, + RealtimeTranslationSegment, saveSettings, } from "../../../lib/store"; -import { SETTINGS_UPDATED_EVENT } from "../../../lib/hotkeyEvents"; +import { + HISTORY_UPDATED_EVENT, + SETTINGS_UPDATED_EVENT, +} from "../../../lib/hotkeyEvents"; import { logError } from "../../../lib/logger"; import { getRealtimeInterpreterStatus, @@ -29,12 +35,16 @@ import { REALTIME_INTERPRETER_ERROR_EVENT, REALTIME_INTERPRETER_PARTIAL_TEXT_EVENT, REALTIME_INTERPRETER_STATUS_EVENT, + REALTIME_LANGUAGE_PAIRS, + REALTIME_TRANSLATE_ENDPOINT, + REALTIME_TRANSLATE_MODEL, RealtimeAudioDevice, RealtimeAudioDevices, RealtimeInterpreterAudioLevelEvent, RealtimeInterpreterErrorEvent, RealtimeInterpreterPartialTextEvent, RealtimeInterpreterStatus, + realtimeLanguagePairDefinition, startRealtimeInterpreter, stopRealtimeInterpreter, testVirtualMicOutput, @@ -91,11 +101,6 @@ function statusColor(status: RealtimeInterpreterStatus | null): string { return "var(--text-low)"; } -function formatCost(value: number | undefined): string { - if (!Number.isFinite(value)) return "$0.068/min"; - return `$${Number(value).toFixed(3)}/min`; -} - function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -117,11 +122,6 @@ function preferredPlaybackDevice( ); } -function resolveOpenAiApiKey(settings: AppSettings): string { - const openAiAdapterKey = settings.apiAdapters?.openai?.apiKey?.trim(); - return openAiAdapterKey || settings.apiKey.trim(); -} - function SettingField({ label, note, @@ -138,7 +138,9 @@ function SettingField({ {label} {note && ( -
+
{note}
)} @@ -148,6 +150,54 @@ function SettingField({ ); } +function formatDurationSeconds(startedAt?: string | null): number { + if (!startedAt) return 0; + const started = Date.parse(startedAt); + if (!Number.isFinite(started)) return 0; + return Math.max(0, Math.round((Date.now() - started) / 1000)); +} + +function directionSpeakerLabel( + direction: RealtimeTranslationSegment["direction"], +): string { + return direction === "user_to_remote" ? "Вы" : "Собеседник"; +} + +function formatRealtimeRawTranscript( + segments: RealtimeTranslationSegment[], +): string { + return segments + .map( + (segment) => + `${directionSpeakerLabel(segment.direction)} (${segment.sourceLanguage}): ${segment.sourceText}`, + ) + .join("\n"); +} + +function formatRealtimeBilingualTranscript( + segments: RealtimeTranslationSegment[], +): string { + return segments + .map( + (segment) => + `${directionSpeakerLabel(segment.direction)}: ${segment.sourceText}\nПеревод (${segment.targetLanguage}): ${segment.translatedText}`, + ) + .join("\n\n"); +} + +function transcriptDisplayText( + payload: RealtimeInterpreterPartialTextEvent, +): string { + const source = payload.sourceText?.trim(); + const translated = payload.translatedText?.trim(); + + if (source && translated) { + return `${source}\n→ ${translated}`; + } + + return translated || source || payload.text; +} + function DeviceSelect({ value, onChange, @@ -237,8 +287,12 @@ function LevelMeter({ return (
-
- +
+ {label} = safeInput ? "var(--success-bright)" : "var(--text-hi)", + safeOutput >= safeInput + ? "var(--success-bright)" + : "var(--text-hi)", transition: "width 0.12s ease", }} /> @@ -290,11 +346,14 @@ export function RealtimeInterpreterTab(): ReactElement { userToRemote: "", remoteToUser: "", }); + const [finalSegments, setFinalSegments] = useState< + RealtimeTranslationSegment[] + >([]); const interpreterSettings = settings?.realtimeInterpreter; - const apiKey = settings ? resolveOpenAiApiKey(settings) : ""; const hasCloudToken = Boolean(settings?.deviceToken); - const apiMode = interpreterSettings?.apiMode || "api"; + const languagePair = interpreterSettings?.languagePair || "ru_en"; + const pairDefinition = realtimeLanguagePairDefinition(languagePair); const selectedVirtualOutput = devices?.virtualMicOutputs.find( (device) => device.id === interpreterSettings?.virtualMicOutputDeviceId, ); @@ -366,12 +425,40 @@ export function RealtimeInterpreterTab(): ReactElement { const textListener = listen( REALTIME_INTERPRETER_PARTIAL_TEXT_EVENT, ({ payload }) => { + const displayText = transcriptDisplayText(payload); setLiveText((current) => ({ ...current, [payload.direction === "user_to_remote" ? "userToRemote" - : "remoteToUser"]: payload.text, + : "remoteToUser"]: displayText, })); + if (!payload.isFinal) return; + + const sourceText = payload.sourceText?.trim() || ""; + const translatedText = + payload.translatedText?.trim() || payload.text.trim(); + if (!sourceText && !translatedText) return; + + setFinalSegments((current) => [ + ...current, + { + direction: payload.direction, + startMs: payload.startMs ?? undefined, + endMs: payload.endMs ?? undefined, + sourceText, + translatedText, + sourceLanguage: + payload.sourceLanguage || + (payload.direction === "user_to_remote" + ? pairDefinition.userSourceLanguage + : pairDefinition.remoteSourceLanguage), + targetLanguage: + payload.targetLanguage || + (payload.direction === "user_to_remote" + ? pairDefinition.userTargetLanguage + : pairDefinition.remoteTargetLanguage), + }, + ]); }, ); @@ -381,7 +468,7 @@ export function RealtimeInterpreterTab(): ReactElement { void levelListener.then((unlisten) => unlisten()); void textListener.then((unlisten) => unlisten()); }; - }, []); + }, [pairDefinition]); useEffect(() => { if (!settings || !devices) return; @@ -424,8 +511,8 @@ export function RealtimeInterpreterTab(): ReactElement { setPatch("localPlaybackDeviceId", preferred?.id || ""); } - if (current.apiMode === "cloud" && !settings.deviceToken) { - setPatch("apiMode", "api"); + if (current.apiMode !== "cloud") { + setPatch("apiMode", "cloud"); } if (Object.keys(patch).length === 0) return; @@ -463,23 +550,22 @@ export function RealtimeInterpreterTab(): ReactElement { const warnings = useMemo(() => { const result = [...(devices?.warnings || [])]; - if (apiMode === "api" && !apiKey) { - result.push("OpenAI API key не указан в разделе «Модели»."); - } - if (apiMode === "cloud" && !hasCloudToken) { + if (!hasCloudToken) { result.push("Для облачного режима войдите в аккаунт Talkis."); } if (!interpreterSettings?.headphonesConfirmed) { - result.push("Для локального перевода нужны наушники или отдельный output."); + result.push( + "Для локального перевода нужны наушники или отдельный output.", + ); } return result; - }, [apiKey, apiMode, devices?.warnings, hasCloudToken, interpreterSettings]); + }, [devices?.warnings, hasCloudToken, interpreterSettings]); const canStart = Boolean( interpreterSettings && - selectedVirtualOutput && - interpreterSettings.headphonesConfirmed && - (apiMode === "cloud" ? hasCloudToken : apiKey), + selectedVirtualOutput && + interpreterSettings.headphonesConfirmed && + hasCloudToken, ); const handleStart = async (): Promise => { @@ -488,17 +574,22 @@ export function RealtimeInterpreterTab(): ReactElement { setActionPending(true); setError(null); setTestMessage(null); + setFinalSegments([]); + setLiveText({ + userToRemote: "", + remoteToUser: "", + }); try { const nextStatus = await startRealtimeInterpreter({ realMicDeviceId: interpreterSettings.realMicDeviceId || null, virtualMicOutputDeviceId: interpreterSettings.virtualMicOutputDeviceId, - localPlaybackDeviceId: interpreterSettings.localPlaybackDeviceId || null, - languagePair: "ru_en", - apiMode, - apiKey: apiMode === "api" ? apiKey : null, - deviceToken: apiMode === "cloud" ? settings.deviceToken : null, - model: "gpt-realtime-translate", - endpoint: "wss://api.openai.com/v1/realtime", + localPlaybackDeviceId: + interpreterSettings.localPlaybackDeviceId || null, + languagePair: interpreterSettings.languagePair, + apiMode: "cloud", + deviceToken: settings.deviceToken, + model: REALTIME_TRANSLATE_MODEL, + endpoint: REALTIME_TRANSLATE_ENDPOINT, headphonesConfirmed: interpreterSettings.headphonesConfirmed, }); setStatus(nextStatus); @@ -509,12 +600,42 @@ export function RealtimeInterpreterTab(): ReactElement { } }; + const saveRealtimeTranslationHistory = async ( + sessionStatus: RealtimeInterpreterStatus | null, + ): Promise => { + if (!sessionStatus?.sessionId || finalSegments.length === 0) return; + + const entry: HistoryEntry = { + id: crypto.randomUUID(), + timestamp: new Date().toISOString(), + duration: formatDurationSeconds(sessionStatus.startedAt), + raw: formatRealtimeRawTranscript(finalSegments), + cleaned: formatRealtimeBilingualTranscript(finalSegments), + source: "call", + fileName: "Перевод звонка", + callSessionId: sessionStatus.sessionId, + status: "completed", + translation: { + provider: "talkis-cloud", + model: sessionStatus.model || REALTIME_TRANSLATE_MODEL, + languagePair: sessionStatus.languagePair, + segments: finalSegments, + }, + }; + + await addHistoryEntry(entry); + await emit(HISTORY_UPDATED_EVENT, entry); + }; + const handleStop = async (): Promise => { + const sessionStatus = status; setActionPending(true); setError(null); try { const nextStatus = await stopRealtimeInterpreter(); + await saveRealtimeTranslationHistory(sessionStatus); setStatus(nextStatus); + setFinalSegments([]); } catch (stopError) { setError(errorMessage(stopError)); } finally { @@ -592,11 +713,8 @@ export function RealtimeInterpreterTab(): ReactElement { }} > - - + +
@@ -618,7 +736,7 @@ export function RealtimeInterpreterTab(): ReactElement {
+ + + + { - void updateInterpreterSettings({ virtualMicOutputDeviceId: value }); + void updateInterpreterSettings({ + virtualMicOutputDeviceId: value, + }); }} devices={devices?.virtualMicOutputs || []} emptyLabel={`Установите ${devices?.virtualDriverName || "virtual driver"}`} - disabled={loadPending || (devices?.virtualMicOutputs.length || 0) === 0} + disabled={ + loadPending || (devices?.virtualMicOutputs.length || 0) === 0 + } /> @@ -661,53 +811,33 @@ export function RealtimeInterpreterTab(): ReactElement { /> - +
- {[ - { id: "api", label: "API" }, - { id: "cloud", label: "Облако" }, - ].map((option) => { - const active = apiMode === option.id; - const disabled = option.id === "cloud" && !hasCloudToken; - - return ( - - ); - })} + Cloud proxy + + {hasCloudToken ? "Аккаунт подключен" : "Нужен вход"} +
@@ -750,7 +880,9 @@ export function RealtimeInterpreterTab(): ReactElement { >
-
+
Проверка и запуск
@@ -836,7 +968,11 @@ export function RealtimeInterpreterTab(): ReactElement { lineHeight: 1.5, }} > - + {error}
)} @@ -889,7 +1025,7 @@ export function RealtimeInterpreterTab(): ReactElement {
Вы → звонок
@@ -899,7 +1035,9 @@ export function RealtimeInterpreterTab(): ReactElement { padding: "10px 12px", borderRadius: 8, background: "var(--control-muted)", - color: liveText.userToRemote ? "var(--text-hi)" : "var(--text-low)", + color: liveText.userToRemote + ? "var(--text-hi)" + : "var(--text-low)", fontSize: 12, lineHeight: 1.5, }} @@ -914,7 +1052,7 @@ export function RealtimeInterpreterTab(): ReactElement {
Звонок → вы
@@ -924,7 +1062,9 @@ export function RealtimeInterpreterTab(): ReactElement { padding: "10px 12px", borderRadius: 8, background: "var(--control-muted)", - color: liveText.remoteToUser ? "var(--text-hi)" : "var(--text-low)", + color: liveText.remoteToUser + ? "var(--text-hi)" + : "var(--text-low)", fontSize: 12, lineHeight: 1.5, }} @@ -934,7 +1074,9 @@ export function RealtimeInterpreterTab(): ReactElement {
-
+
Virtual mic: {selectedVirtualOutput?.label || "не выбран"} · Playback:{" "} {selectedPlayback?.label || "системный output"}