From 3bbcf4dc3e1a0a49d674d79944a2ef3c429a04f2 Mon Sep 17 00:00:00 2001 From: alpha Date: Mon, 24 Aug 2026 14:29:51 -0400 Subject: [PATCH 01/28] feat(language): make the stubbed Language enum real It had one member and was read by nothing. Six now, mirrored across the two processes the way SuggestionMode is, plus the display metadata the picker needs: endonym first, since someone whose interview is in the wrong language recognises "Deutsch" before "German". Six because that is what AssemblyAI's universal-streaming-multilingual model transcribes. Offering one the ASR cannot hear would not degrade gracefully, it would answer a question that was never asked. configStore.getConfig resolves the value on the way out rather than on the way in. The disk holds whatever some build wrote, and every consumer reads through getConfig, so that is the one place an unknown code can be stopped before it reaches the ASR URL and three request bodies. Refs #24 Co-Authored-By: Claude Opus 5 --- src/main/store/config.store.ts | 16 ++++++++++--- src/main/types/language.ts | 38 ++++++++++++++++++++++++++++++ src/renderer/types/config.ts | 6 ++--- src/renderer/types/language.ts | 43 ++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) create mode 100644 src/main/types/language.ts create mode 100644 src/renderer/types/language.ts diff --git a/src/main/store/config.store.ts b/src/main/store/config.store.ts index 64930300..92816bac 100644 --- a/src/main/store/config.store.ts +++ b/src/main/store/config.store.ts @@ -6,11 +6,13 @@ import ElectronStore from 'electron-store'; import { OPACITY_DEFAULT } from '../consts.js'; +import { DEFAULT_LANGUAGE, Language, resolveLanguage } from '../types/language.js'; import { LLMConfig } from '../types/llm.js'; // Runtime configuration (matches Config type in frontend) export interface RuntimeConfig { - language: string; + /** Interview language: what the ASR transcribes and what suggestions come back in. */ + language: Language; sessionToken: string; rememberMe: boolean; email: string; @@ -36,7 +38,7 @@ export interface RuntimeConfig { // Default runtime configuration const DEFAULT_RUNTIME_CONFIG: RuntimeConfig = { - language: 'en', + language: DEFAULT_LANGUAGE, sessionToken: '', rememberMe: true, email: '', @@ -102,7 +104,15 @@ class ConfigStore { getConfig(): RuntimeConfig { const stored: StoredRuntime = { ...this.store.get('runtime', DEFAULT_RUNTIME_CONFIG) }; delete stored.interviewConf; - return { ...DEFAULT_RUNTIME_CONFIG, ...stored } as RuntimeConfig; + const config = { ...DEFAULT_RUNTIME_CONFIG, ...stored } as RuntimeConfig; + + // Resolved on the way out, not on the way in. The disk holds whatever some build wrote - + // a language a later release dropped, or one an older one never knew - and every consumer + // reads through here, so this is the one place that can stop an unknown code reaching the + // ASR URL and the request bodies. + config.language = resolveLanguage(config.language); + + return config; } /** diff --git a/src/main/types/language.ts b/src/main/types/language.ts new file mode 100644 index 00000000..6f7c1e29 --- /dev/null +++ b/src/main/types/language.ts @@ -0,0 +1,38 @@ +/** + * The language an interview runs in: what the ASR transcribes and what suggestions come back in. + * + * ISO 639-1 codes, mirroring `Language` in the backend's `app/schemas/language.py`. The set is + * exactly the six languages AssemblyAI's `universal-streaming-multilingual` model supports, and + * deliberately no wider: offering a language the transcription cannot deliver produces confident + * answers to a question that was never asked, which is worse than not offering it. + * + * `src/renderer/types/language.ts` carries the same enum plus the display metadata the picker + * needs, the way `SuggestionMode` is mirrored across the two processes. + */ +export enum Language { + English = 'en', + Spanish = 'es', + German = 'de', + French = 'fr', + Portuguese = 'pt', + Italian = 'it', +} + +export const DEFAULT_LANGUAGE = Language.English; + +const LANGUAGE_CODES = new Set(Object.values(Language)); + +/** + * Map a stored or incoming value onto the enum, falling back to English. + * + * The config store holds whatever was written to disk, which may be a language a later build + * removed or an older build never knew. Sending that through unchecked puts an unknown code on + * the ASR URL and in every request body, where the backend can only fall back anyway - so it + * resolves here, once, at the point the value leaves the store. + */ +export function resolveLanguage(raw: string | null | undefined): Language { + if (!raw) return DEFAULT_LANGUAGE; + + const normalized = raw.trim().toLowerCase(); + return LANGUAGE_CODES.has(normalized) ? (normalized as Language) : DEFAULT_LANGUAGE; +} diff --git a/src/renderer/types/config.ts b/src/renderer/types/config.ts index bafba163..3a279a37 100644 --- a/src/renderer/types/config.ts +++ b/src/renderer/types/config.ts @@ -1,10 +1,10 @@ +import type { Language } from './language'; import type { LLMConfig } from './llm'; -export enum Language { - English = 'en', -} +export type { Language }; export interface Config { + // Interview language: what the ASR transcribes and what suggestions come back in. language: Language; // Authentication diff --git a/src/renderer/types/language.ts b/src/renderer/types/language.ts new file mode 100644 index 00000000..9ee5c368 --- /dev/null +++ b/src/renderer/types/language.ts @@ -0,0 +1,43 @@ +/** + * Mirrors `Language` in src/main/types/language.ts, which mirrors the backend enum in turn. + */ +export enum Language { + English = 'en', + Spanish = 'es', + German = 'de', + French = 'fr', + Portuguese = 'pt', + Italian = 'it', +} + +export const DEFAULT_LANGUAGE = Language.English; + +export interface LanguageOption { + code: Language; + /** English name, for a user who has not found their language in the list yet. */ + name: string; + /** Endonym. Someone whose app is in the wrong language recognises this one first. */ + nativeName: string; + /** Two letters for the control bar, so the current language is readable without opening it. */ + short: string; +} + +/** + * Display order is the order of the enum, not alphabetical by either name: alphabetical differs + * per naming column, so one of the two would always read as scrambled. + */ +export const LANGUAGES: readonly LanguageOption[] = [ + { code: Language.English, name: 'English', nativeName: 'English', short: 'EN' }, + { code: Language.Spanish, name: 'Spanish', nativeName: 'Español', short: 'ES' }, + { code: Language.German, name: 'German', nativeName: 'Deutsch', short: 'DE' }, + { code: Language.French, name: 'French', nativeName: 'Français', short: 'FR' }, + { code: Language.Portuguese, name: 'Portuguese', nativeName: 'Português', short: 'PT' }, + { code: Language.Italian, name: 'Italian', nativeName: 'Italiano', short: 'IT' }, +]; + +const BY_CODE = new Map(LANGUAGES.map((option) => [option.code, option])); + +/** The option for a stored code, falling back to English for one this build does not know. */ +export function getLanguageOption(code: string | null | undefined): LanguageOption { + return (code ? BY_CODE.get(code) : undefined) ?? BY_CODE.get(DEFAULT_LANGUAGE)!; +} From d35527c2183f4898176a5ce93c80380a0c7c344a Mon Sep 17 00:00:00 2001 From: alpha Date: Mon, 24 Aug 2026 14:30:28 -0400 Subject: [PATCH 02/28] feat(suggestions): send the interview language on every request Live, action and summarize, off one field on the shared LLMRequest base so the three cannot drift. Optional on the wire because the backend defaults it, which keeps the client working against a deployment that predates it. Each service already reads the config store as it builds its request, so this is also what makes the setting changeable mid-interview for free: the next suggestion follows without anything being reconnected. The exported report goes with them. A Spanish interview summarised in English is a document the candidate cannot hand to anyone who was in it. Refs #24 Co-Authored-By: Claude Opus 5 --- src/main/services/suggestion-action.service.ts | 1 + src/main/services/suggestion-live.service.ts | 1 + src/main/services/tools.service.ts | 6 +++++- src/main/types/llm.ts | 7 +++++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/services/suggestion-action.service.ts b/src/main/services/suggestion-action.service.ts index a6ef0977..182bdce5 100644 --- a/src/main/services/suggestion-action.service.ts +++ b/src/main/services/suggestion-action.service.ts @@ -218,6 +218,7 @@ export class ActionSuggestionService { transcripts: transcripts.slice(-TRANSCRIPT_UPLOAD_LIMIT), image_names: [...this.uploadedImageNames], mode: conf.professionalMode ? SuggestionMode.Professional : SuggestionMode.Normal, + language: conf.language, }; const lastQuestion = this.getLastInterviewerQuestion(transcripts); diff --git a/src/main/services/suggestion-live.service.ts b/src/main/services/suggestion-live.service.ts index d7a0b6da..aa23f553 100644 --- a/src/main/services/suggestion-live.service.ts +++ b/src/main/services/suggestion-live.service.ts @@ -125,6 +125,7 @@ class LiveSuggestionService { transcripts: transcripts.slice(-TRANSCRIPT_UPLOAD_LIMIT), mode, turn_verdict: turnVerdict, + language: conf.language, }; armStallTimer(LIVE_SUGGESTION_TTFB_MS); diff --git a/src/main/services/tools.service.ts b/src/main/services/tools.service.ts index bf37643b..25fea073 100644 --- a/src/main/services/tools.service.ts +++ b/src/main/services/tools.service.ts @@ -22,10 +22,14 @@ class ToolsService { const suggestions = appStateService.getState().liveSuggestions; // Call the API to generate the summary text + const conf = configStore.getConfig(); const response = await this.llmApi.generateSummary({ - config: configStore.getConfig().llmConf, + config: conf.llmConf, username, transcripts, + // The exported report is written in the interview's language too. A Spanish interview + // summarised in English is a document the candidate cannot hand to anyone involved in it. + language: conf.language, } as GenerateSummarizeRequest); if (response.error) { throw new Error(response.error.message); diff --git a/src/main/types/llm.ts b/src/main/types/llm.ts index 0ec83eb8..8513e254 100644 --- a/src/main/types/llm.ts +++ b/src/main/types/llm.ts @@ -1,4 +1,5 @@ import { Transcript } from './app-state.js'; +import { Language } from './language.js'; export enum LLMProvider { OPENAI = 'openai', @@ -48,6 +49,12 @@ export interface LLMConfigValidationResult { export interface LLMRequest { config: LLMConfig | null; + + /** + * Interview language. Carried on the shared base so the three request kinds cannot drift, and + * defaulted server-side, so omitting it against an older deployment still means English. + */ + language?: Language; } /** From a52f210366cb104c9ccad501ddf2634219402ac9 Mon Sep 17 00:00:00 2001 From: alpha Date: Mon, 24 Aug 2026 14:30:28 -0400 Subject: [PATCH 03/28] feat(asr): open the transcription sockets on the session's language, and switch them mid-session ?language= on the streaming URL, and English sends no parameter at all rather than language=en - the backend treats an absent language as English and builds the URL it has always built, so a session that never touches the picker produces exactly the traffic it produced before. The language is a connection parameter, so changing it mid-interview means tearing both sockets down and re-opening them. The URL is therefore rebuilt per attempt rather than captured, and two guards keep that from leaving two sockets on one channel, one of them orphaned and still relaying audio into a dead session: - onclose ignores a close from a socket that is no longer this.ws. That is the tail of a replacement, not a disconnect. - setLanguage sets a switching flag so the ordinary backoff reconnect does not fire for the close it caused itself; it reconnects immediately instead of waiting out WS_RETRY_BASE_DELAY_MS. The in-flight utterance is still reported as disconnected, because a switch orphans it exactly the way a dropped connection does. Refs #24 Co-Authored-By: Claude Opus 5 --- src/renderer/hooks/use-assistant-service.ts | 6 +- .../services/live-transcription.service.ts | 86 +++++++++++++++++-- 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/src/renderer/hooks/use-assistant-service.ts b/src/renderer/hooks/use-assistant-service.ts index 265f8224..331d97cf 100644 --- a/src/renderer/hooks/use-assistant-service.ts +++ b/src/renderer/hooks/use-assistant-service.ts @@ -3,6 +3,7 @@ import { create } from 'zustand'; import { getElectron } from '@/lib/utils'; import { liveTranscriptionService } from '@/services/live-transcription.service'; import { RunningState } from '@/types/app-state'; +import { DEFAULT_LANGUAGE } from '@/types/language'; import { useConfigStore } from './use-config-store'; @@ -36,9 +37,12 @@ export const useAssistantService = create((set) => ({ // Start transcription services await electron.transcription.start(); + // The language the sockets open on. It can change mid-session after this, but only + // through useInterviewLanguage, which reconnects them - nothing else reads it again. await liveTranscriptionService.start( config?.audioInputDeviceName ?? '', - config?.sessionToken ?? '' + config?.sessionToken ?? '', + config?.language ?? DEFAULT_LANGUAGE ); // Sleep 3 seconds to ensure the assistant has fully started before allowing stop actions diff --git a/src/renderer/services/live-transcription.service.ts b/src/renderer/services/live-transcription.service.ts index 153d80db..561333ef 100644 --- a/src/renderer/services/live-transcription.service.ts +++ b/src/renderer/services/live-transcription.service.ts @@ -1,4 +1,5 @@ import { getElectron } from '@/lib/utils'; +import { DEFAULT_LANGUAGE, Language } from '@/types/language'; const SAMPLE_RATE = 16000; const MAX_WS_BUFFERED_BYTES = SAMPLE_RATE * 0.3; @@ -12,6 +13,18 @@ const BACKEND_BASE_URL = import.meta.env.DEV : 'https://api.powerinterviewai.com'; const STREAMING_URL = `${BACKEND_BASE_URL.replace('http', 'ws')}/api/asr/streaming`; +/** + * The streaming URL for one channel. + * + * English is sent as no parameter at all rather than as `language=en`. The backend treats an + * absent language as English and builds the AssemblyAI URL it has always built, so an English + * session stays byte-identical to what shipped before the picker existed. + */ +function buildStreamingUrl(language: Language): string { + if (language === DEFAULT_LANGUAGE) return STREAMING_URL; + return `${STREAMING_URL}?language=${encodeURIComponent(language)}`; +} + // Inline AudioWorklet processor (runs off the main thread) const AUDIO_WORKLET_CODE = ` class AudioSenderWorklet extends AudioWorkletProcessor { @@ -53,9 +66,15 @@ class AudioWsStream { private stopping = false; private reconnectTimer: number | null = null; + // Set while setLanguage() is tearing the socket down and bringing it back. The close it + // causes is not a disconnect, so the ordinary reconnect must not also fire: two connects in + // flight leave one socket orphaned and still relaying audio into a dead session. + private switching = false; + constructor( private readonly channel: Channel, private readonly stream: MediaStream, + private language: Language, private readonly onTranscript: (payload: { channel: Channel; type: 'partial' | 'final'; @@ -136,6 +155,37 @@ class AudioWsStream { this.ws = null; } + /** + * Re-open this channel's socket on a different language. + * + * The language is a connection parameter, so there is no way to change it in place: the socket + * has to go and come back. That costs a gap of a second or two in this channel's transcription + * and orphans whatever utterance was mid-flight, which is why the caller is expected to be a + * deliberate user action rather than anything automatic. + */ + async setLanguage(language: Language): Promise { + if (language === this.language) return; + this.language = language; + + // Not started yet, or already stopped: start() reads the field, so there is nothing to do. + if (!this.active || this.stopping) return; + + this.switching = true; + try { + // Cancel a pending backoff reconnect first, or it wakes up later and opens a second socket. + if (this.reconnectTimer !== null) { + window.clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + if (this.ws && this.ws.readyState < WebSocket.CLOSING) { + this.ws.close(); + } + await this.connectWithRetry(); + } finally { + this.switching = false; + } + } + private async connectWithRetry(): Promise { let lastError: unknown; for (let attempt = 0; attempt < WS_RETRY_MAX_ATTEMPTS; attempt++) { @@ -166,7 +216,9 @@ class AudioWsStream { private connectWebSocket(): Promise { return new Promise((resolve, reject) => { - const ws = new WebSocket(STREAMING_URL); + // Rebuilt per attempt rather than captured once, so a reconnect cannot outlive the + // language the session opened with. + const ws = new WebSocket(buildStreamingUrl(this.language)); this.ws = ws; let settled = false; @@ -219,13 +271,22 @@ class AudioWsStream { ws.onclose = () => { if (this.stopping || !this.active) return; + // A close from a socket that is no longer the current one is not a disconnect; it is the + // tail of a replacement that already happened. Reconnecting on it would clobber the live + // socket with a second one. + if (this.ws !== ws) return; // A reconnect starts a fresh backend session, so any in-flight utterance never gets its - // final. Tell main to close it out, or the orphaned partial gates live suggestions. + // final. Tell main to close it out, or the orphaned partial gates live suggestions. A + // language switch is a reconnect too, so this holds for it as well. getElectron() ?.transcription.channelDisconnected(this.channel) .catch((error) => console.error('Failed to report channel disconnect:', error)); + // setLanguage owns the reconnect in that case, and does it immediately rather than after + // the backoff delay this would wait out. + if (this.switching) return; + this.scheduleReconnect(); }; } @@ -275,7 +336,11 @@ class LiveTranscriptionService { private loopbackStream: MediaStream | null = null; private channels: AudioWsStream[] = []; - async start(audioInputDeviceName: string, sessionToken: string): Promise { + async start( + audioInputDeviceName: string, + sessionToken: string, + language: Language = DEFAULT_LANGUAGE + ): Promise { const electron = getElectron(); if (!electron) throw new Error('Electron API not available'); await electron.transcription.setSessionToken(sessionToken); @@ -316,12 +381,23 @@ class LiveTranscriptionService { await electron.transcription.ingest(payload); }; - const micChannel = new AudioWsStream('ch_1', this.micStream, onTranscript); - const loopbackChannel = new AudioWsStream('ch_0', this.loopbackStream, onTranscript); + const micChannel = new AudioWsStream('ch_1', this.micStream, language, onTranscript); + const loopbackChannel = new AudioWsStream('ch_0', this.loopbackStream, language, onTranscript); this.channels = [micChannel, loopbackChannel]; await Promise.all(this.channels.map((channel) => channel.start())); } + /** + * Switch both channels to a new language mid-session. + * + * A no-op when nothing is running: the channels array is empty until start(), and start() + * takes the language it is called with. Suggestions need no equivalent - every request reads + * the config store when it is built, so the next one already follows the new setting. + */ + async setLanguage(language: Language): Promise { + await Promise.all(this.channels.map((channel) => channel.setLanguage(language))); + } + async stop(): Promise { await Promise.all(this.channels.map((channel) => channel.stop())); this.channels = []; From 508d547953589265555ae4ad8c6f106865724466 Mon Sep 17 00:00:00 2001 From: alpha Date: Mon, 24 Aug 2026 14:31:56 -0400 Subject: [PATCH 04/28] feat(ui): interview language picker on the control bar Sits with Audio and Model, because it is an input as much as an output: it picks the speech model before it picks the answer's language. Unlike those two it stays live while the assistant runs. An interview that switches language is the case this control exists for and not one the candidate can prepare for by restarting, so it locks only through the transient Starting and Stopping states. The switch is not instant and the button says so. Suggestions follow at once; the ASR reconnects, so the trigger spins and the menu warns that the sentence being spoken may be cut short - a two-second hole in the transcript is alarming if it arrives unannounced mid-question. The setting is persisted before the reconnect and never rolled back on failure: reverting it would leave the user with no route to the language they picked, while leaving it set means stop-and-start recovers. The trigger carries the code next to the icon. A globe alone is only useful to someone who already knows what it is set to, which is the one question this control has to answer at a glance. Refs #24 Co-Authored-By: Claude Opus 5 --- .../components/custom/control-panel/index.tsx | 6 +- .../custom/control-panel/language-group.tsx | 126 ++++++++++++++++++ .../custom/documentation-dialog.tsx | 20 +++ src/renderer/hooks/use-interview-language.ts | 56 ++++++++ 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 src/renderer/components/custom/control-panel/language-group.tsx create mode 100644 src/renderer/hooks/use-interview-language.ts diff --git a/src/renderer/components/custom/control-panel/index.tsx b/src/renderer/components/custom/control-panel/index.tsx index dc18d938..04b99fd0 100644 --- a/src/renderer/components/custom/control-panel/index.tsx +++ b/src/renderer/components/custom/control-panel/index.tsx @@ -15,6 +15,7 @@ import { RunningState } from '@/types/app-state'; import PermissionGateDialog from '../permission-gate-dialog'; import ZoomControl from '../zoom-control'; import { AudioGroup } from './audio-group'; +import { LanguageGroup } from './language-group'; import { LLMGroup } from './llm-group'; import { MainGroup } from './main-group'; import { ProfessionalModeGroup } from './professional-mode-group'; @@ -160,13 +161,16 @@ export default function ControlPanel() {