diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d611858..e5f1d78 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,6 +43,36 @@ jobs: echo "${{ secrets.APPLE_API_KEY }}" | base64 --decode > "$RUNNER_TEMP/private_keys/AuthKey_${{ secrets.APPLE_API_KEY_ID }}.p8" chmod 600 "$RUNNER_TEMP/private_keys/AuthKey_${{ secrets.APPLE_API_KEY_ID }}.p8" + # macOS: the SpeechAnalyzer helper needs the macOS 26 SDK. The macos-latest + # image (macOS 26) already defaults to Xcode 26.x, so this normally just + # confirms the SDK and moves on. The switch is a fallback for the case that + # bit this image before: a default Xcode pinned to an older major. + - name: Verify macOS 26 SDK + if: matrix.os == 'macos-latest' + run: | + sdk=$(xcrun --show-sdk-version 2>/dev/null || echo 0) + major=$(printf '%s' "$sdk" | cut -d. -f1) + case "$major" in ''|*[!0-9]*) major=0 ;; esac + if [ "$major" -ge 26 ]; then + echo "Default toolchain SDK $sdk supports SpeechAnalyzer." + exit 0 + fi + echo "Default SDK is $sdk; looking for an Xcode 26+ install." + best=$(ls -d /Applications/Xcode_*.app 2>/dev/null \ + | sed 's|.*/Xcode_||; s|\.app$||' \ + | awk -F. '$1 >= 26' | sort -V | tail -1) + if [ -z "$best" ]; then + echo "::error::No Xcode 26+ on this runner; cannot build the SpeechAnalyzer helper." >&2 + ls -d /Applications/Xcode*.app 2>/dev/null >&2 || true + exit 1 + fi + sudo xcode-select -s "/Applications/Xcode_$best.app/Contents/Developer" + echo "Switched to Xcode $best (SDK $(xcrun --show-sdk-version))." + + - name: Build SpeechAnalyzer helper + if: matrix.os == 'macos-latest' + run: make -C native/speechanalyzer build + # macOS: build, sign (Developer ID), notarize, and publish to GitHub Releases. - name: Build, sign & publish (macOS) if: matrix.os == 'macos-latest' diff --git a/.gitignore b/.gitignore index 984463f..031016e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,7 @@ next-env.d.ts # runtime assets copied from node_modules on postinstall /public/vendor/ /public/coi-serviceworker.js + +# Swift SpeechAnalyzer helper build output (binary is copied to resources/bin) +/native/speechanalyzer/.build/ +/resources/bin/rescript-speechanalyzer diff --git a/PLAN.md b/PLAN.md index c7e00c9..995a5e9 100644 --- a/PLAN.md +++ b/PLAN.md @@ -102,10 +102,11 @@ Everything else is derived: 7. ✅ Flexible timeline editing: word-boundary drag, Split at playhead (scene boundaries), clip trim handles, manual cuts merged into export. 8. ✅ Electron desktop shell with signed macOS / Windows / Linux releases. +9. ✅ Native macOS SpeechAnalyzer as an optional transcription backend + (Whisper stays the default everywhere else). ## Future work -- Native macOS SpeechAnalyzer as an optional transcription backend. - Larger Whisper variants + language selection UI; local model import for air-gapped first runs. - Smarter export: stream-copy for keyframe-aligned segments, WebCodecs-based diff --git a/README.md b/README.md index 5fb3bd1..c7b3e31 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,15 @@ builds auto-update from GitHub Releases. Prefer the browser? Use the - 📦 **In-browser / desktop export** — frame-accurate MP4 (video) or M4A (audio) with ffmpeg.wasm - 🎧 **Audio files** — edit podcasts, voice notes, and interviews the same way as video - 🖥️ **Desktop app** — macOS, Windows, and Linux via Electron (signed + notarized on Mac) +- 🍎 **SpeechAnalyzer** — optional on-device transcription via Apple’s Speech framework (macOS 26+, desktop) ## Stack | Piece | Tech | | --- | --- | | App | [Next.js](https://nextjs.org) + React + TypeScript + Tailwind | -| Desktop | [Electron](https://www.electronjs.org/) + [electron-builder](https://www.electron.build/) (auto-update from GitHub Releases) | -| Transcription | [transformers.js](https://github.com/huggingface/transformers.js) running [`whisper-base_timestamped`](https://huggingface.co/onnx-community/whisper-base_timestamped) or [`whisper-small_timestamped`](https://huggingface.co/onnx-community/whisper-small_timestamped) (WebGPU with WASM fallback) in a Web Worker | +| Desktop | [Electron](https://www.electronjs.org/) + [electron-builder](https://www.electron.build/) (auto-update from GitHub Releases); optional macOS [SpeechAnalyzer](https://developer.apple.com/documentation/speech) helper | +| Transcription | [transformers.js](https://github.com/huggingface/transformers.js) Whisper (WebGPU/WASM) in a Web Worker, **or** Apple SpeechAnalyzer on macOS 26+ desktop | | Speaker labels | [`pyannote-segmentation-3.0`](https://huggingface.co/onnx-community/pyannote-segmentation-3.0) (ONNX) | | Media processing | [ffmpeg.wasm](https://ffmpegwasm.netlify.app/) (multi-threaded) for audio extraction and export | | State | zustand | @@ -63,6 +64,8 @@ builds auto-update from GitHub Releases. Prefer the browser? Use the npm install # also copies ffmpeg/onnxruntime WASM into public/vendor npm run dev # Next.js web app (http://localhost:3000) npm run electron:dev # Electron shell + Next.js dev server +# Optional (macOS 26+ only): build the SpeechAnalyzer helper +make -C native/speechanalyzer build npm run build # production web build npm run dist # unsigned desktop installers into dist/ npm run lint # eslint @@ -85,8 +88,8 @@ audio track. For desktop packaging, signing, and cutting releases, see 1. **Extract** — ffmpeg.wasm decodes the audio track to mono 16 kHz PCM. 2. **Transcribe** — Whisper runs in a Web Worker with `return_timestamps: "word"`, streaming text as it goes; pyannote assigns a speaker to every word. - Choose **Whisper Base**, **Whisper Small**, or **Import transcript** - (SRT / VTT / JSON) on the homepage. + On the macOS desktop app you can instead choose **SpeechAnalyzer** (Apple’s + on-device model). Or choose **Import transcript** (SRT / VTT / JSON). 3. **Edit** — deleting words produces "cut ranges" of the original media. The preview player skips them in real time and the timeline shows them in red. **Remove fillers** cuts every detected "um" / "uh" / etc. in one click. diff --git a/RELEASING.md b/RELEASING.md index 2671811..9267570 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -87,3 +87,7 @@ disabled in `npm run electron:dev` and only runs in packaged builds - Desktop builds set `NEXT_PUBLIC_ELECTRON=1` so the static export skips the COI service worker (headers come from the `app://` protocol). Google Analytics still loads in the desktop app the same as on the web. +- The macOS **SpeechAnalyzer** helper lives under `native/speechanalyzer/`. + Release builds on `macos-latest` run `make -C native/speechanalyzer build` + before packaging; the signed binary is copied to `resources/bin/` and shipped + via electron-builder `extraResources`. Requires Xcode 26 / macOS 26 SDK. diff --git a/components/Editor.tsx b/components/Editor.tsx index c65f2ca..1387981 100644 --- a/components/Editor.tsx +++ b/components/Editor.tsx @@ -1,9 +1,11 @@ "use client"; import { useEffect, useRef, useState } from "react"; +import { isSpeechAnalyzerModel, isWhisperModel } from "@/lib/models"; import { useEditorStore } from "@/lib/store"; import { extractAudio, getFFmpeg } from "@/lib/ffmpeg"; import { useTranscriber } from "@/hooks/useTranscriber"; +import { useSpeechAnalyzerTranscriber } from "@/hooks/useSpeechAnalyzer"; import TopBar from "./TopBar"; import UploadScreen from "./UploadScreen"; import TranscriptPanel from "./TranscriptPanel"; @@ -12,9 +14,8 @@ import Timeline from "./Timeline"; import ExportDialog from "./ExportDialog"; import GitHubLink from "./GitHubLink"; import { Download, Redo2, Undo2 } from "lucide-react"; -import { ModelOption, ModelOptionSeparator } from "./ModelSelector"; import ModelSelector from "./ModelSelector"; -import ImportTranscriptOption from "./ImportTranscriptOption"; +import TranscriptSourceOptions from "./TranscriptSourceOptions"; import LogoLoader from "./LogoLoader"; /** How long the desktop mode-change overlay stays up. Matches the macOS @@ -28,6 +29,7 @@ export default function Editor() { const skipTranscription = useEditorStore((s) => s.skipTranscription); const loadVideo = useEditorStore((s) => s.loadVideo); const { transcribe } = useTranscriber(); + const { transcribeFile: transcribeSpeechAnalyzer } = useSpeechAnalyzerTranscriber(); const canUndo = useEditorStore((s) => s.past.length > 0); const canRedo = useEditorStore((s) => s.future.length > 0); @@ -41,11 +43,13 @@ export default function Editor() { // Processing pipeline: load ffmpeg -> extract audio -> (maybe) transcribe. // Restored projects already have words; they only need PCM for the waveform. + // SpeechAnalyzer runs in the Electron main process (skips the Whisper worker). const startedFor = useRef(null); useEffect(() => { if (!videoFile || startedFor.current === videoFile) return; startedFor.current = videoFile; const restoreOnly = useEditorStore.getState().skipTranscription; + const model = useEditorStore.getState().model; (async () => { const s = useEditorStore.getState(); try { @@ -57,15 +61,19 @@ export default function Editor() { if (restoreOnly) { s.setStatus("ready"); s.setProgress({ message: "", value: null }); - } else { + } else if (isSpeechAnalyzerModel(model)) { + await transcribeSpeechAnalyzer(videoFile); + } else if (isWhisperModel(model)) { transcribe(audio, audio.length / 16000); + } else { + s.setError("Select a transcript source before dropping media."); } } catch (err) { console.error("Processing pipeline failed:", err); s.setError(err instanceof Error ? err.message : "Failed to process this file."); } })(); - }, [videoFile, skipTranscription, transcribe]); + }, [videoFile, skipTranscription, transcribe, transcribeSpeechAnalyzer]); // The desktop shell opens as a small upload window and grows once the // three-pane editor takes over (and shrinks back on "start over"). @@ -134,10 +142,7 @@ export default function Editor() { <> {isElectron && - - - - + } diff --git a/components/ImportTranscriptOption.tsx b/components/ImportTranscriptOption.tsx index 25228b2..c718983 100644 --- a/components/ImportTranscriptOption.tsx +++ b/components/ImportTranscriptOption.tsx @@ -7,7 +7,7 @@ import { parseTranscriptFile, TRANSCRIPT_ACCEPT, } from "@/lib/parseTranscript"; -import { isWhisperModel } from "@/lib/models"; +import { isSpeechAnalyzerModel, isWhisperModel } from "@/lib/models"; import { useEditorStore } from "@/lib/store"; import { ModelOption, @@ -39,7 +39,7 @@ export default function ImportTranscriptOption() { ModelOptionContextValue, "keepMenuOpen" | "closeMenu" | "select" > | null>(null); - const previousModelRef = useRef<"base" | "small">("base"); + const previousModelRef = useRef<"base" | "small" | "speechanalyzer">("base"); const pickGenRef = useRef(0); /** Reset import-pick state only — never touch the dropdown open state. */ @@ -52,11 +52,13 @@ export default function ImportTranscriptOption() { } }, [setModel]); - // If the user switches to Whisper while a picker/parse is in flight, invalidate - // so a late onChange/parse cannot flip model back to import. + // If the user switches to another transcript source while a picker/parse is in + // flight, invalidate so a late onChange/parse cannot flip model back to import. useEffect(() => { return useEditorStore.subscribe((state, prev) => { - if (!isWhisperModel(state.model) || state.model === prev.model) return; + const switchedAway = + isWhisperModel(state.model) || isSpeechAnalyzerModel(state.model); + if (!switchedAway || state.model === prev.model) return; pickGenRef.current += 1; queueMicrotask(() => { setPicking(false); @@ -170,7 +172,7 @@ export default function ImportTranscriptOption() { onSelect={(ctx) => { menuRef.current = ctx; const current = useEditorStore.getState().model; - if (isWhisperModel(current)) { + if (isWhisperModel(current) || isSpeechAnalyzerModel(current)) { previousModelRef.current = current; } // Do not set model to "import" until a file is chosen. Close the menu diff --git a/components/ModelSelector.tsx b/components/ModelSelector.tsx index 3486f6a..b1e0c50 100644 --- a/components/ModelSelector.tsx +++ b/components/ModelSelector.tsx @@ -18,7 +18,13 @@ import { Loader2, type LucideIcon, } from "lucide-react"; -import { MODELS, isWhisperModel, type ModelChoice } from "@/lib/models"; +import { + MODELS, + SPEECH_ANALYZER_INFO, + isSpeechAnalyzerModel, + isWhisperModel, + type ModelChoice, +} from "@/lib/models"; import { hydrateModelPreference, useEditorStore } from "@/lib/store"; export type ModelOptionContextValue = { @@ -167,6 +173,8 @@ export default function ModelSelector({ const activeTrigger = triggers[model]; // Prefer the option's registered trigger. Fall back carefully so an unmounted // custom option (e.g. import) never shows the raw id + default wave icon. + // SpeechAnalyzer shares the waveform icon with Whisper — both are "the app + // transcribes for you", as opposed to a user-supplied caption file. const TriggerIcon = activeTrigger?.icon ?? (model === "import" ? FileText : AudioLines); const triggerLabel = @@ -175,7 +183,9 @@ export default function ModelSelector({ ? MODELS[model].label : model === "import" ? "Import transcript" - : String(model)); + : isSpeechAnalyzerModel(model) + ? SPEECH_ANALYZER_INFO.label + : String(model)); // Always mount options (hidden when closed) so custom triggers stay registered. const options = children ?? ( @@ -256,9 +266,19 @@ export function ModelOption({ const selected = selector.value === id; const resolvedLabel = - label ?? (isWhisperModel(id) ? MODELS[id].label : id); + label ?? + (isWhisperModel(id) + ? MODELS[id].label + : isSpeechAnalyzerModel(id) + ? SPEECH_ANALYZER_INFO.label + : id); const resolvedMeta = - meta ?? (isWhisperModel(id) ? MODELS[id].size : undefined); + meta ?? + (isWhisperModel(id) + ? MODELS[id].size + : isSpeechAnalyzerModel(id) + ? SPEECH_ANALYZER_INFO.size + : undefined); const optionCtx = useMemo( () => ({ diff --git a/components/TranscriptSourceOptions.tsx b/components/TranscriptSourceOptions.tsx new file mode 100644 index 0000000..e7adb3e --- /dev/null +++ b/components/TranscriptSourceOptions.tsx @@ -0,0 +1,53 @@ +"use client"; + +import { useEffect } from "react"; +import { SPEECH_ANALYZER_INFO } from "@/lib/models"; +import { useEditorStore } from "@/lib/store"; +import { useSpeechAnalyzerAvailability } from "@/hooks/useSpeechAnalyzer"; +import { ModelOption, ModelOptionSeparator } from "./ModelSelector"; +import ImportTranscriptOption from "./ImportTranscriptOption"; + +/** + * The transcript-source rows shared by both ModelSelector call sites: the + * upload screen's inline header (web) and the Electron title bar. Keeping them + * in one component means a new backend only has to be added once. + * + * SpeechAnalyzer only appears once the Electron helper reports itself usable, + * so web builds and non-macOS desktops see just Whisper + import. + */ +export default function TranscriptSourceOptions() { + const speechAnalyzer = useSpeechAnalyzerAvailability(); + const model = useEditorStore((s) => s.model); + const setModel = useEditorStore((s) => s.setModel); + + // A persisted "speechanalyzer" preference can outlive the helper (moved to a + // web build, downgraded macOS, helper removed) — fall back to Whisper. + useEffect(() => { + if (speechAnalyzer.status === "unavailable" && model === "speechanalyzer") { + setModel("base"); + } + }, [speechAnalyzer.status, model, setModel]); + + return ( + <> + + + {speechAnalyzer.status === "available" && ( + <> + + + + {SPEECH_ANALYZER_INFO.description} + + + + )} + + + + ); +} diff --git a/components/UploadScreen.tsx b/components/UploadScreen.tsx index 2ae0376..f800910 100644 --- a/components/UploadScreen.tsx +++ b/components/UploadScreen.tsx @@ -16,11 +16,8 @@ import { } from "lucide-react"; import logo from "@/assets/logo.png"; import GitHubLink from "./GitHubLink"; -import ModelSelector, { - ModelOption, - ModelOptionSeparator, -} from "./ModelSelector"; -import ImportTranscriptOption from "./ImportTranscriptOption"; +import ModelSelector from "./ModelSelector"; +import TranscriptSourceOptions from "./TranscriptSourceOptions"; import { useCrossOriginIsolated } from "@/hooks/useCrossOriginIsolated"; import { detectMediaKind, MEDIA_ACCEPT } from "@/lib/media"; import { formatTime } from "@/lib/edits"; @@ -267,10 +264,7 @@ export default function UploadScreen({

Rescript

- - - - + }
) : ( diff --git a/electron/ipc/channels.ts b/electron/ipc/channels.ts new file mode 100644 index 0000000..578b694 --- /dev/null +++ b/electron/ipc/channels.ts @@ -0,0 +1,45 @@ +/** IPC channel names shared by Electron main + preload. */ +export const IPC = { + speechAnalyzerCheck: "speechAnalyzer:check", + speechAnalyzerTranscribe: "speechAnalyzer:transcribe", + speechAnalyzerProgress: "speechAnalyzer:progress", +} as const; + +export type SpeechAnalyzerCheckResult = { + available: boolean; + reason?: string; + locale?: string; + installedLocales?: string[]; + helperPath?: string | null; +}; + +export type SpeechAnalyzerWord = { + id: number; + text: string; + start: number; + end: number; + speaker: number; + deleted: boolean; +}; + +export type SpeechAnalyzerProgress = { + message: string; + value: number | null; +}; + +export type SpeechAnalyzerTranscribeRequest = { + /** Absolute path to a media file on disk (preferred in Electron). */ + path?: string; + /** Raw file bytes when no path is available (drag-drop without path). */ + data?: ArrayBuffer; + /** Filename hint used when writing a temp file from `data`. */ + name?: string; + locale?: string; +}; + +export type SpeechAnalyzerTranscribeResult = { + words: SpeechAnalyzerWord[]; + locale: string; + duration: number; + model: string; +}; diff --git a/electron/main.ts b/electron/main.ts index d736a83..5603a3f 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -3,6 +3,11 @@ import { join, normalize, extname } from "node:path"; import { pathToFileURL } from "node:url"; import { existsSync, statSync } from "node:fs"; import { initAutoUpdater } from "./updater"; +import { IPC, type SpeechAnalyzerTranscribeRequest } from "./ipc/channels"; +import { + checkSpeechAnalyzer, + transcribeWithSpeechAnalyzer, +} from "./speechAnalyzer"; const isDev = !app.isPackaged; const DEV_SERVER_URL = process.env.ELECTRON_START_URL ?? "http://localhost:3000"; @@ -240,6 +245,20 @@ if (!gotLock) { app.whenReady().then(() => { if (!isDev) registerAppProtocol(); + + ipcMain.handle(IPC.speechAnalyzerCheck, async () => checkSpeechAnalyzer()); + ipcMain.handle( + IPC.speechAnalyzerTranscribe, + async (event, req: SpeechAnalyzerTranscribeRequest) => { + const win = BrowserWindow.fromWebContents(event.sender); + return transcribeWithSpeechAnalyzer(req, (progress) => { + if (win && !win.isDestroyed()) { + win.webContents.send(IPC.speechAnalyzerProgress, progress); + } + }); + } + ); + createWindow(); initAutoUpdater(); diff --git a/electron/preload.ts b/electron/preload.ts index cc9482a..3e999cd 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,10 +1,18 @@ import { contextBridge, ipcRenderer, type IpcRendererEvent } from "electron"; +import { + IPC, + type SpeechAnalyzerCheckResult, + type SpeechAnalyzerProgress, + type SpeechAnalyzerTranscribeRequest, + type SpeechAnalyzerTranscribeResult, +} from "./ipc/channels"; /** * Minimal bridge for the renderer. Rescript's UI is still a normal web * surface; we only expose host metadata so the page can adapt chrome / skip * the COI service worker (headers come from the app:// protocol instead), - * plus the few window controls the page drives (sizing, title-bar state). + * the few window controls the page drives (sizing, title-bar state), and the + * SpeechAnalyzer helper IPC (macOS 26+). */ contextBridge.exposeInMainWorld("rescriptDesktop", { platform: process.platform as NodeJS.Platform, @@ -25,4 +33,24 @@ contextBridge.exposeInMainWorld("rescriptDesktop", { ipcRenderer.off("window:full-screen-changed", listener); }; }, + + speechAnalyzer: { + check(): Promise { + return ipcRenderer.invoke(IPC.speechAnalyzerCheck); + }, + transcribe( + req: SpeechAnalyzerTranscribeRequest + ): Promise { + return ipcRenderer.invoke(IPC.speechAnalyzerTranscribe, req); + }, + onProgress(handler: (progress: SpeechAnalyzerProgress) => void): () => void { + const listener = (_event: IpcRendererEvent, progress: SpeechAnalyzerProgress) => { + handler(progress); + }; + ipcRenderer.on(IPC.speechAnalyzerProgress, listener); + return () => { + ipcRenderer.removeListener(IPC.speechAnalyzerProgress, listener); + }; + }, + }, }); diff --git a/electron/speechAnalyzer.ts b/electron/speechAnalyzer.ts new file mode 100644 index 0000000..434ae2a --- /dev/null +++ b/electron/speechAnalyzer.ts @@ -0,0 +1,217 @@ +import { app } from "electron"; +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, writeFileSync, unlinkSync } from "node:fs"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import type { + SpeechAnalyzerCheckResult, + SpeechAnalyzerProgress, + SpeechAnalyzerTranscribeRequest, + SpeechAnalyzerTranscribeResult, + SpeechAnalyzerWord, +} from "./ipc/channels"; + +const HELPER_NAME = "rescript-speechanalyzer"; + +/** + * Resolve the SpeechAnalyzer helper binary. Packaged builds look under + * process.resourcesPath/bin; dev looks at resources/bin then the Swift + * build product under native/speechanalyzer/.build/release. + */ +export function resolveHelperPath(): string | null { + const candidates: string[] = []; + if (app.isPackaged) { + candidates.push(join(process.resourcesPath, "bin", HELPER_NAME)); + } else { + const root = join(__dirname, ".."); + candidates.push( + join(root, "resources", "bin", HELPER_NAME), + join(root, "native", "speechanalyzer", ".build", "release", HELPER_NAME) + ); + } + return candidates.find((p) => existsSync(p)) ?? null; +} + +function runHelper( + args: string[], + onProgress?: (p: SpeechAnalyzerProgress) => void +): Promise<{ code: number; stdout: string; stderr: string }> { + const helper = resolveHelperPath(); + if (!helper) { + return Promise.reject( + new Error( + "SpeechAnalyzer helper not found. On macOS 26+, build it with `make -C native/speechanalyzer build`." + ) + ); + } + + return new Promise((resolve, reject) => { + const child = spawn(helper, args, { stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (buf: Buffer) => { + stdout += buf.toString("utf8"); + }); + child.stderr.on("data", (buf: Buffer) => { + const chunk = buf.toString("utf8"); + stderr += chunk; + for (const line of chunk.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) continue; + try { + const msg = JSON.parse(trimmed) as { + type?: string; + message?: string; + value?: number | null; + }; + if (msg.type === "progress" && onProgress) { + onProgress({ + message: msg.message ?? "Transcribing…", + value: typeof msg.value === "number" ? msg.value : null, + }); + } + } catch { + // ignore non-JSON stderr + } + } + }); + child.on("error", reject); + child.on("close", (code) => { + resolve({ code: code ?? 1, stdout, stderr }); + }); + }); +} + +function parseLastJson(stdout: string): Record { + const lines = stdout + .split("\n") + .map((l) => l.trim()) + .filter(Boolean); + const last = lines[lines.length - 1]; + if (!last) throw new Error("SpeechAnalyzer helper produced no output."); + return JSON.parse(last) as Record; +} + +export async function checkSpeechAnalyzer(): Promise { + if (process.platform !== "darwin") { + return { + available: false, + reason: "SpeechAnalyzer is only available on macOS.", + helperPath: null, + }; + } + const helperPath = resolveHelperPath(); + if (!helperPath) { + return { + available: false, + reason: + "SpeechAnalyzer helper is not built yet. Run `make -C native/speechanalyzer build` on macOS 26+.", + helperPath: null, + }; + } + try { + const { code, stdout } = await runHelper(["--check"]); + const payload = parseLastJson(stdout); + if (payload.type === "error") { + return { + available: false, + reason: String(payload.message ?? "SpeechAnalyzer check failed."), + helperPath, + }; + } + return { + available: Boolean(payload.available) && code === 0, + reason: payload.reason ? String(payload.reason) : undefined, + locale: payload.locale ? String(payload.locale) : undefined, + installedLocales: Array.isArray(payload.installedLocales) + ? (payload.installedLocales as string[]) + : undefined, + helperPath, + }; + } catch (err) { + return { + available: false, + reason: err instanceof Error ? err.message : String(err), + helperPath, + }; + } +} + +function normalizeWords(raw: unknown): SpeechAnalyzerWord[] { + if (!Array.isArray(raw)) return []; + return raw.map((item, i) => { + const w = (item ?? {}) as Record; + const text = String(w.text ?? "").trim(); + const start = Number(w.start) || 0; + const end = Math.max(Number(w.end) || 0, start); + return { + id: typeof w.id === "number" ? w.id : i, + text, + start, + end, + speaker: typeof w.speaker === "number" ? w.speaker : 0, + deleted: Boolean(w.deleted), + }; + }).filter((w) => w.text.length > 0); +} + +async function materializePath( + req: SpeechAnalyzerTranscribeRequest +): Promise<{ path: string; cleanup?: string }> { + if (req.path) { + if (!existsSync(req.path)) { + throw new Error(`Media file not found: ${req.path}`); + } + return { path: req.path }; + } + if (!req.data) { + throw new Error("SpeechAnalyzer needs a file path or raw media bytes."); + } + const dir = join(tmpdir(), "rescript-speechanalyzer"); + mkdirSync(dir, { recursive: true }); + const ext = (req.name?.split(".").pop() || "bin").replace(/[^a-z0-9]/gi, "") || "bin"; + const tempPath = join(dir, `${randomUUID()}.${ext}`); + writeFileSync(tempPath, Buffer.from(req.data)); + return { path: tempPath, cleanup: tempPath }; +} + +export async function transcribeWithSpeechAnalyzer( + req: SpeechAnalyzerTranscribeRequest, + onProgress?: (p: SpeechAnalyzerProgress) => void +): Promise { + if (process.platform !== "darwin") { + throw new Error("SpeechAnalyzer is only available on macOS."); + } + const { path, cleanup } = await materializePath(req); + try { + const args = [path]; + if (req.locale) args.push("--locale", req.locale); + const { code, stdout } = await runHelper(args, onProgress); + const payload = parseLastJson(stdout); + if (payload.type === "error" || code !== 0) { + throw new Error(String(payload.message ?? `SpeechAnalyzer exited with code ${code}`)); + } + const words = normalizeWords(payload.words); + if (words.length === 0) { + throw new Error("SpeechAnalyzer returned no words."); + } + // Re-index ids contiguously for the editor. + const reindexed = words.map((w, i) => ({ ...w, id: i })); + return { + words: reindexed, + locale: String(payload.locale ?? "und"), + duration: Number(payload.duration) || 0, + model: String(payload.model ?? "SpeechAnalyzer/macOS26"), + }; + } finally { + if (cleanup) { + try { + unlinkSync(cleanup); + } catch { + // temp cleanup is best-effort + } + } + } +} diff --git a/hooks/useSpeechAnalyzer.ts b/hooks/useSpeechAnalyzer.ts new file mode 100644 index 0000000..a1f89c2 --- /dev/null +++ b/hooks/useSpeechAnalyzer.ts @@ -0,0 +1,113 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; + +export type SpeechAnalyzerAvailability = + | { status: "unknown" } + | { status: "unavailable"; reason?: string } + | { status: "available"; locale?: string }; + +function initialAvailability(): SpeechAnalyzerAvailability { + if (typeof window === "undefined") return { status: "unknown" }; + const desktop = window.rescriptDesktop; + if (!desktop?.speechAnalyzer || desktop.platform !== "darwin") { + return { + status: "unavailable", + reason: desktop ? "SpeechAnalyzer requires macOS." : "Desktop app only.", + }; + } + return { status: "unknown" }; +} + +/** + * Probe whether the Electron SpeechAnalyzer helper is usable on this host. + * Always reports unavailable in the browser / non-macOS builds. + */ +export function useSpeechAnalyzerAvailability(): SpeechAnalyzerAvailability { + const [state, setState] = useState(initialAvailability); + + useEffect(() => { + const desktop = window.rescriptDesktop; + if (!desktop?.speechAnalyzer || desktop.platform !== "darwin") return; + + let cancelled = false; + void desktop.speechAnalyzer.check().then((result) => { + if (cancelled) return; + setState( + result.available + ? { status: "available", locale: result.locale } + : { status: "unavailable", reason: result.reason } + ); + }); + return () => { + cancelled = true; + }; + }, []); + + return state; +} + +/** Electron File objects often expose an absolute `.path` for disk files. */ +function electronFilePath(file: File): string | undefined { + const path = (file as File & { path?: string }).path; + return typeof path === "string" && path.length > 0 ? path : undefined; +} + +/** + * Run SpeechAnalyzer on a media file via the desktop bridge. Updates the + * editor store with progress / words. Caller should only invoke when the + * selected model is `speechanalyzer` and availability is confirmed. + */ +export function useSpeechAnalyzerTranscriber() { + const transcribeFile = useCallback(async (file: File) => { + const desktop = window.rescriptDesktop?.speechAnalyzer; + if (!desktop) { + throw new Error("SpeechAnalyzer is only available in the macOS desktop app."); + } + + const { useEditorStore } = await import("@/lib/store"); + const store = useEditorStore.getState(); + store.setStatus("transcribing"); + store.setProgress({ message: "Starting SpeechAnalyzer…", value: null }); + + const stopProgress = desktop.onProgress((progress) => { + const s = useEditorStore.getState(); + if (s.skipTranscription) return; + s.setProgress({ message: progress.message, value: progress.value }); + }); + + try { + const path = electronFilePath(file); + const result = path + ? await desktop.transcribe({ path }) + : await desktop.transcribe({ + data: await file.arrayBuffer(), + name: file.name, + }); + + const s = useEditorStore.getState(); + if (s.skipTranscription) return; + s.setWords( + result.words.map((w) => ({ + id: w.id, + text: w.text, + start: w.start, + end: w.end, + speaker: w.speaker, + deleted: w.deleted, + })) + ); + s.setStatus("ready"); + s.setPartialText(""); + s.setProgress({ message: "", value: null }); + } catch (err) { + const s = useEditorStore.getState(); + if (s.skipTranscription) return; + s.setError(err instanceof Error ? err.message : "SpeechAnalyzer failed."); + } finally { + stopProgress(); + } + }, []); + + return { transcribeFile }; +} diff --git a/hooks/useTranscriber.ts b/hooks/useTranscriber.ts index 69f276e..93dcb60 100644 --- a/hooks/useTranscriber.ts +++ b/hooks/useTranscriber.ts @@ -27,7 +27,7 @@ export function useTranscriber() { const transcribe = useCallback((audio: Float32Array, duration: number) => { const store = useEditorStore.getState(); if (!isWhisperModel(store.model)) { - store.setError("Select Whisper Base or Small to transcribe."); + store.setError("Select Whisper Base or Small to transcribe with Whisper."); return; } const whisperModel = store.model; diff --git a/lib/models.ts b/lib/models.ts index 9f12ae4..b9bf401 100644 --- a/lib/models.ts +++ b/lib/models.ts @@ -1,6 +1,8 @@ /** Transcription source choices offered on the upload screen. */ export type WhisperModel = "base" | "small"; -export type ModelChoice = WhisperModel | "import"; +/** macOS 26+ SpeechAnalyzer backend (Electron desktop only). */ +export type SpeechAnalyzerModel = "speechanalyzer"; +export type ModelChoice = WhisperModel | "import" | SpeechAnalyzerModel; type DType = "fp32" | "fp16" | "q8" | "int8" | "uint8" | "q4" | "q4f16" | "bnb4"; @@ -50,9 +52,6 @@ export const MODELS: Record = { description: "Faster download and transcription. Good for most clips.", size: "~200 MB", dtype: WHISPER_DTYPE, - // Do not set verbatimPrompt: forcing a long <|startofprev|> prompt via - // decoder_input_ids truncates long-form transcripts (e.g. drops the second - // speaker on mixed clips). Prefer post-process / filler tools instead. }, small: { id: "onnx-community/whisper-small_timestamped", @@ -63,32 +62,43 @@ export const MODELS: Record = { }, }; +/** UI metadata for the macOS SpeechAnalyzer option (not a Whisper HF model). */ +export const SPEECH_ANALYZER_INFO = { + label: "SpeechAnalyzer", + description: "Apple’s on-device speech model (macOS 26+, Electron).", + size: "System", +} as const; + export function isWhisperModel(value: unknown): value is WhisperModel { return value === "base" || value === "small"; } +export function isSpeechAnalyzerModel(value: unknown): value is SpeechAnalyzerModel { + return value === "speechanalyzer"; +} + export function isModelChoice(value: unknown): value is ModelChoice { - return isWhisperModel(value) || value === "import"; + return isWhisperModel(value) || value === "import" || isSpeechAnalyzerModel(value); } const MODEL_STORAGE_KEY = "rescript.model"; -/** Read the last-selected Whisper model from localStorage (defaults to base). */ -export function loadModelPreference(): WhisperModel { +/** Read the last-selected persistent model from localStorage (defaults to base). */ +export function loadModelPreference(): WhisperModel | SpeechAnalyzerModel { if (typeof window === "undefined") return "base"; try { const raw = window.localStorage.getItem(MODEL_STORAGE_KEY); // Ignore a stale "import" preference — that choice is session-only until a // transcript file is picked again. - if (isWhisperModel(raw)) return raw; + if (isWhisperModel(raw) || isSpeechAnalyzerModel(raw)) return raw; } catch { // private mode / disabled storage } return "base"; } -/** Persist the selected Whisper model for the next visit. */ -export function saveModelPreference(model: WhisperModel) { +/** Persist the selected model for the next visit (Whisper or SpeechAnalyzer). */ +export function saveModelPreference(model: WhisperModel | SpeechAnalyzerModel) { if (typeof window === "undefined") return; try { window.localStorage.setItem(MODEL_STORAGE_KEY, model); diff --git a/lib/store.ts b/lib/store.ts index a41e6b8..cb328de 100644 --- a/lib/store.ts +++ b/lib/store.ts @@ -21,6 +21,7 @@ import { } from "./edits"; import { isModelChoice, + isSpeechAnalyzerModel, isWhisperModel, loadModelPreference, saveModelPreference, @@ -269,7 +270,11 @@ export const useEditorStore = create((set, get) => ({ mediaKind: kind, projectId: null, skipTranscription: Boolean(imported), - model: imported ? "import" : isWhisperModel(current) ? current : "base", + model: imported + ? "import" + : isWhisperModel(current) || isSpeechAnalyzerModel(current) + ? current + : "base", pendingTranscript: null, status: "preparing", progress: { @@ -342,7 +347,7 @@ export const useEditorStore = create((set, get) => ({ }, setModel: (model) => { - if (isWhisperModel(model)) { + if (isWhisperModel(model) || isSpeechAnalyzerModel(model)) { saveModelPreference(model); set({ model, pendingTranscript: null }); } else { @@ -676,6 +681,18 @@ export const useEditorStore = create((set, get) => ({ /** Apply the stored model choice after mount (avoids SSR/localStorage mismatch). */ export function hydrateModelPreference() { const stored = loadModelPreference(); + // SpeechAnalyzer only works in the macOS Electron app — fall back on web. + if ( + isSpeechAnalyzerModel(stored) && + (typeof window === "undefined" || + window.rescriptDesktop?.platform !== "darwin" || + !window.rescriptDesktop.speechAnalyzer) + ) { + if (useEditorStore.getState().model !== "base") { + useEditorStore.setState({ model: "base" }); + } + return; + } if (stored !== useEditorStore.getState().model) { useEditorStore.setState({ model: stored }); } diff --git a/native/speechanalyzer/Makefile b/native/speechanalyzer/Makefile new file mode 100644 index 0000000..8318fdc --- /dev/null +++ b/native/speechanalyzer/Makefile @@ -0,0 +1,27 @@ +.PHONY: build sign clean + +# SpeechAnalyzer requires macOS 26 + a signed binary. `swift run` skips +# signing and will crash on first analyzer touch — always go through `make`. + +BINARY ?= .build/release/rescript-speechanalyzer +DEST ?= ../../resources/bin/rescript-speechanalyzer + +build: + @uname -s | grep -q Darwin || (echo "error: SpeechAnalyzer helper only builds on macOS" >&2; exit 1) + swift build -c release \ + -Xlinker -sectcreate -Xlinker __TEXT \ + -Xlinker __info_plist -Xlinker Resources/Info.plist + $(MAKE) sign + mkdir -p "$(dir $(DEST))" + cp "$(BINARY)" "$(DEST)" + @echo "installed $(DEST)" + +sign: + codesign --sign - --options runtime \ + --entitlements Resources/entitlements.plist \ + --force "$(BINARY)" + +clean: + swift package clean + rm -rf .build + rm -f "$(DEST)" diff --git a/native/speechanalyzer/Package.swift b/native/speechanalyzer/Package.swift new file mode 100644 index 0000000..27840f7 --- /dev/null +++ b/native/speechanalyzer/Package.swift @@ -0,0 +1,18 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "rescript-speechanalyzer", + platforms: [ + .macOS("26.0"), + ], + products: [ + .executable(name: "rescript-speechanalyzer", targets: ["RescriptSpeechAnalyzer"]), + ], + targets: [ + .executableTarget( + name: "RescriptSpeechAnalyzer", + path: "Sources/RescriptSpeechAnalyzer" + ), + ] +) diff --git a/native/speechanalyzer/README.md b/native/speechanalyzer/README.md new file mode 100644 index 0000000..75e94a6 --- /dev/null +++ b/native/speechanalyzer/README.md @@ -0,0 +1,20 @@ +# native/speechanalyzer + +macOS 26+ CLI helper that wraps Apple’s `SpeechAnalyzer` / `SpeechTranscriber` +for Rescript’s Electron shell. + +```bash +# Requires macOS 26 + Xcode 26 SDK. Always build via Make (signing matters). +make -C native/speechanalyzer build +``` + +The signed binary is installed to `resources/bin/rescript-speechanalyzer` and +bundled into the Mac desktop app as an `extraResource`. + +```bash +./resources/bin/rescript-speechanalyzer --check +./resources/bin/rescript-speechanalyzer /path/to/clip.m4a +``` + +Progress JSON lines are written to stderr; the final `{ type: "complete", words }` +payload goes to stdout. diff --git a/native/speechanalyzer/Resources/Info.plist b/native/speechanalyzer/Resources/Info.plist new file mode 100644 index 0000000..9b65fb8 --- /dev/null +++ b/native/speechanalyzer/Resources/Info.plist @@ -0,0 +1,12 @@ + + + + + CFBundleIdentifier + io.rescript.speechanalyzer + CFBundleName + rescript-speechanalyzer + NSSpeechRecognitionUsageDescription + Rescript uses on-device speech recognition to transcribe your media. + + diff --git a/native/speechanalyzer/Resources/entitlements.plist b/native/speechanalyzer/Resources/entitlements.plist new file mode 100644 index 0000000..feb3f39 --- /dev/null +++ b/native/speechanalyzer/Resources/entitlements.plist @@ -0,0 +1,12 @@ + + + + + com.apple.security.device.audio-input + + com.apple.security.cs.disable-library-validation + + com.apple.security.cs.allow-jit + + + diff --git a/native/speechanalyzer/Sources/RescriptSpeechAnalyzer/main.swift b/native/speechanalyzer/Sources/RescriptSpeechAnalyzer/main.swift new file mode 100644 index 0000000..42ce5f5 --- /dev/null +++ b/native/speechanalyzer/Sources/RescriptSpeechAnalyzer/main.swift @@ -0,0 +1,393 @@ +import AVFoundation +import CoreMedia +import Foundation +import Speech + +// Minimal CLI that wraps Apple's SpeechAnalyzer / SpeechTranscriber +// (macOS 26+) for Rescript's Electron shell. +// +// Usage: +// rescript-speechanalyzer --check [--locale en-US] +// rescript-speechanalyzer [--locale en-US] +// +// Progress JSON lines go to stderr; the final result JSON goes to stdout. +// Lessons encoded here come from the dictamac / steno projects: +// - analyzer lifecycle MUST run on @MainActor +// - keep the main RunLoop alive (dispatchMain) or finals never arrive +// - AssetInventory.reserve(locale:) is required or analyzeSequence hangs +// - use finalizeAndFinishThroughEndOfInput(), not finalize(through: infinity) +// - ad-hoc codesign with JIT + disable-library-validation entitlements + +@main +enum RescriptSpeechAnalyzerMain { + static func main() { + let args = Array(CommandLine.arguments.dropFirst()) + if args.contains("-h") || args.contains("--help") { + fputs( + """ + Usage: + rescript-speechanalyzer --check [--locale ] + rescript-speechanalyzer [--locale ] + + """, + stderr + ) + exit(0) + } + + let locale = parseLocale(from: args) + let checkOnly = args.contains("--check") + let filePath = positionalFilePath(from: args) + + // Kick off async work, then park on the main RunLoop so SpeechAnalyzer + // can deliver results (see dictamac DefaultTranscriber notes). + Task { + do { + if checkOnly { + let payload = try await checkAvailability(locale: locale) + printJSON(payload) + exit(0) + } + guard let filePath else { + throw HelperError.usage("missing audio/video file path") + } + let payload = try await transcribe(path: filePath, locale: locale) + printJSON(payload) + exit(0) + } catch let err as HelperError { + printJSON([ + "type": "error", + "message": err.message, + "code": err.code, + ]) + exit(Int32(err.exitCode)) + } catch { + printJSON([ + "type": "error", + "message": error.localizedDescription, + "code": "unknown", + ]) + exit(1) + } + } + dispatchMain() + } +} + +// MARK: - CLI helpers + +enum HelperError: Error { + case usage(String) + case unavailable(String) + case file(String) + case transcribe(String) + + var message: String { + switch self { + case .usage(let m), .unavailable(let m), .file(let m), .transcribe(let m): + return m + } + } + + var code: String { + switch self { + case .usage: return "usage" + case .unavailable: return "unavailable" + case .file: return "file" + case .transcribe: return "transcribe" + } + } + + var exitCode: Int { + switch self { + case .usage: return 2 + case .unavailable: return 67 + case .file: return 66 + case .transcribe: return 1 + } + } +} + +func parseLocale(from args: [String]) -> Locale { + if let idx = args.firstIndex(of: "--locale"), args.index(after: idx) < args.endIndex { + return Locale(identifier: args[args.index(after: idx)]) + } + return Locale.current +} + +func positionalFilePath(from args: [String]) -> String? { + var skipNext = false + for arg in args { + if skipNext { + skipNext = false + continue + } + if arg == "--locale" { + skipNext = true + continue + } + if arg.hasPrefix("-") { continue } + return arg + } + return nil +} + +func printJSON(_ value: Any) { + guard JSONSerialization.isValidJSONObject(value), + let data = try? JSONSerialization.data(withJSONObject: value), + let line = String(data: data, encoding: .utf8) + else { + fputs("{\"type\":\"error\",\"message\":\"failed to encode JSON\"}\n", stderr) + return + } + print(line) + fflush(stdout) +} + +func progress(_ message: String, value: Double? = nil) { + var payload: [String: Any] = ["type": "progress", "message": message] + if let value { payload["value"] = value } + else { payload["value"] = NSNull() } + if let data = try? JSONSerialization.data(withJSONObject: payload), + let line = String(data: data, encoding: .utf8) + { + fputs(line + "\n", stderr) + fflush(stderr) + } +} + +func localeIdentifier(_ locale: Locale) -> String { + let id = locale.identifier(.bcp47) + return id.isEmpty ? locale.identifier : id +} + +// MARK: - Availability + +func checkAvailability(locale: Locale) async throws -> [String: Any] { + guard #available(macOS 26.0, *) else { + return [ + "type": "check", + "available": false, + "reason": "SpeechAnalyzer requires macOS 26 or later.", + ] + } + return try await checkAvailabilityModern(locale: locale) +} + +@available(macOS 26.0, *) +func checkAvailabilityModern(locale: Locale) async throws -> [String: Any] { + // Touch the type so we fail closed if Speech framework symbols resolve + // but the runtime refuses to construct a transcriber. + _ = SpeechTranscriber( + locale: locale, + transcriptionOptions: [], + reportingOptions: [], + attributeOptions: [.audioTimeRange] + ) + let installed = SpeechTranscriber.installedLocales + let supported = SpeechTranscriber.supportedLocales + return [ + "type": "check", + "available": true, + "locale": localeIdentifier(locale), + "installedLocales": installed.map { localeIdentifier($0) }, + "supportedLocales": Array(supported.prefix(32)).map { localeIdentifier($0) }, + ] +} + +// MARK: - Transcription entry + +func transcribe(path: String, locale: Locale) async throws -> [String: Any] { + guard #available(macOS 26.0, *) else { + throw HelperError.unavailable("SpeechAnalyzer requires macOS 26 or later.") + } + return try await transcribeModern(path: path, locale: locale) +} + +// MARK: - Model bootstrap + analyze + +@available(macOS 26.0, *) +func ensureModel(for locale: Locale) async throws { + let probe = SpeechTranscriber( + locale: locale, + transcriptionOptions: [], + reportingOptions: [], + attributeOptions: [] + ) + let status = await AssetInventory.status(forModules: [probe]) + switch status { + case .installed: + break + case .supported, .downloading: + progress("Downloading on-device speech model…") + if let request = try await AssetInventory.assetInstallationRequest(supporting: [probe]) { + try await request.downloadAndInstall() + } + case .unsupported: + throw HelperError.unavailable( + "SpeechAnalyzer has no model for locale \(localeIdentifier(locale)). Install one in System Settings." + ) + @unknown default: + throw HelperError.unavailable( + "SpeechAnalyzer reported an unknown asset status for locale \(localeIdentifier(locale))." + ) + } + + // Without reserve(), analyzeSequence hangs forever. + do { + _ = try await AssetInventory.reserve(locale: locale) + } catch { + throw HelperError.unavailable( + "Could not reserve SpeechAnalyzer locale \(localeIdentifier(locale)): \(error.localizedDescription)" + ) + } +} + +@available(macOS 26.0, *) +func transcribeModern(path: String, locale: Locale) async throws -> [String: Any] { + let url = URL(fileURLWithPath: path) + guard FileManager.default.fileExists(atPath: url.path) else { + throw HelperError.file("File not found: \(path)") + } + + progress("Preparing SpeechAnalyzer…") + try await ensureModel(for: locale) + + let audioFile: AVAudioFile + do { + audioFile = try AVAudioFile(forReading: url) + } catch { + throw HelperError.file("Could not decode audio from \(path): \(error.localizedDescription)") + } + + let transcriber = SpeechTranscriber( + locale: locale, + transcriptionOptions: [], + reportingOptions: [], + attributeOptions: [.audioTimeRange] + ) + let analyzer = SpeechAnalyzer(modules: [transcriber]) + + progress("Transcribing with SpeechAnalyzer…", value: 0) + + async let analyzedEnd: CMTime? = Task { @MainActor in + let last = try await analyzer.analyzeSequence(from: audioFile) + try await analyzer.finalizeAndFinishThroughEndOfInput() + return last + }.value + + var words: [[String: Any]] = [] + var nextId = 0 + do { + for try await result in transcriber.results { + guard result.isFinal else { continue } + let chunk = wordsFromResult(result, startingId: nextId) + nextId += chunk.count + words.append(contentsOf: chunk) + if let last = chunk.last, let end = last["end"] as? Double, end > 0 { + let approxDuration = + Double(audioFile.length) / max(audioFile.processingFormat.sampleRate, 1) + if approxDuration > 0 { + progress("Transcribing with SpeechAnalyzer…", value: min(0.99, end / approxDuration)) + } + } + } + } catch { + _ = try? await analyzedEnd + throw HelperError.transcribe(error.localizedDescription) + } + + let end = try await analyzedEnd + let duration: Double = { + if let end, end.isNumeric, end.seconds.isFinite, end.seconds > 0 { + return end.seconds + } + let frames = Double(audioFile.length) + let rate = audioFile.processingFormat.sampleRate + if rate > 0, frames > 0 { return frames / rate } + return (words.last?["end"] as? Double) ?? 0 + }() + + return [ + "type": "complete", + "locale": localeIdentifier(locale), + "duration": duration, + "model": "SpeechAnalyzer/macOS26", + "words": words, + ] +} + +@available(macOS 26.0, *) +func wordsFromResult(_ result: SpeechTranscriber.Result, startingId: Int) -> [[String: Any]] { + var out: [[String: Any]] = [] + var id = startingId + + // Prefer per-run audioTimeRange (word/phrase level) when present. + var emittedFromRuns = false + for run in result.text.runs { + guard let range = run.audioTimeRange else { continue } + let start = seconds(range.start) + let end = seconds(range.end) + let text = String(run.text).trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty, end >= start else { continue } + emittedFromRuns = true + for piece in splitWords(text, start: start, end: end) { + out.append([ + "id": id, + "text": piece.text, + "start": piece.start, + "end": piece.end, + "speaker": 0, + "deleted": false, + ]) + id += 1 + } + } + + if emittedFromRuns { return out } + + // Fallback: one timed segment → split words proportionally. + let start = seconds(result.range.start) + let end = seconds(result.range.end) + let text = String(result.text.characters).trimmingCharacters(in: .whitespacesAndNewlines) + guard !text.isEmpty else { return out } + for piece in splitWords(text, start: start, end: max(end, start)) { + out.append([ + "id": id, + "text": piece.text, + "start": piece.start, + "end": piece.end, + "speaker": 0, + "deleted": false, + ]) + id += 1 + } + return out +} + +func seconds(_ time: CMTime) -> Double { + guard time.isNumeric else { return 0 } + let value = time.seconds + return value.isFinite ? value : 0 +} + +struct TimedWord { + let text: String + let start: Double + let end: Double +} + +func splitWords(_ text: String, start: Double, end: Double) -> [TimedWord] { + let tokens = text.split { $0.isWhitespace }.map(String.init).filter { !$0.isEmpty } + guard !tokens.isEmpty else { return [] } + if tokens.count == 1 { + return [TimedWord(text: tokens[0], start: start, end: max(end, start))] + } + let span = max(end - start, 0) + let step = span / Double(tokens.count) + return tokens.enumerated().map { i, token in + let s = start + Double(i) * step + let e = i == tokens.count - 1 ? max(end, s) : start + Double(i + 1) * step + return TimedWord(text: token, start: s, end: e) + } +} diff --git a/package.json b/package.json index e5f6ba2..0fc6783 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "lint": "eslint", "postinstall": "node scripts/copy-assets.mjs", "build:electron": "node scripts/build-electron.mjs", + "build:native": "make -C native/speechanalyzer build", "typecheck:electron": "tsc -p electron/tsconfig.json --noEmit", "electron:dev": "npm run build:electron && concurrently -k -n next,electron -c cyan,magenta \"next dev\" \"node scripts/electron-dev.mjs\"", "export:desktop": "cross-env STATIC_EXPORT=1 NEXT_PUBLIC_ELECTRON=1 next build", @@ -84,6 +85,13 @@ "!electron-dist/**/*.map", "!**/node_modules/**" ], + "extraResources": [ + { + "from": "resources/bin", + "to": "bin", + "filter": ["**/*", "!.gitkeep"] + } + ], "asar": true, "mac": { "target": [ @@ -111,6 +119,7 @@ "notarize": true, "extendInfo": { "NSMicrophoneUsageDescription": "Rescript can use the microphone for on-device speech transcription features.", + "NSSpeechRecognitionUsageDescription": "Rescript uses on-device speech recognition to transcribe your media with Apple SpeechAnalyzer.", "NSDocumentsFolderUsageDescription": "Rescript opens and exports video and audio files you select." } }, diff --git a/resources/bin/.gitkeep b/resources/bin/.gitkeep new file mode 100644 index 0000000..ad439c5 --- /dev/null +++ b/resources/bin/.gitkeep @@ -0,0 +1 @@ +# SpeechAnalyzer helper binaries land here after `make -C native/speechanalyzer build` diff --git a/types/rescript-desktop.d.ts b/types/rescript-desktop.d.ts index 8aa43cd..663339c 100644 --- a/types/rescript-desktop.d.ts +++ b/types/rescript-desktop.d.ts @@ -1,6 +1,47 @@ /** Resting sizes the Electron shell switches between. */ export type WindowMode = "compact" | "expanded"; +/** + * SpeechAnalyzer payloads, mirrored from electron/ipc/channels.ts. The renderer + * can't import from electron/ (different tsconfig + Node types), so the shapes + * are declared structurally on both sides. + */ +export type SpeechAnalyzerCheckResult = { + available: boolean; + reason?: string; + locale?: string; + installedLocales?: string[]; + helperPath?: string | null; +}; + +export type SpeechAnalyzerProgress = { + message: string; + value: number | null; +}; + +export type SpeechAnalyzerTranscribeRequest = { + path?: string; + data?: ArrayBuffer; + name?: string; + locale?: string; +}; + +export type SpeechAnalyzerWord = { + id: number; + text: string; + start: number; + end: number; + speaker: number; + deleted: boolean; +}; + +export type SpeechAnalyzerTranscribeResult = { + words: SpeechAnalyzerWord[]; + locale: string; + duration: number; + model: string; +}; + /** Desktop bridge exposed by electron/preload.ts when running inside Electron. */ export interface RescriptDesktop { platform: NodeJS.Platform; @@ -14,6 +55,13 @@ export interface RescriptDesktop { isFullScreen: () => Promise; /** Subscribe to full-screen changes; returns an unsubscribe function. */ onFullScreenChange: (callback: (value: boolean) => void) => () => void; + speechAnalyzer?: { + check: () => Promise; + transcribe: ( + req: SpeechAnalyzerTranscribeRequest + ) => Promise; + onProgress: (handler: (progress: SpeechAnalyzerProgress) => void) => () => void; + }; } declare global {