From 328e2ce76216e27a2d8e78d4723014adef079803 Mon Sep 17 00:00:00 2001 From: gitb0i <64463178+gitb0i@users.noreply.github.com> Date: Sun, 23 Aug 2026 06:39:13 -0700 Subject: [PATCH] VoiceMessageTranscriber: Added Embed, model size estimates (#1302) * feat(voiceMessageTranscriber): add embed and maintain horizontal options with model size indicators * Apply suggestions from code review Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix(voiceMessageTranscriber): scope parent selectors and resolve transcription race condition * refactor(voiceMessageTranscriber): adhere strictly to .rules types and formatting * fix(voiceMessageTranscriber): reliably wrap parent container for embed layout * feat(voiceMessageTranscriber): add tt to authors * some fixes --------- Co-authored-by: thororen <78185467+thororen1234@users.noreply.github.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- .../voiceMessageTranscriber.desktop/index.tsx | 473 ++++++++++++++---- .../voiceMessageTranscriber.desktop/style.css | 171 +++++++ .../voiceMessageTranscriber.desktop/utils.ts | 28 +- 3 files changed, 573 insertions(+), 99 deletions(-) diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx index 76b23c6c6..45e6eb9bb 100644 --- a/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx @@ -10,27 +10,88 @@ import { DataStore } from "@api/index"; import { definePluginSettings } from "@api/Settings"; import { BaseText } from "@components/BaseText"; import { Button } from "@components/Button"; +import ErrorBoundary from "@components/ErrorBoundary"; import { Flex } from "@components/Flex"; -import { FormSwitch } from "@components/FormSwitch"; import { Heading } from "@components/Heading"; +import { ClockIcon, CopyIcon } from "@components/Icons"; import { Span } from "@components/Span"; import { copyToClipboard } from "@utils/clipboard"; -import { Devs } from "@utils/constants"; +import { Devs, EquicordDevs } from "@utils/constants"; +import { classes } from "@utils/misc"; import definePlugin, { OptionType, PluginNative } from "@utils/types"; import { RenderModalProps } from "@vencord/discord-types"; import { findComponentByCodeLazy } from "@webpack"; -import { lodash, Modal, openModal, ScrollerAuto, SearchableSelect, useEffect, useRef, useState } from "@webpack/common"; +import { Modal, openModal, ScrollerAuto, SearchableSelect, Tooltip, useEffect, useRef, useState } from "@webpack/common"; -import { cl, decodeAudio, LANGUAGES, TranscriptionWorker } from "./utils"; +import { cl, decodeAudio, LANGUAGES, TranscriptionResult, TranscriptionWorker } from "./utils"; const Native = VencordNative.pluginHelpers.VoiceMessageTranscriber as PluginNative; const ChannelListIcon = findComponentByCodeLazy("1-1-1ZM2 8a1"); -let ManaBaseRadioGroup; +let ManaBaseRadioGroup: React.ComponentType<{ options: { name: string; value: string; }[]; value: string; onChange: (v: string) => void; }>; + +function formatTimestamp(seconds: number) { + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`; +} + +function CloseIcon({ size = 14 }: { size?: number; }) { + return ( + + + + + ); +} + +function CheckmarkIcon({ size = 14 }: { size?: number; }) { + return ( + + + + ); +} + +const MODEL_SIZES: Record = { + "Xenova/whisper-tiny": { quantized: "~40 MB", full: "~150 MB" }, + "Xenova/whisper-base": { quantized: "~77 MB", full: "~290 MB" }, + "Xenova/whisper-small": { quantized: "~250 MB", full: "~960 MB" }, + "Xenova/whisper-medium": { quantized: "~765 MB", full: "~3.1 GB" }, +}; + +function renderModelOption(option?: { label: string; value: string; }) { + if (!option) return null; + const isQuantized = settings.use(["quantized"])?.quantized ?? true; + const size = MODEL_SIZES[option.value]?.[isQuantized ? "quantized" : "full"]; + + return ( +
+ {option.label} + {size && ( + + {size} + + )} +
+ ); +} const settings = definePluginSettings({ + embed: { + type: OptionType.BOOLEAN, + description: "Display transcription directly in the voice message attachment instead of a modal.", + default: false, + restartNeeded: false + }, + maintainHorizontal: { + type: OptionType.BOOLEAN, + description: "Maintain horizontal size for the embedded transcription box and expand vertically.", + default: false, + restartNeeded: false + }, selectedModel: { type: OptionType.SELECT, - description: "Model size", + description: "Model size.", options: [ { label: "Tiny (Fastest, lowest accuracy)", @@ -50,38 +111,45 @@ const settings = definePluginSettings({ value: "Xenova/whisper-medium" } ], + componentProps: { + renderOptionLabel: (option: { label: string; value: string; }) => renderModelOption(option), + renderOptionValue: (options: { label: string; value: string; }[]) => renderModelOption(options?.[0]), + }, restartNeeded: false }, quantized: { type: OptionType.BOOLEAN, - description: "Use quantized model (smaller size, slightly lower accuracy)", + description: "Use quantized models (smaller size, slight quality loss).", default: true, restartNeeded: false }, - delete: { + deleteModalFiles: { type: OptionType.COMPONENT, + description: "Delete cached files from storage.", component: () => { const [size, setSize] = useState(0); const [deleteKeys, setDeleteKeys] = useState([]); useEffect(() => { + let unmounted = false; DataStore.entries().then(entries => { - let size = 0; - const keys = [] as string[]; - - entries.forEach(([key, val]) => { - if (typeof key === "string" && key.startsWith("VoiceMessageTranscriber") && lodash.isArrayBuffer(val)) { + if (unmounted) return; + let totalSize = 0; + const keys: string[] = []; + for (const [key, value] of entries) { + if (typeof key === "string" && (key.startsWith("whisper-") || key.startsWith("onnx-"))) { + totalSize += (value as string).length; keys.push(key); - size += val.byteLength ?? 0; } - }); - - setSize(size); + } + setSize(totalSize); setDeleteKeys(keys); }); + return () => { unmounted = true; }; }, []); return + {embed && isOpen && ( +
e.stopPropagation()} + > +
+
+ + + {status === "downloading_audio" && "Downloading Audio..."} + {status === "processing_audio" && "Processing Audio..."} + {status === "loading" && "Loading Model..."} + {status === "transcribing" && "Transcribing..."} + {status === "complete" && "Transcription"} + {status === "error" && "Transcription Error"} + +
+
+ {result && ( + <> + + {props => ( + + )} + + + {props => ( + + )} + + + )} + + {props => ( + + )} + +
+
+
+ {error ? ( + + {error} + + + ) : displayText ? ( + + {displayText} + + ) : ( + + {isWorking ? "Transcribing in progress..." : "Initializing..."} + + )} +
+
+ )} + + ); +} + +const VoiceMessageTranscriberWrapped = ErrorBoundary.wrap(VoiceMessageTranscriber, { noop: true }); + export default definePlugin({ name: "VoiceMessageTranscriber", - authors: [Devs.TheSun], + authors: [Devs.TheSun, EquicordDevs.tt], description: "On-device transcriptions for voice messages powered by Whisper v3", tags: ["Chat", "Media", "Utility", "Voice"], patches: [ @@ -341,20 +638,12 @@ export default definePlugin({ } }, ], - set ManaBaseRadioGroup(value: any) { + set ManaBaseRadioGroup(value: typeof ManaBaseRadioGroup) { ManaBaseRadioGroup = value; }, settings, button(src: string) { - return ; + return ; }, }); diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/style.css b/src/equicordplugins/voiceMessageTranscriber.desktop/style.css index 3049ba7bc..feb8ce510 100644 --- a/src/equicordplugins/voiceMessageTranscriber.desktop/style.css +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/style.css @@ -24,4 +24,175 @@ user-select: text; max-height: 400px; overflow-y: auto; +} + +.vc-transcription-model-option { + display: flex; + align-items: center; + gap: 8px; + width: 100%; +} + +.vc-transcription-model-size { + font-size: 12px; + color: var(--text-muted); + opacity: 0.75; + flex-shrink: 0; +} + +.vc-transcription-container:has(> .vc-transcription-embed), +.vc-transcription-container.vc-transcription-has-embed { + flex-wrap: wrap !important; +} + +.vc-transcription-container:has(> .vc-transcription-embed-maintain-horizontal), +.vc-transcription-container.vc-transcription-maintain-horizontal { + flex-wrap: wrap !important; + width: fit-content !important; + max-width: 100% !important; +} + +.vc-transcription-embed { + order: 9999 !important; + display: flex; + flex-direction: column; + width: 100%; + flex-basis: 100%; + min-width: 0; + margin-top: 8px; + border-radius: 8px; + background: rgb(0 0 0 / 25%); + border: 1px solid var(--border-subtle, rgb(255 255 255 / 8%)); + backdrop-filter: blur(4px); + box-sizing: border-box; + overflow: hidden; + user-select: text; +} + +.vc-transcription-embed-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 6px 10px; + background: rgb(0 0 0 / 15%); + border-bottom: 1px solid var(--border-subtle, rgb(255 255 255 / 6%)); +} + +.vc-transcription-embed-title { + display: flex; + align-items: center; + gap: 6px; + font-size: 11px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--text-muted); +} + +.vc-transcription-status-dot { + width: 6px; + height: 6px; + border-radius: 50%; + background-color: var(--text-muted); + transition: background-color 0.2s ease; +} + +.vc-transcription-status-dot-active { + background-color: var(--main-color, var(--brand-experiment, var(--brand-500, #5865f2))); + animation: vc-transcription-pulse 1.5s infinite; +} + +.vc-transcription-status-dot-error { + background-color: var(--text-feedback-critical, #ed4245); +} + +@keyframes vc-transcription-pulse { + 0%, 100% { + opacity: 1; + transform: scale(1); + } + + 50% { + opacity: 0.35; + transform: scale(0.85); + } +} + +.vc-transcription-embed-actions { + display: flex; + align-items: center; + gap: 4px; +} + +.vc-transcription-action-btn { + display: flex; + align-items: center; + justify-content: center; + width: 22px; + height: 22px; + padding: 0; + border: none; + border-radius: 4px; + background: transparent; + color: var(--interactive-icon-default, var(--text-muted)); + cursor: pointer; + transition: color 0.15s ease, background-color 0.15s ease; +} + +.vc-transcription-action-btn:hover { + color: var(--interactive-icon-hover, var(--text-normal)); + background-color: var(--background-modifier-hover, rgb(255 255 255 / 8%)); +} + +.vc-transcription-action-btn-active { + color: var(--main-color, var(--brand-experiment, var(--brand-500, #5865f2))); + background-color: var(--background-modifier-selected, rgb(255 255 255 / 12%)); +} + +.vc-transcription-embed-body { + padding: 8px 12px; + max-height: 180px; + overflow-y: auto; +} + +.vc-transcription-embed-text { + white-space: pre-wrap; + user-select: text; +} + +.vc-transcription-embed-maintain-horizontal { + order: 9999 !important; + width: 0 !important; + min-width: 100% !important; + max-width: 100% !important; + flex-basis: 100% !important; + flex-grow: 1 !important; + box-sizing: border-box !important; + margin-left: 0 !important; + margin-right: 0 !important; +} + +.vc-transcription-embed-maintain-horizontal .vc-transcription-embed-header { + width: 100% !important; + min-width: 0 !important; + max-width: 100% !important; + box-sizing: border-box !important; +} + +.vc-transcription-embed-maintain-horizontal .vc-transcription-embed-body { + width: 100% !important; + min-width: 0 !important; + max-width: 100% !important; + max-height: 280px; + box-sizing: border-box !important; + overflow-wrap: anywhere; +} + +.vc-transcription-embed-maintain-horizontal .vc-transcription-embed-text { + width: 100% !important; + min-width: 0 !important; + max-width: 100% !important; + box-sizing: border-box !important; + overflow-wrap: anywhere; + display: block; } \ No newline at end of file diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts index a2147f75f..cdb12ebe3 100644 --- a/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts @@ -112,7 +112,11 @@ export const LANGUAGES = { export const cl = classNameFactory("vc-transcription-"); -const getAudioContext = () => new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 }); +const AudioContextClass = window.AudioContext || (window as Window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; +const getAudioContext = () => { + if (!AudioContextClass) throw new Error("AudioContext is not supported in this environment"); + return new AudioContextClass({ sampleRate: 16000 }); +}; export async function decodeAudio(blob: Blob): Promise { const arrayBuffer = await blob.arrayBuffer(); const audioContext = getAudioContext(); @@ -258,18 +262,28 @@ async function runTranscription({ audio, model, quantized, language, task }) { } `; +export interface TranscriptionChunk { + timestamp: [number, number]; + text: string; +} + +export interface TranscriptionResult { + text: string; + chunks: TranscriptionChunk[]; +} + export class TranscriptionWorker { private worker: Worker; private onStatus: (status: string) => void; - private onComplete: (output: any) => void; - private onError: (error: any) => void; - private onPartial: (output: any) => void; + private onComplete: (output: TranscriptionResult) => void; + private onError: (error: unknown) => void; + private onPartial: (output: TranscriptionResult) => void; constructor( onStatus: (status: string) => void, - onComplete: (output: any) => void, - onError: (error: any) => void, - onPartial: (output: any) => void + onComplete: (output: TranscriptionResult) => void, + onError: (error: unknown) => void, + onPartial: (output: TranscriptionResult) => void ) { this.onStatus = onStatus; this.onComplete = onComplete;