From 3e5cbe92147d4095c043cce7bb6daa742601a699 Mon Sep 17 00:00:00 2001 From: thororen <78185467+thororen1234@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:01:47 -0400 Subject: [PATCH] VoiceMessageTransriber: Bug Fixes --- .../components/DeleteCacheButton.tsx | 42 ++ .../components/LanguageSelectionModal.tsx | 91 +++ .../components/TranscriptionModal.tsx | 147 +++++ .../components/VoiceMessageButton.tsx | 249 +++++++ .../voiceMessageTranscriber.desktop/index.tsx | 619 +----------------- .../settings.tsx | 93 +++ .../utils/audio.ts | 52 ++ .../utils/languages.ts | 107 +++ .../utils/misc.ts | 24 + .../{utils.ts => utils/worker.ts} | 178 +---- 10 files changed, 843 insertions(+), 759 deletions(-) create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/components/DeleteCacheButton.tsx create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/components/LanguageSelectionModal.tsx create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/components/TranscriptionModal.tsx create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/components/VoiceMessageButton.tsx create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/settings.tsx create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/utils/audio.ts create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/utils/languages.ts create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/utils/misc.ts rename src/equicordplugins/voiceMessageTranscriber.desktop/{utils.ts => utils/worker.ts} (67%) diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/components/DeleteCacheButton.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/components/DeleteCacheButton.tsx new file mode 100644 index 000000000..f1ed5167d --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/components/DeleteCacheButton.tsx @@ -0,0 +1,42 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { DataStore } from "@api/index"; +import { Button } from "@components/Button"; +import { useEffect, useState } from "@webpack/common"; + +export function DeleteCacheButton() { + const [size, setSize] = useState(0); + const [deleteKeys, setDeleteKeys] = useState([]); + + useEffect(() => { + let unmounted = false; + DataStore.entries().then(entries => { + if (unmounted) return; + let totalSize = 0; + const keys: string[] = []; + for (const [key, value] of entries) { + if (typeof key === "string" && (key.startsWith("whisper-") || key.startsWith("onnx-"))) { + totalSize += (value as string).length; + keys.push(key); + } + } + setSize(totalSize); + setDeleteKeys(keys); + }); + return () => { unmounted = true; }; + }, []); + + return ; +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/components/LanguageSelectionModal.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/components/LanguageSelectionModal.tsx new file mode 100644 index 000000000..fb50e02ae --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/components/LanguageSelectionModal.tsx @@ -0,0 +1,91 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { BaseText } from "@components/BaseText"; +import { Flex } from "@components/Flex"; +import { RenderModalProps } from "@vencord/discord-types"; +import { Modal, openModal, SearchableSelect, useState } from "@webpack/common"; + +import { LANGUAGES } from "../utils/languages"; +import { TranscriptionModal } from "./TranscriptionModal"; + +type RadioGroup = React.ComponentType<{ options: { name: string; value: string; }[]; value: string; onChange: (v: string) => void; }>; + +let ManaBaseRadioGroup: RadioGroup; + +export function setManaBaseRadioGroup(component: RadioGroup) { + ManaBaseRadioGroup = component; +} + +export function LanguageSelectionModal(props: { modalProps: RenderModalProps, src: string; }) { + const { modalProps, src } = props; + const [language, setLanguage] = useState("auto"); + const [task, setTask] = useState("transcribe"); + + const languageOptions = [ + { label: "Auto Detect", value: "auto" }, + ...Object.entries(LANGUAGES).map(([code, name]) => ({ + label: name.charAt(0).toUpperCase() + name.slice(1), + value: code + })) + ]; + + const start = () => { + modalProps.onClose(); + openModal(modalProps => ( + + )); + }; + + return ( + + +
+ + Audio Language + + o.value === language)?.value} + onChange={setLanguage} + /> +
+ +
+ + Action + + setTask(v as string)} + /> +
+ + + ); +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/components/TranscriptionModal.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/components/TranscriptionModal.tsx new file mode 100644 index 000000000..5e07ccc83 --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/components/TranscriptionModal.tsx @@ -0,0 +1,147 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { BaseText } from "@components/BaseText"; +import { Flex } from "@components/Flex"; +import { Heading } from "@components/Heading"; +import { copyToClipboard } from "@utils/clipboard"; +import { RenderModalProps } from "@vencord/discord-types"; +import { Modal, useEffect, useRef, useState } from "@webpack/common"; + +import { settings } from "../settings"; +import { decodeAudio, fetchVoiceMessage } from "../utils/audio"; +import { cl, formatTranscript } from "../utils/misc"; +import { TranscriptionResult, TranscriptionWorker } from "../utils/worker"; + +export function TranscriptionModal(props: { modalProps: RenderModalProps, src: string, options: { language: string, task: string; }; }) { + const { modalProps, src, options } = props; + const [status, setStatus] = useState("initializing"); + const [result, setResult] = useState(null); + const [showTimestamps, setShowTimestamps] = useState(false); + const [error, setError] = useState(null); + const [copied, setCopied] = useState(false); + const [retryCount, setRetryCount] = useState(0); + + const workerRef = useRef(null); + + useEffect(() => { + let active = true; + (async () => { + try { + setStatus("downloading_audio"); + setError(null); + + const blob = await fetchVoiceMessage(src); + + if (!active) return; + setStatus("processing_audio"); + const audioData = await decodeAudio(blob); + + if (!active) return; + workerRef.current?.terminate(); + workerRef.current = new TranscriptionWorker( + s => { + if (active) setStatus(s); + }, + out => { + if (active) { + setResult(out); + setStatus("complete"); + } + }, + err => { + if (active) { + setError(String(err)); + setStatus("error"); + } + }, + partial => { + if (active) setResult(partial); + } + ); + + const { quantized, selectedModel } = settings.store; + workerRef.current.run( + audioData, + selectedModel, + quantized, + options.language === "auto" ? undefined : options.language, + options.task + ); + } catch (err) { + if (active) { + setError(String(err)); + setStatus("error"); + } + } + })(); + + return () => { + active = false; + workerRef.current?.terminate(); + }; + }, [retryCount]); + + const displayText = formatTranscript(result, showTimestamps); + + const handleCopy = () => { + if (!displayText) return; + copyToClipboard(displayText); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + return ( + setRetryCount(c => c + 1) + }] : []), + ...(result ? [ + { + text: showTimestamps ? "Hide Timestamps" : "Show Timestamps", + variant: "secondary" as const, + onClick: () => setShowTimestamps(!showTimestamps) + }, + { + text: copied ? "Copied!" : "Copy Text", + variant: "secondary" as const, + onClick: handleCopy + } + ] : []) + ]} + > +
+ {error ? ( + + Transcription Failed + + {error} + + + ) : displayText ? ( +
+ {displayText} +
+ ) : ( + + + {status === "downloading_audio" && "Downloading Audio..."} + {status === "processing_audio" && "Processing Audio..."} + {status === "loading" && "Loading Model..."} + {status === "transcribing" && "Transcribing..."} + + + )} +
+ + ); +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/components/VoiceMessageButton.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/components/VoiceMessageButton.tsx new file mode 100644 index 000000000..8d161f6ad --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/components/VoiceMessageButton.tsx @@ -0,0 +1,249 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { Button } from "@components/Button"; +import { Flex } from "@components/Flex"; +import { CheckmarkLargeBoldIcon, ClockIcon, CopyIcon, XLargeBoldIcon } from "@components/Icons"; +import { Span } from "@components/Span"; +import { copyToClipboard } from "@utils/clipboard"; +import { classes } from "@utils/misc"; +import { findComponentByCodeLazy } from "@webpack"; +import { openModal, ScrollerAuto, Tooltip, useEffect, useRef, useState } from "@webpack/common"; + +import { settings } from "../settings"; +import { decodeAudio, fetchVoiceMessage } from "../utils/audio"; +import { cl, formatTranscript } from "../utils/misc"; +import { TranscriptionResult, TranscriptionWorker } from "../utils/worker"; +import { LanguageSelectionModal } from "./LanguageSelectionModal"; + +const ChannelListIcon = findComponentByCodeLazy("1-1-1ZM2 8a1"); + +export function VoiceMessageButton({ src }: { src: string; }) { + const { embed, maintainHorizontal, quantized, selectedModel } = settings.use(["embed", "maintainHorizontal", "quantized", "selectedModel"]); + const [isOpen, setIsOpen] = useState(false); + const [status, setStatus] = useState("idle"); + const [result, setResult] = useState(null); + const [showTimestamps, setShowTimestamps] = useState(false); + const [error, setError] = useState(null); + const [copied, setCopied] = useState(false); + + const workerRef = useRef(null); + const activeRunId = useRef(0); + const buttonRef = useRef(null); + + useEffect(() => { + const parent = buttonRef.current?.parentElement; + if (!parent) return; + parent.classList.add("vc-transcription-container"); + return () => { + parent.classList.remove("vc-transcription-container", "vc-transcription-has-embed", "vc-transcription-maintain-horizontal"); + }; + }, []); + + useEffect(() => { + const parent = buttonRef.current?.parentElement; + if (!parent) return; + parent.classList.toggle("vc-transcription-has-embed", Boolean(embed && isOpen)); + parent.classList.toggle("vc-transcription-maintain-horizontal", Boolean(embed && isOpen && maintainHorizontal)); + }, [embed, isOpen, maintainHorizontal]); + + useEffect(() => { + return () => { + activeRunId.current++; + workerRef.current?.terminate(); + workerRef.current = null; + }; + }, []); + + const startTranscription = async () => { + const runId = ++activeRunId.current; + workerRef.current?.terminate(); + workerRef.current = null; + + setIsOpen(true); + setError(null); + setStatus("downloading_audio"); + + try { + const blob = await fetchVoiceMessage(src); + + if (runId !== activeRunId.current) return; + + setStatus("processing_audio"); + const audioData = await decodeAudio(blob); + + if (runId !== activeRunId.current) return; + + workerRef.current = new TranscriptionWorker( + s => { + if (runId === activeRunId.current) setStatus(s); + }, + out => { + if (runId === activeRunId.current) { + setResult(out); + setStatus("complete"); + } + }, + err => { + if (runId === activeRunId.current) { + setError(String(err)); + setStatus("error"); + } + }, + partial => { + if (runId === activeRunId.current) setResult(partial); + } + ); + + workerRef.current.run( + audioData, + selectedModel, + quantized, + undefined, + "transcribe" + ); + } catch (err) { + if (runId === activeRunId.current) { + setError(String(err)); + setStatus("error"); + } + } + }; + + const handleClick = (e: React.MouseEvent) => { + e.stopPropagation(); + if (embed) { + if (!isOpen) { + if (status === "idle" || status === "error") { + startTranscription(); + } else { + setIsOpen(true); + } + } else { + setIsOpen(false); + } + } else { + openModal(modalProps => ); + } + }; + + const displayText = formatTranscript(result, showTimestamps); + + const handleCopy = (e: React.MouseEvent) => { + e.stopPropagation(); + if (!displayText) return; + copyToClipboard(displayText); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + }; + + const isWorking = status === "downloading_audio" || status === "processing_audio" || status === "loading" || status === "transcribing"; + + return ( + <> + + {embed && isOpen && ( +
e.stopPropagation()} + > +
+
+ + + {status === "downloading_audio" && "Downloading Audio..."} + {status === "processing_audio" && "Processing Audio..."} + {status === "loading" && "Loading Model..."} + {status === "transcribing" && "Transcribing..."} + {status === "complete" && "Transcription"} + {status === "error" && "Transcription Error"} + +
+
+ {result && ( + <> + + {props => ( + + )} + + + {props => ( + + )} + + + )} + + {props => ( + + )} + +
+
+
+ {error ? ( + + {error} + + + ) : displayText ? ( + + {displayText} + + ) : ( + + {isWorking ? "Transcribing in progress..." : "Initializing..."} + + )} +
+
+ )} + + ); +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx index 45e6eb9bb..17e1486e8 100644 --- a/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx @@ -6,621 +6,20 @@ import "./style.css"; -import { DataStore } from "@api/index"; -import { definePluginSettings } from "@api/Settings"; -import { BaseText } from "@components/BaseText"; -import { Button } from "@components/Button"; import ErrorBoundary from "@components/ErrorBoundary"; -import { Flex } from "@components/Flex"; -import { Heading } from "@components/Heading"; -import { ClockIcon, CopyIcon } from "@components/Icons"; -import { Span } from "@components/Span"; -import { copyToClipboard } from "@utils/clipboard"; import { Devs, EquicordDevs } from "@utils/constants"; -import { classes } from "@utils/misc"; -import definePlugin, { OptionType, PluginNative } from "@utils/types"; -import { RenderModalProps } from "@vencord/discord-types"; -import { findComponentByCodeLazy } from "@webpack"; -import { Modal, openModal, ScrollerAuto, SearchableSelect, Tooltip, useEffect, useRef, useState } from "@webpack/common"; +import definePlugin from "@utils/types"; -import { cl, decodeAudio, LANGUAGES, TranscriptionResult, TranscriptionWorker } from "./utils"; -const Native = VencordNative.pluginHelpers.VoiceMessageTranscriber as PluginNative; +import { setManaBaseRadioGroup } from "./components/LanguageSelectionModal"; +import { VoiceMessageButton } from "./components/VoiceMessageButton"; +import { settings } from "./settings"; -const ChannelListIcon = findComponentByCodeLazy("1-1-1ZM2 8a1"); -let ManaBaseRadioGroup: React.ComponentType<{ options: { name: string; value: string; }[]; value: string; onChange: (v: string) => void; }>; - -function formatTimestamp(seconds: number) { - const m = Math.floor(seconds / 60); - const s = Math.floor(seconds % 60); - return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`; -} - -function CloseIcon({ size = 14 }: { size?: number; }) { - return ( - - - - - ); -} - -function CheckmarkIcon({ size = 14 }: { size?: number; }) { - return ( - - - - ); -} - -const MODEL_SIZES: Record = { - "Xenova/whisper-tiny": { quantized: "~40 MB", full: "~150 MB" }, - "Xenova/whisper-base": { quantized: "~77 MB", full: "~290 MB" }, - "Xenova/whisper-small": { quantized: "~250 MB", full: "~960 MB" }, - "Xenova/whisper-medium": { quantized: "~765 MB", full: "~3.1 GB" }, -}; - -function renderModelOption(option?: { label: string; value: string; }) { - if (!option) return null; - const isQuantized = settings.use(["quantized"])?.quantized ?? true; - const size = MODEL_SIZES[option.value]?.[isQuantized ? "quantized" : "full"]; - - return ( -
- {option.label} - {size && ( - - {size} - - )} -
- ); -} - -const settings = definePluginSettings({ - embed: { - type: OptionType.BOOLEAN, - description: "Display transcription directly in the voice message attachment instead of a modal.", - default: false, - restartNeeded: false - }, - maintainHorizontal: { - type: OptionType.BOOLEAN, - description: "Maintain horizontal size for the embedded transcription box and expand vertically.", - default: false, - restartNeeded: false - }, - selectedModel: { - type: OptionType.SELECT, - description: "Model size.", - options: [ - { - label: "Tiny (Fastest, lowest accuracy)", - value: "Xenova/whisper-tiny", - }, - { - label: "Base (Recommended)", - value: "Xenova/whisper-base", - default: true - }, - { - label: "Small", - value: "Xenova/whisper-small" - }, - { - label: "Medium (Slowest, best accuracy)", - value: "Xenova/whisper-medium" - } - ], - componentProps: { - renderOptionLabel: (option: { label: string; value: string; }) => renderModelOption(option), - renderOptionValue: (options: { label: string; value: string; }[]) => renderModelOption(options?.[0]), - }, - restartNeeded: false - }, - quantized: { - type: OptionType.BOOLEAN, - description: "Use quantized models (smaller size, slight quality loss).", - default: true, - restartNeeded: false - }, - deleteModalFiles: { - type: OptionType.COMPONENT, - description: "Delete cached files from storage.", - component: () => { - const [size, setSize] = useState(0); - const [deleteKeys, setDeleteKeys] = useState([]); - - useEffect(() => { - let unmounted = false; - DataStore.entries().then(entries => { - if (unmounted) return; - let totalSize = 0; - const keys: string[] = []; - for (const [key, value] of entries) { - if (typeof key === "string" && (key.startsWith("whisper-") || key.startsWith("onnx-"))) { - totalSize += (value as string).length; - keys.push(key); - } - } - setSize(totalSize); - setDeleteKeys(keys); - }); - return () => { unmounted = true; }; - }, []); - - return ; - } - } -}); - -function LanguageSelectionModal(props: { modalProps: RenderModalProps, src: string; }) { - const { modalProps, src } = props; - const [language, setLanguage] = useState("auto"); - const [task, setTask] = useState("transcribe"); - - const languageOptions = [ - { label: "Auto Detect", value: "auto" }, - ...Object.entries(LANGUAGES).map(([code, name]) => ({ - label: name.charAt(0).toUpperCase() + name.slice(1), - value: code - })) - ]; - - const start = () => { - modalProps.onClose(); - openModal(modalProps => ( - - )); - }; - - return ( - - -
- - Audio Language - - o.value === language)?.value} - onChange={setLanguage} - /> -
- -
- - Action - - setTask(v as string)} - /> -
- - - ); -} - -function TranscriptionModal(props: { modalProps: RenderModalProps, src: string, options: { language: string, task: string; }; }) { - const { modalProps, src, options } = props; - const [status, setStatus] = useState("initializing"); - const [result, setResult] = useState(null); - const [showTimestamps, setShowTimestamps] = useState(false); - const [error, setError] = useState(null); - const [copied, setCopied] = useState(false); - const [retryCount, setRetryCount] = useState(0); - - const workerRef = useRef(null); - - useEffect(() => { - let active = true; - (async () => { - try { - setStatus("downloading_audio"); - setError(null); - - let blob: Blob; - if (IS_DISCORD_DESKTOP || IS_EQUIBOP) { - const data = await Native.fetchAudio(src); - blob = new Blob([new Uint8Array(data)]); - } else { - const res = await fetch(src); - if (!res.ok) throw new Error("Failed to download audio"); - blob = await res.blob(); - } - - if (!active) return; - setStatus("processing_audio"); - const audioData = await decodeAudio(blob); - - if (!active) return; - workerRef.current?.terminate(); - workerRef.current = new TranscriptionWorker( - s => { - if (active) setStatus(s); - }, - out => { - if (active) { - setResult(out); - setStatus("complete"); - } - }, - err => { - if (active) { - setError(String(err)); - setStatus("error"); - } - }, - partial => { - if (active) setResult(partial); - } - ); - - const { quantized, selectedModel } = settings.store; - workerRef.current.run( - audioData, - selectedModel, - quantized, - options.language === "auto" ? undefined : options.language, - options.task - ); - } catch (err) { - if (active) { - setError(String(err)); - setStatus("error"); - } - } - })(); - - return () => { - active = false; - workerRef.current?.terminate(); - }; - }, [retryCount]); - - const handleCopy = () => { - if (!result) return; - const text = showTimestamps - ? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n") - : result.text; - copyToClipboard(text); - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }; - - const displayText = result ? ( - showTimestamps - ? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n") - : result.text - ) : ""; - - return ( - setRetryCount(c => c + 1) - }] : []), - ...(result ? [ - { - text: showTimestamps ? "Hide Timestamps" : "Show Timestamps", - variant: "secondary" as const, - onClick: () => setShowTimestamps(!showTimestamps) - }, - { - text: copied ? "Copied!" : "Copy Text", - variant: "secondary" as const, - onClick: handleCopy - } - ] : []) - ]} - > -
- {error ? ( - - Transcription Failed - - {error} - - - ) : displayText ? ( -
- {displayText} -
- ) : ( - - - {status === "downloading_audio" && "Downloading Audio..."} - {status === "processing_audio" && "Processing Audio..."} - {status === "loading" && "Loading Model..."} - {status === "transcribing" && "Transcribing..."} - - - )} -
- - ); -} - -function VoiceMessageTranscriber({ src }: { src: string; }) { - const { embed, maintainHorizontal, quantized, selectedModel } = settings.use(["embed", "maintainHorizontal", "quantized", "selectedModel"]); - const [isOpen, setIsOpen] = useState(false); - const [status, setStatus] = useState("idle"); - const [result, setResult] = useState(null); - const [showTimestamps, setShowTimestamps] = useState(false); - const [error, setError] = useState(null); - const [copied, setCopied] = useState(false); - - const workerRef = useRef(null); - const activeRunId = useRef(0); - const buttonRef = useRef(null); - - useEffect(() => { - const parent = buttonRef.current?.parentElement; - if (!parent) return; - parent.classList.add("vc-transcription-container"); - return () => { - parent.classList.remove("vc-transcription-container", "vc-transcription-has-embed", "vc-transcription-maintain-horizontal"); - }; - }, []); - - useEffect(() => { - const parent = buttonRef.current?.parentElement; - if (!parent) return; - parent.classList.toggle("vc-transcription-has-embed", Boolean(embed && isOpen)); - parent.classList.toggle("vc-transcription-maintain-horizontal", Boolean(embed && isOpen && maintainHorizontal)); - }, [embed, isOpen, maintainHorizontal]); - - useEffect(() => { - return () => { - activeRunId.current++; - workerRef.current?.terminate(); - workerRef.current = null; - }; - }, []); - - const startTranscription = async () => { - const runId = ++activeRunId.current; - workerRef.current?.terminate(); - workerRef.current = null; - - setIsOpen(true); - setError(null); - setStatus("downloading_audio"); - - try { - let blob: Blob; - if (IS_DISCORD_DESKTOP || IS_EQUIBOP) { - const data = await Native.fetchAudio(src); - blob = new Blob([new Uint8Array(data)]); - } else { - const res = await fetch(src); - if (!res.ok) throw new Error("Failed to download audio"); - blob = await res.blob(); - } - - if (runId !== activeRunId.current) return; - - setStatus("processing_audio"); - const audioData = await decodeAudio(blob); - - if (runId !== activeRunId.current) return; - - workerRef.current = new TranscriptionWorker( - s => { - if (runId === activeRunId.current) setStatus(s); - }, - out => { - if (runId === activeRunId.current) { - setResult(out); - setStatus("complete"); - } - }, - err => { - if (runId === activeRunId.current) { - setError(String(err)); - setStatus("error"); - } - }, - partial => { - if (runId === activeRunId.current) setResult(partial); - } - ); - - workerRef.current.run( - audioData, - selectedModel, - quantized, - undefined, - "transcribe" - ); - } catch (err) { - if (runId === activeRunId.current) { - setError(String(err)); - setStatus("error"); - } - } - }; - - const handleClick = (e: React.MouseEvent) => { - e.stopPropagation(); - if (embed) { - if (!isOpen) { - if (status === "idle" || status === "error") { - startTranscription(); - } else { - setIsOpen(true); - } - } else { - setIsOpen(false); - } - } else { - openModal(modalProps => ); - } - }; - - const displayText = result ? ( - showTimestamps - ? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n") - : result.text - ) : ""; - - const handleCopy = (e: React.MouseEvent) => { - e.stopPropagation(); - if (!displayText) return; - copyToClipboard(displayText); - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }; - - const isWorking = status === "downloading_audio" || status === "processing_audio" || status === "loading" || status === "transcribing"; - - return ( - <> - - {embed && isOpen && ( -
e.stopPropagation()} - > -
-
- - - {status === "downloading_audio" && "Downloading Audio..."} - {status === "processing_audio" && "Processing Audio..."} - {status === "loading" && "Loading Model..."} - {status === "transcribing" && "Transcribing..."} - {status === "complete" && "Transcription"} - {status === "error" && "Transcription Error"} - -
-
- {result && ( - <> - - {props => ( - - )} - - - {props => ( - - )} - - - )} - - {props => ( - - )} - -
-
-
- {error ? ( - - {error} - - - ) : displayText ? ( - - {displayText} - - ) : ( - - {isWorking ? "Transcribing in progress..." : "Initializing..."} - - )} -
-
- )} - - ); -} - -const VoiceMessageTranscriberWrapped = ErrorBoundary.wrap(VoiceMessageTranscriber, { noop: true }); +const VoiceMessageButtonWrapped = ErrorBoundary.wrap(VoiceMessageButton, { noop: true }); export default definePlugin({ name: "VoiceMessageTranscriber", authors: [Devs.TheSun, EquicordDevs.tt], - description: "On-device transcriptions for voice messages powered by Whisper v3", + description: "On-device transcriptions for voice messages powered by Whisper", tags: ["Chat", "Media", "Utility", "Voice"], patches: [ { @@ -638,12 +37,12 @@ export default definePlugin({ } }, ], - set ManaBaseRadioGroup(value: typeof ManaBaseRadioGroup) { - ManaBaseRadioGroup = value; + set ManaBaseRadioGroup(value: Parameters[0]) { + setManaBaseRadioGroup(value); }, settings, button(src: string) { - return ; + return ; }, }); diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/settings.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/settings.tsx new file mode 100644 index 000000000..7c77411d2 --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/settings.tsx @@ -0,0 +1,93 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { definePluginSettings } from "@api/Settings"; +import { OptionType } from "@utils/types"; + +import { DeleteCacheButton } from "./components/DeleteCacheButton"; +import { cl } from "./utils/misc"; + +const MODEL_SIZES: Record = { + "Xenova/whisper-tiny": { quantized: "~40 MB", full: "~150 MB" }, + "Xenova/whisper-base": { quantized: "~77 MB", full: "~290 MB" }, + "Xenova/whisper-small": { quantized: "~250 MB", full: "~960 MB" }, + "Xenova/whisper-medium": { quantized: "~765 MB", full: "~3.1 GB" }, +}; + +function renderModelOption(option?: { label: string; value: string; }) { + if (!option) return null; + return ; +} + +function ModelOption({ option }: { option: { label: string; value: string; }; }) { + const isQuantized = settings.use(["quantized"]).quantized ?? true; + const size = MODEL_SIZES[option.value]?.[isQuantized ? "quantized" : "full"]; + + return ( +
+ {option.label} + {size && ( + + {size} + + )} +
+ ); +} + +export const settings = definePluginSettings({ + embed: { + type: OptionType.BOOLEAN, + description: "Display transcription directly in the voice message attachment instead of a modal.", + default: false, + restartNeeded: false + }, + maintainHorizontal: { + type: OptionType.BOOLEAN, + description: "Maintain horizontal size for the embedded transcription box and expand vertically.", + default: false, + restartNeeded: false + }, + selectedModel: { + type: OptionType.SELECT, + description: "Model size.", + options: [ + { + label: "Tiny (Fastest, lowest accuracy)", + value: "Xenova/whisper-tiny", + }, + { + label: "Base (Recommended)", + value: "Xenova/whisper-base", + default: true + }, + { + label: "Small", + value: "Xenova/whisper-small" + }, + { + label: "Medium (Slowest, best accuracy)", + value: "Xenova/whisper-medium" + } + ], + componentProps: { + renderOptionLabel: (option: { label: string; value: string; }) => renderModelOption(option), + renderOptionValue: (options: { label: string; value: string; }[]) => renderModelOption(options?.[0]), + }, + restartNeeded: false + }, + quantized: { + type: OptionType.BOOLEAN, + description: "Use quantized models (smaller size, slight quality loss).", + default: true, + restartNeeded: false + }, + deleteModalFiles: { + type: OptionType.COMPONENT, + description: "Delete cached files from storage.", + component: DeleteCacheButton + } +}); diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/utils/audio.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/audio.ts new file mode 100644 index 000000000..73068410a --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/audio.ts @@ -0,0 +1,52 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { PluginNative } from "@utils/types"; + +const Native = VencordNative.pluginHelpers.VoiceMessageTranscriber as PluginNative; + +export async function fetchVoiceMessage(src: string): Promise { + if (IS_DISCORD_DESKTOP || IS_EQUIBOP) { + const data = await Native.fetchAudio(src); + return new Blob([new Uint8Array(data)]); + } + + const res = await fetch(src); + if (!res.ok) throw new Error("Failed to download audio"); + return res.blob(); +} + +const AudioContextClass = window.AudioContext || (window as Window & { webkitAudioContext?: typeof AudioContext; }).webkitAudioContext; +const getAudioContext = () => { + if (!AudioContextClass) throw new Error("AudioContext is not supported in this environment"); + return new AudioContextClass({ sampleRate: 16000 }); +}; +export async function decodeAudio(blob: Blob): Promise { + const arrayBuffer = await blob.arrayBuffer(); + const audioContext = getAudioContext(); + let audioBuffer: AudioBuffer; + try { + audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + } finally { + audioContext.close().catch(() => { }); + } + + // Mix down to mono + const channelData = audioBuffer.getChannelData(0); + if (audioBuffer.numberOfChannels > 1) { + for (let i = 1; i < audioBuffer.numberOfChannels; i++) { + const channel = audioBuffer.getChannelData(i); + for (let j = 0; j < channelData.length; j++) { + channelData[j] += channel[j]; + } + } + for (let i = 0; i < channelData.length; i++) { + channelData[i] /= audioBuffer.numberOfChannels; + } + } + + return channelData; +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/utils/languages.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/languages.ts new file mode 100644 index 000000000..baff25a97 --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/languages.ts @@ -0,0 +1,107 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +export const LANGUAGES = { + en: "english", + zh: "chinese", + de: "german", + es: "spanish/castilian", + ru: "russian", + ko: "korean", + fr: "french", + ja: "japanese", + pt: "portuguese", + tr: "turkish", + pl: "polish", + ca: "catalan/valencian", + nl: "dutch/flemish", + ar: "arabic", + sv: "swedish", + it: "italian", + id: "indonesian", + hi: "hindi", + fi: "finnish", + vi: "vietnamese", + he: "hebrew", + uk: "ukrainian", + el: "greek", + ms: "malay", + cs: "czech", + ro: "romanian/moldavian/moldovan", + da: "danish", + hu: "hungarian", + ta: "tamil", + no: "norwegian", + th: "thai", + ur: "urdu", + hr: "croatian", + bg: "bulgarian", + lt: "lithuanian", + la: "latin", + mi: "maori", + ml: "malayalam", + cy: "welsh", + sk: "slovak", + te: "telugu", + fa: "persian", + lv: "latvian", + bn: "bengali", + sr: "serbian", + az: "azerbaijani", + sl: "slovenian", + kn: "kannada", + et: "estonian", + mk: "macedonian", + br: "breton", + eu: "basque", + is: "icelandic", + hy: "armenian", + ne: "nepali", + mn: "mongolian", + bs: "bosnian", + kk: "kazakh", + sq: "albanian", + sw: "swahili", + gl: "galician", + mr: "marathi", + pa: "punjabi/panjabi", + si: "sinhala/sinhalese", + km: "khmer", + sn: "shona", + yo: "yoruba", + so: "somali", + af: "afrikaans", + oc: "occitan", + ka: "georgian", + be: "belarusian", + tg: "tajik", + sd: "sindhi", + gu: "gujarati", + am: "amharic", + yi: "yiddish", + lo: "lao", + uz: "uzbek", + fo: "faroese", + ht: "haitian creole/haitian", + ps: "pashto/pushto", + tk: "turkmen", + nn: "nynorsk", + mt: "maltese", + sa: "sanskrit", + lb: "luxembourgish/letzeburgesch", + my: "myanmar/burmese", + bo: "tibetan", + tl: "tagalog", + mg: "malagasy", + as: "assamese", + tt: "tatar", + haw: "hawaiian", + ln: "lingala", + ha: "hausa", + ba: "bashkir", + jw: "javanese", + su: "sundanese", +}; diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/utils/misc.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/misc.ts new file mode 100644 index 000000000..706139a59 --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/misc.ts @@ -0,0 +1,24 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { classNameFactory } from "@utils/css"; + +import type { TranscriptionResult } from "./worker"; + +export const cl = classNameFactory("vc-transcription-"); + +export function formatTimestamp(seconds: number) { + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`; +} + +export function formatTranscript(result: TranscriptionResult | null, showTimestamps: boolean) { + if (!result) return ""; + return showTimestamps + ? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n") + : result.text; +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/worker.ts similarity index 67% rename from src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts rename to src/equicordplugins/voiceMessageTranscriber.desktop/utils/worker.ts index af728b854..968dd97d7 100644 --- a/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/utils/worker.ts @@ -5,140 +5,8 @@ */ import { DataStore } from "@api/index"; -import { classNameFactory } from "@utils/css"; import { lodash } from "@webpack/common"; -export const LANGUAGES = { - en: "english", - zh: "chinese", - de: "german", - es: "spanish/castilian", - ru: "russian", - ko: "korean", - fr: "french", - ja: "japanese", - pt: "portuguese", - tr: "turkish", - pl: "polish", - ca: "catalan/valencian", - nl: "dutch/flemish", - ar: "arabic", - sv: "swedish", - it: "italian", - id: "indonesian", - hi: "hindi", - fi: "finnish", - vi: "vietnamese", - he: "hebrew", - uk: "ukrainian", - el: "greek", - ms: "malay", - cs: "czech", - ro: "romanian/moldavian/moldovan", - da: "danish", - hu: "hungarian", - ta: "tamil", - no: "norwegian", - th: "thai", - ur: "urdu", - hr: "croatian", - bg: "bulgarian", - lt: "lithuanian", - la: "latin", - mi: "maori", - ml: "malayalam", - cy: "welsh", - sk: "slovak", - te: "telugu", - fa: "persian", - lv: "latvian", - bn: "bengali", - sr: "serbian", - az: "azerbaijani", - sl: "slovenian", - kn: "kannada", - et: "estonian", - mk: "macedonian", - br: "breton", - eu: "basque", - is: "icelandic", - hy: "armenian", - ne: "nepali", - mn: "mongolian", - bs: "bosnian", - kk: "kazakh", - sq: "albanian", - sw: "swahili", - gl: "galician", - mr: "marathi", - pa: "punjabi/panjabi", - si: "sinhala/sinhalese", - km: "khmer", - sn: "shona", - yo: "yoruba", - so: "somali", - af: "afrikaans", - oc: "occitan", - ka: "georgian", - be: "belarusian", - tg: "tajik", - sd: "sindhi", - gu: "gujarati", - am: "amharic", - yi: "yiddish", - lo: "lao", - uz: "uzbek", - fo: "faroese", - ht: "haitian creole/haitian", - ps: "pashto/pushto", - tk: "turkmen", - nn: "nynorsk", - mt: "maltese", - sa: "sanskrit", - lb: "luxembourgish/letzeburgesch", - my: "myanmar/burmese", - bo: "tibetan", - tl: "tagalog", - mg: "malagasy", - as: "assamese", - tt: "tatar", - haw: "hawaiian", - ln: "lingala", - ha: "hausa", - ba: "bashkir", - jw: "javanese", - su: "sundanese", -}; - -export const cl = classNameFactory("vc-transcription-"); - -const AudioContextClass = window.AudioContext || (window as Window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext; -const getAudioContext = () => { - if (!AudioContextClass) throw new Error("AudioContext is not supported in this environment"); - return new AudioContextClass({ sampleRate: 16000 }); -}; -export async function decodeAudio(blob: Blob): Promise { - const arrayBuffer = await blob.arrayBuffer(); - const audioContext = getAudioContext(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - - // Mix down to mono - const channelData = audioBuffer.getChannelData(0); - if (audioBuffer.numberOfChannels > 1) { - for (let i = 1; i < audioBuffer.numberOfChannels; i++) { - const channel = audioBuffer.getChannelData(i); - for (let j = 0; j < channelData.length; j++) { - channelData[j] += channel[j]; - } - } - for (let i = 0; i < channelData.length; i++) { - channelData[i] /= audioBuffer.numberOfChannels; - } - } - - return channelData; -} - const workerCode = ` import { pipeline, env } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2'; @@ -183,6 +51,13 @@ globalThis.fetch = async (input, init) => { let transcriber = null; +async function compressionRatio(text) { + const bytes = new TextEncoder().encode(text); + if (!bytes.length) return 0; + const compressed = await new Response(new Blob([bytes]).stream().pipeThrough(new CompressionStream('deflate-raw'))).arrayBuffer(); + return bytes.length / compressed.byteLength; +} + async function runTranscription({ audio, model, quantized, language, task }) { try { if (!transcriber) { @@ -201,12 +76,7 @@ async function runTranscription({ audio, model, quantized, language, task }) { transcriber.processor.feature_extractor.config.chunk_length / transcriber.model.config.max_source_positions; - let chunks_to_process = [ - { - tokens: [], - finalised: false, - }, - ]; + let chunks_to_process; function chunk_callback(chunk) { let last = chunks_to_process[chunks_to_process.length - 1]; @@ -242,17 +112,27 @@ async function runTranscription({ audio, model, quantized, language, task }) { }); } - const output = await transcriber(audio, { - top_k: 0, - do_sample: false, - chunk_length_s: 30, - stride_length_s: 5, - return_timestamps: true, - callback_function, - chunk_callback, - language, - task: task === "translate" ? "translate" : undefined - }); + async function transcribe(extraOptions) { + chunks_to_process = [{ tokens: [], finalised: false }]; + return transcriber(audio, { + top_k: 0, + do_sample: false, + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + callback_function, + chunk_callback, + language, + task: task === "translate" ? "translate" : undefined, + ...extraOptions + }); + } + + let output = await transcribe(); + + if (await compressionRatio(output.text) > 2.4) { + output = await transcribe({ no_repeat_ngram_size: 4 }); + } self.postMessage({ type: 'complete', output });