From a3ba20144154a7b3631f5067b9b6963462819efc Mon Sep 17 00:00:00 2001 From: sunnie <78964224+sunnniee@users.noreply.github.com> Date: Sat, 17 Jan 2026 21:41:29 +0200 Subject: [PATCH] feat(plugin): VoiceMessageTranscriber (#652) * plugin: VoiceMessageTranscriber * lint * standardize class names * bleh * bleh * bleh --------- Co-authored-by: thororen1234 <78185467+thororen1234@users.noreply.github.com> --- .../voiceMessageTranscriber.desktop/index.tsx | 339 ++++++++++++++++ .../voiceMessageTranscriber.desktop/native.ts | 16 + .../voiceMessageTranscriber.desktop/style.css | 27 ++ .../voiceMessageTranscriber.desktop/utils.ts | 363 ++++++++++++++++++ 4 files changed, 745 insertions(+) create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/native.ts create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/style.css create mode 100644 src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx b/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx new file mode 100644 index 000000000..3981890c2 --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx @@ -0,0 +1,339 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2023 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import "./style.css"; + +import { DataStore } from "@api/index"; +import { definePluginSettings } from "@api/Settings"; +import { BaseText } from "@components/BaseText"; +import { Button } from "@components/Button"; +import { Flex } from "@components/Flex"; +import { FormSwitch } from "@components/FormSwitch"; +import { Heading } from "@components/Heading"; +import { Span } from "@components/Span"; +import { ManaBaseRadioGroupProps } from "@equicordplugins/components.dev/types"; +import { copyToClipboard } from "@utils/clipboard"; +import { Devs } from "@utils/constants"; +import { ModalCloseButton, ModalContent, ModalHeader, ModalProps, ModalRoot, ModalSize, openModal } from "@utils/modal"; +import definePlugin, { OptionType, PluginNative } from "@utils/types"; +import { findComponentByCodeLazy } from "@webpack"; +import { lodash, ScrollerAuto, SearchableSelect, useEffect, useRef, useState } from "@webpack/common"; + +import { cl, decodeAudio, LANGUAGES, TranscriptionWorker } from "./utils"; +const Native = VencordNative.pluginHelpers.VoiceMessageTranscriber as PluginNative; + +const ManaBaseRadioGroup = findComponentByCodeLazy('"data-mana-component":"BaseRadioGroup"') as React.ComponentType; +const ChannelListIcon = findComponentByCodeLazy("1-1-1ZM2 8a1"); + +const settings = definePluginSettings({ + selectedModel: { + type: OptionType.SELECT, + description: "Model size", + options: [ + { + label: "Tiny (Fastest, lowest accuracy)", + value: "Xenova/whisper-tiny", + }, + { + label: "Base (Recommended)", + value: "Xenova/whisper-base", + default: true + }, + { + label: "Small", + value: "Xenova/whisper-small" + }, + { + label: "Medium (Slowest, best accuracy)", + value: "Xenova/whisper-medium" + } + ], + restartNeeded: false + }, + quantized: { + type: OptionType.BOOLEAN, + description: "Use quantized model (smaller size, slightly lower accuracy)", + default: true, + restartNeeded: false + }, + delete: { + type: OptionType.COMPONENT, + component: () => { + const [size, setSize] = useState(0); + const [deleteKeys, setDeleteKeys] = useState([]); + + useEffect(() => { + DataStore.entries().then(entries => { + let size = 0; + const keys = [] as string[]; + + entries.forEach(([key, val]) => { + if (typeof key === "string" && key.startsWith("VoiceMessageTranscriber") && lodash.isArrayBuffer(val)) { + keys.push(key); + size += val.byteLength ?? 0; + } + }); + + setSize(size); + setDeleteKeys(keys); + }); + }, []); + + return ; + } + } +}); + +function LanguageSelectionModal(props: { modalProps: ModalProps, src: string; }) { + const { modalProps, src } = props; + const [language, setLanguage] = useState("auto"); + const [task, setTask] = useState("transcribe"); + + const languageOptions = [ + { label: "Auto Detect", value: "auto" }, + ...Object.entries(LANGUAGES).map(([code, name]) => ({ + label: name.charAt(0).toUpperCase() + name.slice(1), + value: code + })) + ]; + + const start = () => { + modalProps.onClose(); + openModal(modalProps => ( + + )); + }; + + return ( + + + Transcription Options + + + + +
+ + Audio Language + + o.value === language)} + onChange={setLanguage} + /> +
+ +
+ + Action + + setTask(v as string)} + /> +
+ + + + + + ); +} + +function TranscriptionModal(props: { modalProps: ModalProps, src: string, options: { language: string, task: string; }; }) { + const { modalProps, src, options } = props; + const [status, setStatus] = useState("initializing"); + const [result, setResult] = useState<{ text: string, chunks: { timestamp: [number, number], text: string; }[]; } | null>(null); + const [showTimestamps, setShowTimestamps] = useState(false); + const [error, setError] = useState(null); + const [copied, setCopied] = useState(false); + const [retryCount, setRetryCount] = useState(0); + + const workerRef = useRef(null); + + useEffect(() => { + let active = true; + (async () => { + try { + setStatus("downloading_audio"); + setError(null); + + let blob: Blob; + if (IS_DISCORD_DESKTOP || IS_EQUIBOP) { + const arrayBuffer = await Native.fetchAudio(src); + blob = new Blob([arrayBuffer as any]); + } else { + const res = await fetch(src); + if (!res.ok) throw new Error("Failed to download audio"); + blob = await res.blob(); + } + + if (!active) return; + setStatus("processing_audio"); + const audioData = await decodeAudio(blob); + + if (!active) return; + workerRef.current = new TranscriptionWorker( + s => { + if (active) setStatus(s); + }, + out => { + if (active) { + setResult(out); + setStatus("complete"); + } + }, + err => { + if (active) setError(String(err)); + }, + partial => { + if (active) setResult(partial); + } + ); + + const { quantized, selectedModel } = settings.store; + workerRef.current.run( + audioData, + selectedModel, + quantized, + options.language === "auto" ? undefined : options.language, + options.task + ); + } catch (err) { + if (active) setError(String(err)); + } + })(); + + return () => { + active = false; + workerRef.current?.terminate(); + }; + }, [src, retryCount]); + + const retry = () => { + setError(null); + setStatus("initializing"); + setResult(null); + setCopied(false); + setRetryCount(prev => prev + 1); + }; + + const formatTimestamp = (seconds: number) => { + const m = Math.floor(seconds / 60); + const s = Math.floor(seconds % 60); + return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`; + }; + + const displayText = result ? ( + showTimestamps + ? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n") + : result.text + ) : ""; + + return + + Transcription + + + + {error ? ( + + Error + {error} + + + ) : (status === "complete" || (status === "transcribing" && result)) ? ( + + {status === "transcribing" && ( + Transcribing in progress... + )} + + {displayText} + + +
+ +
+ + + + ) : ( + + + {status === "initializing" && "Initializing..."} + {status === "downloading_audio" && "Downloading Audio..."} + {status === "processing_audio" && "Processing Audio..."} + {status === "loading" && "Loading Model..."} + {status === "transcribing" && "Transcribing..."} + + + )} + + ; +} + +export default definePlugin({ + name: "VoiceMessageTranscriber", + authors: [Devs.TheSun], + description: "On-device transcriptions for voice messages powered by Whisper v3", + patches: [ + { + find: ".volumeButtonIcon,", + replacement: { + match: /"source",{src:(\i).{0,700}duration:\i}\),/, + replace: "$&$self.button($1)," + } + } + ], + settings, + + button(src: string) { + return ; + }, +}); diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/native.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/native.ts new file mode 100644 index 000000000..e5c5a9c0a --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/native.ts @@ -0,0 +1,16 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { IpcMainInvokeEvent } from "electron"; + +// we love CORS +export async function fetchAudio(_: IpcMainInvokeEvent, url: string): Promise { + if (!url.startsWith("https://cdn.discordapp.com")) throw new Error("Unknown URL"); + const res = await fetch(url); + if (!res.ok) throw new Error(`Failed to fetch ${url}: ${res.statusText}`); + if (!res.headers.get("Content-Type")?.startsWith("audio/")) throw new Error(`${url} is not an audio file`); + return new Uint8Array(await res.arrayBuffer()); +} diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/style.css b/src/equicordplugins/voiceMessageTranscriber.desktop/style.css new file mode 100644 index 000000000..3049ba7bc --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/style.css @@ -0,0 +1,27 @@ +.vc-transcription-button { + width: 24px; + height: 24px; + padding: 0; + cursor: pointer; + position: relative; +} + +.vc-transcription-icon { + color: var(--interactive-icon-default); +} + +.vc-transcription-icon:hover { + color: var(--interactive-background-hover); +} + +.vc-transcription-content { + min-height: 200px; +} + +.vc-transcription-result { + padding: 16px; + white-space: pre-wrap; + user-select: text; + max-height: 400px; + overflow-y: auto; +} \ No newline at end of file diff --git a/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts b/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts new file mode 100644 index 000000000..a2147f75f --- /dev/null +++ b/src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts @@ -0,0 +1,363 @@ +/* + * Vencord, a Discord client mod + * Copyright (c) 2026 Vendicated and contributors + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +import { DataStore } from "@api/index"; +import { classNameFactory } from "@utils/css"; +import { lodash } from "@webpack/common"; + +export const LANGUAGES = { + en: "english", + zh: "chinese", + de: "german", + es: "spanish/castilian", + ru: "russian", + ko: "korean", + fr: "french", + ja: "japanese", + pt: "portuguese", + tr: "turkish", + pl: "polish", + ca: "catalan/valencian", + nl: "dutch/flemish", + ar: "arabic", + sv: "swedish", + it: "italian", + id: "indonesian", + hi: "hindi", + fi: "finnish", + vi: "vietnamese", + he: "hebrew", + uk: "ukrainian", + el: "greek", + ms: "malay", + cs: "czech", + ro: "romanian/moldavian/moldovan", + da: "danish", + hu: "hungarian", + ta: "tamil", + no: "norwegian", + th: "thai", + ur: "urdu", + hr: "croatian", + bg: "bulgarian", + lt: "lithuanian", + la: "latin", + mi: "maori", + ml: "malayalam", + cy: "welsh", + sk: "slovak", + te: "telugu", + fa: "persian", + lv: "latvian", + bn: "bengali", + sr: "serbian", + az: "azerbaijani", + sl: "slovenian", + kn: "kannada", + et: "estonian", + mk: "macedonian", + br: "breton", + eu: "basque", + is: "icelandic", + hy: "armenian", + ne: "nepali", + mn: "mongolian", + bs: "bosnian", + kk: "kazakh", + sq: "albanian", + sw: "swahili", + gl: "galician", + mr: "marathi", + pa: "punjabi/panjabi", + si: "sinhala/sinhalese", + km: "khmer", + sn: "shona", + yo: "yoruba", + so: "somali", + af: "afrikaans", + oc: "occitan", + ka: "georgian", + be: "belarusian", + tg: "tajik", + sd: "sindhi", + gu: "gujarati", + am: "amharic", + yi: "yiddish", + lo: "lao", + uz: "uzbek", + fo: "faroese", + ht: "haitian creole/haitian", + ps: "pashto/pushto", + tk: "turkmen", + nn: "nynorsk", + mt: "maltese", + sa: "sanskrit", + lb: "luxembourgish/letzeburgesch", + my: "myanmar/burmese", + bo: "tibetan", + tl: "tagalog", + mg: "malagasy", + as: "assamese", + tt: "tatar", + haw: "hawaiian", + ln: "lingala", + ha: "hausa", + ba: "bashkir", + jw: "javanese", + su: "sundanese", +}; + +export const cl = classNameFactory("vc-transcription-"); + +const getAudioContext = () => new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 }); +export async function decodeAudio(blob: Blob): Promise { + const arrayBuffer = await blob.arrayBuffer(); + const audioContext = getAudioContext(); + const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); + + // Mix down to mono + const channelData = audioBuffer.getChannelData(0); + if (audioBuffer.numberOfChannels > 1) { + for (let i = 1; i < audioBuffer.numberOfChannels; i++) { + const channel = audioBuffer.getChannelData(i); + for (let j = 0; j < channelData.length; j++) { + channelData[j] += channel[j]; + } + } + for (let i = 0; i < channelData.length; i++) { + channelData[i] /= audioBuffer.numberOfChannels; + } + } + + return channelData; +} + +const workerCode = ` +import { pipeline, env } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2'; + +env.allowLocalModels = false; +env.useBrowserCache = false; + +const pendingRequests = new Map(); + +self.addEventListener('message', (event) => { + const { type, id, response, error, headers } = event.data; + + if (type === 'fetch_response') { + const resolver = pendingRequests.get(id); + if (resolver) { + pendingRequests.delete(id); + if (error) { + resolver.reject(new Error(error)); + } else { + const res = new Response(response, { + headers: headers || { 'Content-Type': 'application/octet-stream' } + }); + resolver.resolve(res); + } + } + } else if (type === 'run') { + runTranscription(event.data); + } +}); + +const originalFetch = globalThis.fetch; +globalThis.fetch = async (input, init) => { + const url = input.toString(); + if (url.includes('huggingface.co') || url.includes('cdn.jsdelivr.net')) { + const id = Math.random().toString(36).substring(7); + return new Promise((resolve, reject) => { + pendingRequests.set(id, { resolve, reject }); + self.postMessage({ type: 'fetch_request', url, id }); + }); + } + return originalFetch(input, init); +}; + +let transcriber = null; + +async function runTranscription({ audio, model, quantized, language, task }) { + try { + if (!transcriber) { + self.postMessage({ type: 'status', status: 'loading' }); + transcriber = await pipeline('automatic-speech-recognition', model, { + quantized: quantized, + progress_callback: (data) => { + self.postMessage({ type: 'progress', data }); + } + }); + } + + self.postMessage({ type: 'status', status: 'transcribing' }); + + const time_precision = + transcriber.processor.feature_extractor.config.chunk_length / + transcriber.model.config.max_source_positions; + + let chunks_to_process = [ + { + tokens: [], + finalised: false, + }, + ]; + + function chunk_callback(chunk) { + let last = chunks_to_process[chunks_to_process.length - 1]; + + Object.assign(last, chunk); + last.finalised = true; + + if (!chunk.is_last) { + chunks_to_process.push({ + tokens: [], + finalised: false, + }); + } + } + + function callback_function(item) { + let last = chunks_to_process[chunks_to_process.length - 1]; + + last.tokens = [...item[0].output_token_ids]; + + let data = transcriber.tokenizer._decode_asr(chunks_to_process, { + time_precision: time_precision, + return_timestamps: true, + force_full_sequences: false, + }); + + self.postMessage({ + type: 'partial', + output: { + text: data[0], + chunks: data[1].chunks + } + }); + } + + const output = await transcriber(audio, { + top_k: 0, + do_sample: false, + chunk_length_s: 30, + stride_length_s: 5, + return_timestamps: true, + callback_function, + chunk_callback, + language, + task: task === "translate" ? "translate" : undefined + }); + + self.postMessage({ type: 'complete', output }); + + } catch (e) { + self.postMessage({ type: 'error', error: e.toString() }); + } +} +`; + +export class TranscriptionWorker { + private worker: Worker; + private onStatus: (status: string) => void; + private onComplete: (output: any) => void; + private onError: (error: any) => void; + private onPartial: (output: any) => void; + + constructor( + onStatus: (status: string) => void, + onComplete: (output: any) => void, + onError: (error: any) => void, + onPartial: (output: any) => void + ) { + this.onStatus = onStatus; + this.onComplete = onComplete; + this.onError = onError; + this.onPartial = onPartial; + + const blob = new Blob([workerCode], { type: "text/javascript" }); + this.worker = new Worker(URL.createObjectURL(blob), { type: "module" }); + this.worker.onmessage = this.handleMessage.bind(this); + } + + private getMimeType(url: string): string { + if (url.endsWith(".wasm")) return "application/wasm"; + if (url.endsWith(".json")) return "application/json"; + if (url.endsWith(".onnx")) return "application/octet-stream"; + return "application/octet-stream"; + } + + private async handleMessage(event: MessageEvent) { + const { type, id, url, status, output, error } = event.data; + + switch (type) { + case "fetch_request": + try { + const cachedData = await DataStore.get(`VoiceMessageTranscriber_${url}`); + + if (cachedData && lodash.isArrayBuffer(cachedData)) { + this.worker.postMessage({ + type: "fetch_response", + id, + response: cachedData, + headers: { + "Content-Length": cachedData.byteLength.toString(), + "Content-Type": this.getMimeType(url) + } + }); + } else { + const res = await fetch(url); + if (!res.ok) throw new Error("Failed to fetch " + url); + + const buffer = await res.arrayBuffer(); + await DataStore.set(`VoiceMessageTranscriber_${url}`, buffer); + + this.worker.postMessage({ + type: "fetch_response", + id, + response: buffer, + headers: { + "Content-Length": res.headers.get("Content-Length") || buffer.byteLength.toString(), + "Content-Type": this.getMimeType(url) + } + }); + } + } catch (err) { + this.worker.postMessage({ + type: "fetch_response", + id, + error: String(err) + }); + } + break; + case "status": + this.onStatus(status); + break; + case "complete": + this.onComplete(output); + break; + case "partial": + this.onPartial(output); + break; + case "error": + this.onError(error); + break; + } + } + + public run(audio: Float32Array, model: string, quantized: boolean = true, language?: string, task?: string) { + this.worker.postMessage({ + type: "run", + audio, + model, + quantized, + language, + task + }); + } + + public terminate() { + this.worker.terminate(); + } +}