feat(plugin): VoiceMessageTranscriber (#652)
* plugin: VoiceMessageTranscriber * lint * standardize class names * bleh * bleh * bleh --------- Co-authored-by: thororen1234 <78185467+thororen1234@users.noreply.github.com>
This commit is contained in:
sunnie
GitHub
parent
9ef9f46ae8
commit
a3ba201441
4 changed files with 745 additions and 0 deletions
339
src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx
Normal file
339
src/equicordplugins/voiceMessageTranscriber.desktop/index.tsx
Normal file
|
|
@ -0,0 +1,339 @@
|
|||
/*
|
||||
* Vencord, a Discord client mod
|
||||
* Copyright (c) 2023 Vendicated and contributors
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||
*/
|
||||
|
||||
import "./style.css";
|
||||
|
||||
import { DataStore } from "@api/index";
|
||||
import { definePluginSettings } from "@api/Settings";
|
||||
import { BaseText } from "@components/BaseText";
|
||||
import { Button } from "@components/Button";
|
||||
import { Flex } from "@components/Flex";
|
||||
import { FormSwitch } from "@components/FormSwitch";
|
||||
import { Heading } from "@components/Heading";
|
||||
import { Span } from "@components/Span";
|
||||
import { ManaBaseRadioGroupProps } from "@equicordplugins/components.dev/types";
|
||||
import { copyToClipboard } from "@utils/clipboard";
|
||||
import { Devs } from "@utils/constants";
|
||||
import { ModalCloseButton, ModalContent, ModalHeader, ModalProps, ModalRoot, ModalSize, openModal } from "@utils/modal";
|
||||
import definePlugin, { OptionType, PluginNative } from "@utils/types";
|
||||
import { findComponentByCodeLazy } from "@webpack";
|
||||
import { lodash, ScrollerAuto, SearchableSelect, useEffect, useRef, useState } from "@webpack/common";
|
||||
|
||||
import { cl, decodeAudio, LANGUAGES, TranscriptionWorker } from "./utils";
|
||||
const Native = VencordNative.pluginHelpers.VoiceMessageTranscriber as PluginNative<typeof import("./native")>;
|
||||
|
||||
const ManaBaseRadioGroup = findComponentByCodeLazy('"data-mana-component":"BaseRadioGroup"') as React.ComponentType<ManaBaseRadioGroupProps>;
|
||||
const ChannelListIcon = findComponentByCodeLazy("1-1-1ZM2 8a1");
|
||||
|
||||
const settings = definePluginSettings({
|
||||
selectedModel: {
|
||||
type: OptionType.SELECT,
|
||||
description: "Model size",
|
||||
options: [
|
||||
{
|
||||
label: "Tiny (Fastest, lowest accuracy)",
|
||||
value: "Xenova/whisper-tiny",
|
||||
},
|
||||
{
|
||||
label: "Base (Recommended)",
|
||||
value: "Xenova/whisper-base",
|
||||
default: true
|
||||
},
|
||||
{
|
||||
label: "Small",
|
||||
value: "Xenova/whisper-small"
|
||||
},
|
||||
{
|
||||
label: "Medium (Slowest, best accuracy)",
|
||||
value: "Xenova/whisper-medium"
|
||||
}
|
||||
],
|
||||
restartNeeded: false
|
||||
},
|
||||
quantized: {
|
||||
type: OptionType.BOOLEAN,
|
||||
description: "Use quantized model (smaller size, slightly lower accuracy)",
|
||||
default: true,
|
||||
restartNeeded: false
|
||||
},
|
||||
delete: {
|
||||
type: OptionType.COMPONENT,
|
||||
component: () => {
|
||||
const [size, setSize] = useState(0);
|
||||
const [deleteKeys, setDeleteKeys] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
DataStore.entries().then(entries => {
|
||||
let size = 0;
|
||||
const keys = [] as string[];
|
||||
|
||||
entries.forEach(([key, val]) => {
|
||||
if (typeof key === "string" && key.startsWith("VoiceMessageTranscriber") && lodash.isArrayBuffer(val)) {
|
||||
keys.push(key);
|
||||
size += val.byteLength ?? 0;
|
||||
}
|
||||
});
|
||||
|
||||
setSize(size);
|
||||
setDeleteKeys(keys);
|
||||
});
|
||||
}, []);
|
||||
|
||||
return <Button
|
||||
variant="dangerPrimary"
|
||||
onClick={() => {
|
||||
DataStore.delMany(deleteKeys).then(() => { setSize(0); setDeleteKeys([]); });
|
||||
}}
|
||||
>
|
||||
Delete all cached files ({(size / 1024 / 1024).toFixed(2)} MB)
|
||||
</Button>;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
function LanguageSelectionModal(props: { modalProps: ModalProps, src: string; }) {
|
||||
const { modalProps, src } = props;
|
||||
const [language, setLanguage] = useState<string>("auto");
|
||||
const [task, setTask] = useState<string>("transcribe");
|
||||
|
||||
const languageOptions = [
|
||||
{ label: "Auto Detect", value: "auto" },
|
||||
...Object.entries(LANGUAGES).map(([code, name]) => ({
|
||||
label: name.charAt(0).toUpperCase() + name.slice(1),
|
||||
value: code
|
||||
}))
|
||||
];
|
||||
|
||||
const start = () => {
|
||||
modalProps.onClose();
|
||||
openModal(modalProps => (
|
||||
<TranscriptionModal
|
||||
modalProps={modalProps}
|
||||
src={src}
|
||||
options={{ language, task }}
|
||||
/>
|
||||
));
|
||||
};
|
||||
|
||||
return (
|
||||
<ModalRoot {...modalProps} size={ModalSize.MEDIUM}>
|
||||
<ModalHeader>
|
||||
<BaseText size="lg" weight="bold" style={{ flexGrow: 1 }}>Transcription Options</BaseText>
|
||||
<ModalCloseButton onClick={modalProps.onClose} />
|
||||
</ModalHeader>
|
||||
<ModalContent style={{ padding: "16px" }}>
|
||||
<Flex flexDirection="column" gap={20}>
|
||||
<div>
|
||||
<BaseText size="sm" weight="semibold" style={{ marginBottom: "8px" }}>
|
||||
Audio Language
|
||||
</BaseText>
|
||||
<SearchableSelect
|
||||
options={languageOptions}
|
||||
value={languageOptions.find(o => o.value === language)}
|
||||
onChange={setLanguage}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<BaseText size="sm" weight="semibold" style={{ marginBottom: "8px" }}>
|
||||
Action
|
||||
</BaseText>
|
||||
<ManaBaseRadioGroup
|
||||
options={[{
|
||||
name: "Transcribe",
|
||||
value: "transcribe"
|
||||
}, {
|
||||
name: "Translate to English",
|
||||
value: "translate"
|
||||
}]}
|
||||
value={task}
|
||||
onChange={v => setTask(v as string)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Button onClick={start} variant="primary">
|
||||
Start
|
||||
</Button>
|
||||
</Flex>
|
||||
</ModalContent>
|
||||
</ModalRoot>
|
||||
);
|
||||
}
|
||||
|
||||
function TranscriptionModal(props: { modalProps: ModalProps, src: string, options: { language: string, task: string; }; }) {
|
||||
const { modalProps, src, options } = props;
|
||||
const [status, setStatus] = useState<string>("initializing");
|
||||
const [result, setResult] = useState<{ text: string, chunks: { timestamp: [number, number], text: string; }[]; } | null>(null);
|
||||
const [showTimestamps, setShowTimestamps] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [copied, setCopied] = useState(false);
|
||||
const [retryCount, setRetryCount] = useState(0);
|
||||
|
||||
const workerRef = useRef<TranscriptionWorker | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let active = true;
|
||||
(async () => {
|
||||
try {
|
||||
setStatus("downloading_audio");
|
||||
setError(null);
|
||||
|
||||
let blob: Blob;
|
||||
if (IS_DISCORD_DESKTOP || IS_EQUIBOP) {
|
||||
const arrayBuffer = await Native.fetchAudio(src);
|
||||
blob = new Blob([arrayBuffer as any]);
|
||||
} else {
|
||||
const res = await fetch(src);
|
||||
if (!res.ok) throw new Error("Failed to download audio");
|
||||
blob = await res.blob();
|
||||
}
|
||||
|
||||
if (!active) return;
|
||||
setStatus("processing_audio");
|
||||
const audioData = await decodeAudio(blob);
|
||||
|
||||
if (!active) return;
|
||||
workerRef.current = new TranscriptionWorker(
|
||||
s => {
|
||||
if (active) setStatus(s);
|
||||
},
|
||||
out => {
|
||||
if (active) {
|
||||
setResult(out);
|
||||
setStatus("complete");
|
||||
}
|
||||
},
|
||||
err => {
|
||||
if (active) setError(String(err));
|
||||
},
|
||||
partial => {
|
||||
if (active) setResult(partial);
|
||||
}
|
||||
);
|
||||
|
||||
const { quantized, selectedModel } = settings.store;
|
||||
workerRef.current.run(
|
||||
audioData,
|
||||
selectedModel,
|
||||
quantized,
|
||||
options.language === "auto" ? undefined : options.language,
|
||||
options.task
|
||||
);
|
||||
} catch (err) {
|
||||
if (active) setError(String(err));
|
||||
}
|
||||
})();
|
||||
|
||||
return () => {
|
||||
active = false;
|
||||
workerRef.current?.terminate();
|
||||
};
|
||||
}, [src, retryCount]);
|
||||
|
||||
const retry = () => {
|
||||
setError(null);
|
||||
setStatus("initializing");
|
||||
setResult(null);
|
||||
setCopied(false);
|
||||
setRetryCount(prev => prev + 1);
|
||||
};
|
||||
|
||||
const formatTimestamp = (seconds: number) => {
|
||||
const m = Math.floor(seconds / 60);
|
||||
const s = Math.floor(seconds % 60);
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
};
|
||||
|
||||
const displayText = result ? (
|
||||
showTimestamps
|
||||
? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n")
|
||||
: result.text
|
||||
) : "";
|
||||
|
||||
return <ModalRoot {...modalProps} size={ModalSize.LARGE}>
|
||||
<ModalHeader>
|
||||
<BaseText size="lg" weight="bold" style={{ flexGrow: 1 }}>Transcription</BaseText>
|
||||
<ModalCloseButton onClick={modalProps.onClose} />
|
||||
</ModalHeader>
|
||||
<ModalContent className={cl("content")}>
|
||||
{error ? (
|
||||
<Flex flexDirection="column" gap={16} style={{ padding: "20px 0" }}>
|
||||
<Heading tag="h3" style={{ color: "var(--red-360)" }}>Error</Heading>
|
||||
<Span style={{ whiteSpace: "pre-wrap" }}>{error}</Span>
|
||||
<Button onClick={retry}>
|
||||
Retry
|
||||
</Button>
|
||||
</Flex>
|
||||
) : (status === "complete" || (status === "transcribing" && result)) ? (
|
||||
<Flex flexDirection="column" gap={16} style={{ paddingBottom: "20px" }}>
|
||||
{status === "transcribing" && (
|
||||
<Span size="sm" color="text-muted">Transcribing in progress...</Span>
|
||||
)}
|
||||
<ScrollerAuto className={cl("result")}>
|
||||
<Span>{displayText}</Span>
|
||||
</ScrollerAuto>
|
||||
<Flex flexDirection="row" gap={12} alignItems="center">
|
||||
<div style={{ flexGrow: 1 }}>
|
||||
<FormSwitch
|
||||
title="Show Timestamps"
|
||||
value={showTimestamps}
|
||||
onChange={setShowTimestamps}
|
||||
/>
|
||||
</div>
|
||||
</Flex>
|
||||
<Button
|
||||
disabled={status === "transcribing" || copied}
|
||||
onClick={() => {
|
||||
copyToClipboard(displayText);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
}}
|
||||
>
|
||||
{copied ? "Copied!" : "Copy to Clipboard"}
|
||||
</Button>
|
||||
</Flex>
|
||||
) : (
|
||||
<Flex flexDirection="column" gap={16} style={{ padding: "20px 0", alignItems: "center" }}>
|
||||
<Heading tag="h3">
|
||||
{status === "initializing" && "Initializing..."}
|
||||
{status === "downloading_audio" && "Downloading Audio..."}
|
||||
{status === "processing_audio" && "Processing Audio..."}
|
||||
{status === "loading" && "Loading Model..."}
|
||||
{status === "transcribing" && "Transcribing..."}
|
||||
</Heading>
|
||||
</Flex>
|
||||
)}
|
||||
</ModalContent>
|
||||
</ModalRoot>;
|
||||
}
|
||||
|
||||
export default definePlugin({
|
||||
name: "VoiceMessageTranscriber",
|
||||
authors: [Devs.TheSun],
|
||||
description: "On-device transcriptions for voice messages powered by Whisper v3",
|
||||
patches: [
|
||||
{
|
||||
find: ".volumeButtonIcon,",
|
||||
replacement: {
|
||||
match: /"source",{src:(\i).{0,700}duration:\i}\),/,
|
||||
replace: "$&$self.button($1),"
|
||||
}
|
||||
}
|
||||
],
|
||||
settings,
|
||||
|
||||
button(src: string) {
|
||||
return <button
|
||||
className={cl("button")}
|
||||
style={{ backgroundColor: "transparent" }}
|
||||
onClick={e => {
|
||||
e.stopPropagation();
|
||||
openModal(modalProps => <LanguageSelectionModal modalProps={modalProps} src={src} />);
|
||||
}}>
|
||||
<ChannelListIcon colorClass={cl("icon")} />
|
||||
</button>;
|
||||
},
|
||||
});
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
/*
|
||||
* Vencord, a Discord client mod
|
||||
* Copyright (c) 2026 Vendicated and contributors
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||
*/
|
||||
|
||||
import { IpcMainInvokeEvent } from "electron";
|
||||
|
||||
// we love CORS
|
||||
export async function fetchAudio(_: IpcMainInvokeEvent, url: string): Promise<Uint8Array> {
|
||||
if (!url.startsWith("https://cdn.discordapp.com")) throw new Error("Unknown URL");
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`Failed to fetch ${url}: ${res.statusText}`);
|
||||
if (!res.headers.get("Content-Type")?.startsWith("audio/")) throw new Error(`${url} is not an audio file`);
|
||||
return new Uint8Array(await res.arrayBuffer());
|
||||
}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
.vc-transcription-button {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.vc-transcription-icon {
|
||||
color: var(--interactive-icon-default);
|
||||
}
|
||||
|
||||
.vc-transcription-icon:hover {
|
||||
color: var(--interactive-background-hover);
|
||||
}
|
||||
|
||||
.vc-transcription-content {
|
||||
min-height: 200px;
|
||||
}
|
||||
|
||||
.vc-transcription-result {
|
||||
padding: 16px;
|
||||
white-space: pre-wrap;
|
||||
user-select: text;
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
363
src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts
Normal file
363
src/equicordplugins/voiceMessageTranscriber.desktop/utils.ts
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
/*
|
||||
* Vencord, a Discord client mod
|
||||
* Copyright (c) 2026 Vendicated and contributors
|
||||
* SPDX-License-Identifier: GPL-3.0-or-later
|
||||
*/
|
||||
|
||||
import { DataStore } from "@api/index";
|
||||
import { classNameFactory } from "@utils/css";
|
||||
import { lodash } from "@webpack/common";
|
||||
|
||||
export const LANGUAGES = {
|
||||
en: "english",
|
||||
zh: "chinese",
|
||||
de: "german",
|
||||
es: "spanish/castilian",
|
||||
ru: "russian",
|
||||
ko: "korean",
|
||||
fr: "french",
|
||||
ja: "japanese",
|
||||
pt: "portuguese",
|
||||
tr: "turkish",
|
||||
pl: "polish",
|
||||
ca: "catalan/valencian",
|
||||
nl: "dutch/flemish",
|
||||
ar: "arabic",
|
||||
sv: "swedish",
|
||||
it: "italian",
|
||||
id: "indonesian",
|
||||
hi: "hindi",
|
||||
fi: "finnish",
|
||||
vi: "vietnamese",
|
||||
he: "hebrew",
|
||||
uk: "ukrainian",
|
||||
el: "greek",
|
||||
ms: "malay",
|
||||
cs: "czech",
|
||||
ro: "romanian/moldavian/moldovan",
|
||||
da: "danish",
|
||||
hu: "hungarian",
|
||||
ta: "tamil",
|
||||
no: "norwegian",
|
||||
th: "thai",
|
||||
ur: "urdu",
|
||||
hr: "croatian",
|
||||
bg: "bulgarian",
|
||||
lt: "lithuanian",
|
||||
la: "latin",
|
||||
mi: "maori",
|
||||
ml: "malayalam",
|
||||
cy: "welsh",
|
||||
sk: "slovak",
|
||||
te: "telugu",
|
||||
fa: "persian",
|
||||
lv: "latvian",
|
||||
bn: "bengali",
|
||||
sr: "serbian",
|
||||
az: "azerbaijani",
|
||||
sl: "slovenian",
|
||||
kn: "kannada",
|
||||
et: "estonian",
|
||||
mk: "macedonian",
|
||||
br: "breton",
|
||||
eu: "basque",
|
||||
is: "icelandic",
|
||||
hy: "armenian",
|
||||
ne: "nepali",
|
||||
mn: "mongolian",
|
||||
bs: "bosnian",
|
||||
kk: "kazakh",
|
||||
sq: "albanian",
|
||||
sw: "swahili",
|
||||
gl: "galician",
|
||||
mr: "marathi",
|
||||
pa: "punjabi/panjabi",
|
||||
si: "sinhala/sinhalese",
|
||||
km: "khmer",
|
||||
sn: "shona",
|
||||
yo: "yoruba",
|
||||
so: "somali",
|
||||
af: "afrikaans",
|
||||
oc: "occitan",
|
||||
ka: "georgian",
|
||||
be: "belarusian",
|
||||
tg: "tajik",
|
||||
sd: "sindhi",
|
||||
gu: "gujarati",
|
||||
am: "amharic",
|
||||
yi: "yiddish",
|
||||
lo: "lao",
|
||||
uz: "uzbek",
|
||||
fo: "faroese",
|
||||
ht: "haitian creole/haitian",
|
||||
ps: "pashto/pushto",
|
||||
tk: "turkmen",
|
||||
nn: "nynorsk",
|
||||
mt: "maltese",
|
||||
sa: "sanskrit",
|
||||
lb: "luxembourgish/letzeburgesch",
|
||||
my: "myanmar/burmese",
|
||||
bo: "tibetan",
|
||||
tl: "tagalog",
|
||||
mg: "malagasy",
|
||||
as: "assamese",
|
||||
tt: "tatar",
|
||||
haw: "hawaiian",
|
||||
ln: "lingala",
|
||||
ha: "hausa",
|
||||
ba: "bashkir",
|
||||
jw: "javanese",
|
||||
su: "sundanese",
|
||||
};
|
||||
|
||||
export const cl = classNameFactory("vc-transcription-");
|
||||
|
||||
const getAudioContext = () => new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 });
|
||||
export async function decodeAudio(blob: Blob): Promise<Float32Array> {
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
const audioContext = getAudioContext();
|
||||
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
|
||||
|
||||
// Mix down to mono
|
||||
const channelData = audioBuffer.getChannelData(0);
|
||||
if (audioBuffer.numberOfChannels > 1) {
|
||||
for (let i = 1; i < audioBuffer.numberOfChannels; i++) {
|
||||
const channel = audioBuffer.getChannelData(i);
|
||||
for (let j = 0; j < channelData.length; j++) {
|
||||
channelData[j] += channel[j];
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < channelData.length; i++) {
|
||||
channelData[i] /= audioBuffer.numberOfChannels;
|
||||
}
|
||||
}
|
||||
|
||||
return channelData;
|
||||
}
|
||||
|
||||
const workerCode = `
|
||||
import { pipeline, env } from 'https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2';
|
||||
|
||||
env.allowLocalModels = false;
|
||||
env.useBrowserCache = false;
|
||||
|
||||
const pendingRequests = new Map();
|
||||
|
||||
self.addEventListener('message', (event) => {
|
||||
const { type, id, response, error, headers } = event.data;
|
||||
|
||||
if (type === 'fetch_response') {
|
||||
const resolver = pendingRequests.get(id);
|
||||
if (resolver) {
|
||||
pendingRequests.delete(id);
|
||||
if (error) {
|
||||
resolver.reject(new Error(error));
|
||||
} else {
|
||||
const res = new Response(response, {
|
||||
headers: headers || { 'Content-Type': 'application/octet-stream' }
|
||||
});
|
||||
resolver.resolve(res);
|
||||
}
|
||||
}
|
||||
} else if (type === 'run') {
|
||||
runTranscription(event.data);
|
||||
}
|
||||
});
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
globalThis.fetch = async (input, init) => {
|
||||
const url = input.toString();
|
||||
if (url.includes('huggingface.co') || url.includes('cdn.jsdelivr.net')) {
|
||||
const id = Math.random().toString(36).substring(7);
|
||||
return new Promise((resolve, reject) => {
|
||||
pendingRequests.set(id, { resolve, reject });
|
||||
self.postMessage({ type: 'fetch_request', url, id });
|
||||
});
|
||||
}
|
||||
return originalFetch(input, init);
|
||||
};
|
||||
|
||||
let transcriber = null;
|
||||
|
||||
async function runTranscription({ audio, model, quantized, language, task }) {
|
||||
try {
|
||||
if (!transcriber) {
|
||||
self.postMessage({ type: 'status', status: 'loading' });
|
||||
transcriber = await pipeline('automatic-speech-recognition', model, {
|
||||
quantized: quantized,
|
||||
progress_callback: (data) => {
|
||||
self.postMessage({ type: 'progress', data });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
self.postMessage({ type: 'status', status: 'transcribing' });
|
||||
|
||||
const time_precision =
|
||||
transcriber.processor.feature_extractor.config.chunk_length /
|
||||
transcriber.model.config.max_source_positions;
|
||||
|
||||
let chunks_to_process = [
|
||||
{
|
||||
tokens: [],
|
||||
finalised: false,
|
||||
},
|
||||
];
|
||||
|
||||
function chunk_callback(chunk) {
|
||||
let last = chunks_to_process[chunks_to_process.length - 1];
|
||||
|
||||
Object.assign(last, chunk);
|
||||
last.finalised = true;
|
||||
|
||||
if (!chunk.is_last) {
|
||||
chunks_to_process.push({
|
||||
tokens: [],
|
||||
finalised: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function callback_function(item) {
|
||||
let last = chunks_to_process[chunks_to_process.length - 1];
|
||||
|
||||
last.tokens = [...item[0].output_token_ids];
|
||||
|
||||
let data = transcriber.tokenizer._decode_asr(chunks_to_process, {
|
||||
time_precision: time_precision,
|
||||
return_timestamps: true,
|
||||
force_full_sequences: false,
|
||||
});
|
||||
|
||||
self.postMessage({
|
||||
type: 'partial',
|
||||
output: {
|
||||
text: data[0],
|
||||
chunks: data[1].chunks
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const output = await transcriber(audio, {
|
||||
top_k: 0,
|
||||
do_sample: false,
|
||||
chunk_length_s: 30,
|
||||
stride_length_s: 5,
|
||||
return_timestamps: true,
|
||||
callback_function,
|
||||
chunk_callback,
|
||||
language,
|
||||
task: task === "translate" ? "translate" : undefined
|
||||
});
|
||||
|
||||
self.postMessage({ type: 'complete', output });
|
||||
|
||||
} catch (e) {
|
||||
self.postMessage({ type: 'error', error: e.toString() });
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
export class TranscriptionWorker {
|
||||
private worker: Worker;
|
||||
private onStatus: (status: string) => void;
|
||||
private onComplete: (output: any) => void;
|
||||
private onError: (error: any) => void;
|
||||
private onPartial: (output: any) => void;
|
||||
|
||||
constructor(
|
||||
onStatus: (status: string) => void,
|
||||
onComplete: (output: any) => void,
|
||||
onError: (error: any) => void,
|
||||
onPartial: (output: any) => void
|
||||
) {
|
||||
this.onStatus = onStatus;
|
||||
this.onComplete = onComplete;
|
||||
this.onError = onError;
|
||||
this.onPartial = onPartial;
|
||||
|
||||
const blob = new Blob([workerCode], { type: "text/javascript" });
|
||||
this.worker = new Worker(URL.createObjectURL(blob), { type: "module" });
|
||||
this.worker.onmessage = this.handleMessage.bind(this);
|
||||
}
|
||||
|
||||
private getMimeType(url: string): string {
|
||||
if (url.endsWith(".wasm")) return "application/wasm";
|
||||
if (url.endsWith(".json")) return "application/json";
|
||||
if (url.endsWith(".onnx")) return "application/octet-stream";
|
||||
return "application/octet-stream";
|
||||
}
|
||||
|
||||
private async handleMessage(event: MessageEvent) {
|
||||
const { type, id, url, status, output, error } = event.data;
|
||||
|
||||
switch (type) {
|
||||
case "fetch_request":
|
||||
try {
|
||||
const cachedData = await DataStore.get(`VoiceMessageTranscriber_${url}`);
|
||||
|
||||
if (cachedData && lodash.isArrayBuffer(cachedData)) {
|
||||
this.worker.postMessage({
|
||||
type: "fetch_response",
|
||||
id,
|
||||
response: cachedData,
|
||||
headers: {
|
||||
"Content-Length": cachedData.byteLength.toString(),
|
||||
"Content-Type": this.getMimeType(url)
|
||||
}
|
||||
});
|
||||
} else {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error("Failed to fetch " + url);
|
||||
|
||||
const buffer = await res.arrayBuffer();
|
||||
await DataStore.set(`VoiceMessageTranscriber_${url}`, buffer);
|
||||
|
||||
this.worker.postMessage({
|
||||
type: "fetch_response",
|
||||
id,
|
||||
response: buffer,
|
||||
headers: {
|
||||
"Content-Length": res.headers.get("Content-Length") || buffer.byteLength.toString(),
|
||||
"Content-Type": this.getMimeType(url)
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
this.worker.postMessage({
|
||||
type: "fetch_response",
|
||||
id,
|
||||
error: String(err)
|
||||
});
|
||||
}
|
||||
break;
|
||||
case "status":
|
||||
this.onStatus(status);
|
||||
break;
|
||||
case "complete":
|
||||
this.onComplete(output);
|
||||
break;
|
||||
case "partial":
|
||||
this.onPartial(output);
|
||||
break;
|
||||
case "error":
|
||||
this.onError(error);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
public run(audio: Float32Array, model: string, quantized: boolean = true, language?: string, task?: string) {
|
||||
this.worker.postMessage({
|
||||
type: "run",
|
||||
audio,
|
||||
model,
|
||||
quantized,
|
||||
language,
|
||||
task
|
||||
});
|
||||
}
|
||||
|
||||
public terminate() {
|
||||
this.worker.terminate();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue