VoiceMessageTranscriber: Added Embed, model size estimates (#1302)
All checks were successful
Sync to Codeberg / codeberg (push) Has been skipped
All checks were successful
Sync to Codeberg / codeberg (push) Has been skipped
* feat(voiceMessageTranscriber): add embed and maintain horizontal options with model size indicators * Apply suggestions from code review Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * fix(voiceMessageTranscriber): scope parent selectors and resolve transcription race condition * refactor(voiceMessageTranscriber): adhere strictly to .rules types and formatting * fix(voiceMessageTranscriber): reliably wrap parent container for embed layout * feat(voiceMessageTranscriber): add tt to authors * some fixes --------- Co-authored-by: thororen <78185467+thororen1234@users.noreply.github.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
parent
9c543bad54
commit
328e2ce762
3 changed files with 573 additions and 99 deletions
|
|
@ -10,27 +10,88 @@ import { DataStore } from "@api/index";
|
|||
import { definePluginSettings } from "@api/Settings";
|
||||
import { BaseText } from "@components/BaseText";
|
||||
import { Button } from "@components/Button";
|
||||
import ErrorBoundary from "@components/ErrorBoundary";
|
||||
import { Flex } from "@components/Flex";
|
||||
import { FormSwitch } from "@components/FormSwitch";
|
||||
import { Heading } from "@components/Heading";
|
||||
import { ClockIcon, CopyIcon } from "@components/Icons";
|
||||
import { Span } from "@components/Span";
|
||||
import { copyToClipboard } from "@utils/clipboard";
|
||||
import { Devs } from "@utils/constants";
|
||||
import { Devs, EquicordDevs } from "@utils/constants";
|
||||
import { classes } from "@utils/misc";
|
||||
import definePlugin, { OptionType, PluginNative } from "@utils/types";
|
||||
import { RenderModalProps } from "@vencord/discord-types";
|
||||
import { findComponentByCodeLazy } from "@webpack";
|
||||
import { lodash, Modal, openModal, ScrollerAuto, SearchableSelect, useEffect, useRef, useState } from "@webpack/common";
|
||||
import { Modal, openModal, ScrollerAuto, SearchableSelect, Tooltip, useEffect, useRef, useState } from "@webpack/common";
|
||||
|
||||
import { cl, decodeAudio, LANGUAGES, TranscriptionWorker } from "./utils";
|
||||
import { cl, decodeAudio, LANGUAGES, TranscriptionResult, TranscriptionWorker } from "./utils";
|
||||
const Native = VencordNative.pluginHelpers.VoiceMessageTranscriber as PluginNative<typeof import("./native")>;
|
||||
|
||||
const ChannelListIcon = findComponentByCodeLazy("1-1-1ZM2 8a1");
|
||||
let ManaBaseRadioGroup;
|
||||
let ManaBaseRadioGroup: React.ComponentType<{ options: { name: string; value: string; }[]; value: string; onChange: (v: string) => void; }>;
|
||||
|
||||
function formatTimestamp(seconds: number) {
|
||||
const m = Math.floor(seconds / 60);
|
||||
const s = Math.floor(seconds % 60);
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function CloseIcon({ size = 14 }: { size?: number; }) {
|
||||
return (
|
||||
<svg width={size} height={size} viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2" strokeLinecap="round" strokeLinejoin="round">
|
||||
<line x1="18" y1="6" x2="6" y2="18" />
|
||||
<line x1="6" y1="6" x2="18" y2="18" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
function CheckmarkIcon({ size = 14 }: { size?: number; }) {
|
||||
return (
|
||||
<svg width={size} height={size} viewBox="0 0 24 24" fill="none" stroke="currentColor" strokeWidth="2.5" strokeLinecap="round" strokeLinejoin="round">
|
||||
<polyline points="20 6 9 17 4 12" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
const MODEL_SIZES: Record<string, { quantized: string; full: string; }> = {
|
||||
"Xenova/whisper-tiny": { quantized: "~40 MB", full: "~150 MB" },
|
||||
"Xenova/whisper-base": { quantized: "~77 MB", full: "~290 MB" },
|
||||
"Xenova/whisper-small": { quantized: "~250 MB", full: "~960 MB" },
|
||||
"Xenova/whisper-medium": { quantized: "~765 MB", full: "~3.1 GB" },
|
||||
};
|
||||
|
||||
function renderModelOption(option?: { label: string; value: string; }) {
|
||||
if (!option) return null;
|
||||
const isQuantized = settings.use(["quantized"])?.quantized ?? true;
|
||||
const size = MODEL_SIZES[option.value]?.[isQuantized ? "quantized" : "full"];
|
||||
|
||||
return (
|
||||
<div className={cl("model-option")}>
|
||||
<span>{option.label}</span>
|
||||
{size && (
|
||||
<span className={cl("model-size")}>
|
||||
{size}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const settings = definePluginSettings({
|
||||
embed: {
|
||||
type: OptionType.BOOLEAN,
|
||||
description: "Display transcription directly in the voice message attachment instead of a modal.",
|
||||
default: false,
|
||||
restartNeeded: false
|
||||
},
|
||||
maintainHorizontal: {
|
||||
type: OptionType.BOOLEAN,
|
||||
description: "Maintain horizontal size for the embedded transcription box and expand vertically.",
|
||||
default: false,
|
||||
restartNeeded: false
|
||||
},
|
||||
selectedModel: {
|
||||
type: OptionType.SELECT,
|
||||
description: "Model size",
|
||||
description: "Model size.",
|
||||
options: [
|
||||
{
|
||||
label: "Tiny (Fastest, lowest accuracy)",
|
||||
|
|
@ -50,38 +111,45 @@ const settings = definePluginSettings({
|
|||
value: "Xenova/whisper-medium"
|
||||
}
|
||||
],
|
||||
componentProps: {
|
||||
renderOptionLabel: (option: { label: string; value: string; }) => renderModelOption(option),
|
||||
renderOptionValue: (options: { label: string; value: string; }[]) => renderModelOption(options?.[0]),
|
||||
},
|
||||
restartNeeded: false
|
||||
},
|
||||
quantized: {
|
||||
type: OptionType.BOOLEAN,
|
||||
description: "Use quantized model (smaller size, slightly lower accuracy)",
|
||||
description: "Use quantized models (smaller size, slight quality loss).",
|
||||
default: true,
|
||||
restartNeeded: false
|
||||
},
|
||||
delete: {
|
||||
deleteModalFiles: {
|
||||
type: OptionType.COMPONENT,
|
||||
description: "Delete cached files from storage.",
|
||||
component: () => {
|
||||
const [size, setSize] = useState(0);
|
||||
const [deleteKeys, setDeleteKeys] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
let unmounted = false;
|
||||
DataStore.entries().then(entries => {
|
||||
let size = 0;
|
||||
const keys = [] as string[];
|
||||
|
||||
entries.forEach(([key, val]) => {
|
||||
if (typeof key === "string" && key.startsWith("VoiceMessageTranscriber") && lodash.isArrayBuffer(val)) {
|
||||
if (unmounted) return;
|
||||
let totalSize = 0;
|
||||
const keys: string[] = [];
|
||||
for (const [key, value] of entries) {
|
||||
if (typeof key === "string" && (key.startsWith("whisper-") || key.startsWith("onnx-"))) {
|
||||
totalSize += (value as string).length;
|
||||
keys.push(key);
|
||||
size += val.byteLength ?? 0;
|
||||
}
|
||||
});
|
||||
|
||||
setSize(size);
|
||||
}
|
||||
setSize(totalSize);
|
||||
setDeleteKeys(keys);
|
||||
});
|
||||
return () => { unmounted = true; };
|
||||
}, []);
|
||||
|
||||
return <Button
|
||||
disabled={size === 0}
|
||||
variant="dangerPrimary"
|
||||
onClick={() => {
|
||||
DataStore.delMany(deleteKeys).then(() => { setSize(0); setDeleteKeys([]); });
|
||||
|
|
@ -166,7 +234,7 @@ function LanguageSelectionModal(props: { modalProps: RenderModalProps, src: stri
|
|||
function TranscriptionModal(props: { modalProps: RenderModalProps, src: string, options: { language: string, task: string; }; }) {
|
||||
const { modalProps, src, options } = props;
|
||||
const [status, setStatus] = useState<string>("initializing");
|
||||
const [result, setResult] = useState<{ text: string, chunks: { timestamp: [number, number], text: string; }[]; } | null>(null);
|
||||
const [result, setResult] = useState<TranscriptionResult | null>(null);
|
||||
const [showTimestamps, setShowTimestamps] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
|
@ -183,8 +251,8 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
|
||||
let blob: Blob;
|
||||
if (IS_DISCORD_DESKTOP || IS_EQUIBOP) {
|
||||
const arrayBuffer = await Native.fetchAudio(src);
|
||||
blob = new Blob([arrayBuffer as any]);
|
||||
const data = await Native.fetchAudio(src);
|
||||
blob = new Blob([new Uint8Array(data)]);
|
||||
} else {
|
||||
const res = await fetch(src);
|
||||
if (!res.ok) throw new Error("Failed to download audio");
|
||||
|
|
@ -196,6 +264,7 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
const audioData = await decodeAudio(blob);
|
||||
|
||||
if (!active) return;
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = new TranscriptionWorker(
|
||||
s => {
|
||||
if (active) setStatus(s);
|
||||
|
|
@ -207,7 +276,10 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
}
|
||||
},
|
||||
err => {
|
||||
if (active) setError(String(err));
|
||||
if (active) {
|
||||
setError(String(err));
|
||||
setStatus("error");
|
||||
}
|
||||
},
|
||||
partial => {
|
||||
if (active) setResult(partial);
|
||||
|
|
@ -223,7 +295,10 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
options.task
|
||||
);
|
||||
} catch (err) {
|
||||
if (active) setError(String(err));
|
||||
if (active) {
|
||||
setError(String(err));
|
||||
setStatus("error");
|
||||
}
|
||||
}
|
||||
})();
|
||||
|
||||
|
|
@ -231,20 +306,16 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
active = false;
|
||||
workerRef.current?.terminate();
|
||||
};
|
||||
}, [src, retryCount]);
|
||||
}, [retryCount]);
|
||||
|
||||
const retry = () => {
|
||||
setError(null);
|
||||
setStatus("initializing");
|
||||
setResult(null);
|
||||
setCopied(false);
|
||||
setRetryCount(prev => prev + 1);
|
||||
};
|
||||
|
||||
const formatTimestamp = (seconds: number) => {
|
||||
const m = Math.floor(seconds / 60);
|
||||
const s = Math.floor(seconds % 60);
|
||||
return `${m.toString().padStart(2, "0")}:${s.toString().padStart(2, "0")}`;
|
||||
const handleCopy = () => {
|
||||
if (!result) return;
|
||||
const text = showTimestamps
|
||||
? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n")
|
||||
: result.text;
|
||||
copyToClipboard(text);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
};
|
||||
|
||||
const displayText = result ? (
|
||||
|
|
@ -253,61 +324,46 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
: result.text
|
||||
) : "";
|
||||
|
||||
const actions: any[] = [];
|
||||
if (error) {
|
||||
actions.push({
|
||||
text: "Retry",
|
||||
variant: "primary",
|
||||
onClick: retry
|
||||
});
|
||||
} else if (status === "complete" || (status === "transcribing" && result)) {
|
||||
actions.push({
|
||||
text: copied ? "Copied!" : "Copy to Clipboard",
|
||||
variant: "primary",
|
||||
disabled: status === "transcribing" || copied,
|
||||
onClick: () => {
|
||||
copyToClipboard(displayText);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return (
|
||||
<Modal
|
||||
{...modalProps}
|
||||
size="lg"
|
||||
title="Transcription"
|
||||
actions={actions}
|
||||
size="md"
|
||||
title={options.task === "translate" ? "Voice Message Translation" : "Voice Message Transcription"}
|
||||
actions={[
|
||||
...(error ? [{
|
||||
text: "Retry",
|
||||
variant: "primary" as const,
|
||||
onClick: () => setRetryCount(c => c + 1)
|
||||
}] : []),
|
||||
...(result ? [
|
||||
{
|
||||
text: showTimestamps ? "Hide Timestamps" : "Show Timestamps",
|
||||
variant: "secondary" as const,
|
||||
onClick: () => setShowTimestamps(!showTimestamps)
|
||||
},
|
||||
{
|
||||
text: copied ? "Copied!" : "Copy Text",
|
||||
variant: "secondary" as const,
|
||||
onClick: handleCopy
|
||||
}
|
||||
] : [])
|
||||
]}
|
||||
>
|
||||
<div className={cl("content")}>
|
||||
{error ? (
|
||||
<Flex flexDirection="column" gap={16} style={{ padding: "20px 0" }}>
|
||||
<Heading tag="h3" style={{ color: "var(--red-360)" }}>Error</Heading>
|
||||
<Span style={{ whiteSpace: "pre-wrap" }}>{error}</Span>
|
||||
</Flex>
|
||||
) : (status === "complete" || (status === "transcribing" && result)) ? (
|
||||
<Flex flexDirection="column" gap={16} style={{ paddingBottom: "20px" }}>
|
||||
{status === "transcribing" && (
|
||||
<Span size="sm" color="text-muted">Transcribing in progress...</Span>
|
||||
)}
|
||||
<ScrollerAuto className={cl("result")}>
|
||||
<Span>{displayText}</Span>
|
||||
</ScrollerAuto>
|
||||
<Flex flexDirection="row" gap={12} alignItems="center">
|
||||
<div style={{ flexGrow: 1 }}>
|
||||
<FormSwitch
|
||||
title="Show Timestamps"
|
||||
value={showTimestamps}
|
||||
onChange={setShowTimestamps}
|
||||
/>
|
||||
</div>
|
||||
</Flex>
|
||||
<Flex flexDirection="column" alignItems="center" gap={12} style={{ padding: "32px 16px" }}>
|
||||
<Heading tag="h3" style={{ color: "var(--red-360)" }}>Transcription Failed</Heading>
|
||||
<BaseText size="sm" color="text-muted" style={{ textAlign: "center" }}>
|
||||
{error}
|
||||
</BaseText>
|
||||
</Flex>
|
||||
) : displayText ? (
|
||||
<div className={cl("result")}>
|
||||
<BaseText size="md">{displayText}</BaseText>
|
||||
</div>
|
||||
) : (
|
||||
<Flex flexDirection="column" gap={16} style={{ padding: "20px 0", alignItems: "center" }}>
|
||||
<Flex flexDirection="column" alignItems="center" justifyContent="center" gap={16} style={{ height: "200px" }}>
|
||||
<Heading tag="h3">
|
||||
{status === "initializing" && "Initializing..."}
|
||||
{status === "downloading_audio" && "Downloading Audio..."}
|
||||
{status === "processing_audio" && "Processing Audio..."}
|
||||
{status === "loading" && "Loading Model..."}
|
||||
|
|
@ -320,9 +376,250 @@ function TranscriptionModal(props: { modalProps: RenderModalProps, src: string,
|
|||
);
|
||||
}
|
||||
|
||||
function VoiceMessageTranscriber({ src }: { src: string; }) {
|
||||
const { embed, maintainHorizontal, quantized, selectedModel } = settings.use(["embed", "maintainHorizontal", "quantized", "selectedModel"]);
|
||||
const [isOpen, setIsOpen] = useState(false);
|
||||
const [status, setStatus] = useState<string>("idle");
|
||||
const [result, setResult] = useState<TranscriptionResult | null>(null);
|
||||
const [showTimestamps, setShowTimestamps] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [copied, setCopied] = useState(false);
|
||||
|
||||
const workerRef = useRef<TranscriptionWorker | null>(null);
|
||||
const activeRunId = useRef(0);
|
||||
const buttonRef = useRef<HTMLButtonElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const parent = buttonRef.current?.parentElement;
|
||||
if (!parent) return;
|
||||
parent.classList.add("vc-transcription-container");
|
||||
return () => {
|
||||
parent.classList.remove("vc-transcription-container", "vc-transcription-has-embed", "vc-transcription-maintain-horizontal");
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const parent = buttonRef.current?.parentElement;
|
||||
if (!parent) return;
|
||||
parent.classList.toggle("vc-transcription-has-embed", Boolean(embed && isOpen));
|
||||
parent.classList.toggle("vc-transcription-maintain-horizontal", Boolean(embed && isOpen && maintainHorizontal));
|
||||
}, [embed, isOpen, maintainHorizontal]);
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
activeRunId.current++;
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
};
|
||||
}, []);
|
||||
|
||||
const startTranscription = async () => {
|
||||
const runId = ++activeRunId.current;
|
||||
workerRef.current?.terminate();
|
||||
workerRef.current = null;
|
||||
|
||||
setIsOpen(true);
|
||||
setError(null);
|
||||
setStatus("downloading_audio");
|
||||
|
||||
try {
|
||||
let blob: Blob;
|
||||
if (IS_DISCORD_DESKTOP || IS_EQUIBOP) {
|
||||
const data = await Native.fetchAudio(src);
|
||||
blob = new Blob([new Uint8Array(data)]);
|
||||
} else {
|
||||
const res = await fetch(src);
|
||||
if (!res.ok) throw new Error("Failed to download audio");
|
||||
blob = await res.blob();
|
||||
}
|
||||
|
||||
if (runId !== activeRunId.current) return;
|
||||
|
||||
setStatus("processing_audio");
|
||||
const audioData = await decodeAudio(blob);
|
||||
|
||||
if (runId !== activeRunId.current) return;
|
||||
|
||||
workerRef.current = new TranscriptionWorker(
|
||||
s => {
|
||||
if (runId === activeRunId.current) setStatus(s);
|
||||
},
|
||||
out => {
|
||||
if (runId === activeRunId.current) {
|
||||
setResult(out);
|
||||
setStatus("complete");
|
||||
}
|
||||
},
|
||||
err => {
|
||||
if (runId === activeRunId.current) {
|
||||
setError(String(err));
|
||||
setStatus("error");
|
||||
}
|
||||
},
|
||||
partial => {
|
||||
if (runId === activeRunId.current) setResult(partial);
|
||||
}
|
||||
);
|
||||
|
||||
workerRef.current.run(
|
||||
audioData,
|
||||
selectedModel,
|
||||
quantized,
|
||||
undefined,
|
||||
"transcribe"
|
||||
);
|
||||
} catch (err) {
|
||||
if (runId === activeRunId.current) {
|
||||
setError(String(err));
|
||||
setStatus("error");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const handleClick = (e: React.MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
if (embed) {
|
||||
if (!isOpen) {
|
||||
if (status === "idle" || status === "error") {
|
||||
startTranscription();
|
||||
} else {
|
||||
setIsOpen(true);
|
||||
}
|
||||
} else {
|
||||
setIsOpen(false);
|
||||
}
|
||||
} else {
|
||||
openModal(modalProps => <LanguageSelectionModal modalProps={modalProps} src={src} />);
|
||||
}
|
||||
};
|
||||
|
||||
const displayText = result ? (
|
||||
showTimestamps
|
||||
? result.chunks.map(c => `[${formatTimestamp(c.timestamp[0])} - ${formatTimestamp(c.timestamp[1])}] ${c.text}`).join("\n")
|
||||
: result.text
|
||||
) : "";
|
||||
|
||||
const handleCopy = (e: React.MouseEvent) => {
|
||||
e.stopPropagation();
|
||||
if (!displayText) return;
|
||||
copyToClipboard(displayText);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
};
|
||||
|
||||
const isWorking = status === "downloading_audio" || status === "processing_audio" || status === "loading" || status === "transcribing";
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
ref={buttonRef}
|
||||
className={cl("button")}
|
||||
style={{ backgroundColor: "transparent" }}
|
||||
onClick={handleClick}
|
||||
title="Transcribe Voice Message"
|
||||
>
|
||||
<ChannelListIcon colorClass={cl("icon")} />
|
||||
</button>
|
||||
{embed && isOpen && (
|
||||
<div
|
||||
className={classes(
|
||||
cl("embed"),
|
||||
maintainHorizontal && cl("embed-maintain-horizontal")
|
||||
)}
|
||||
onClick={e => e.stopPropagation()}
|
||||
>
|
||||
<div className={cl("embed-header")}>
|
||||
<div className={cl("embed-title")}>
|
||||
<span className={classes(cl("status-dot"), isWorking && cl("status-dot-active"), status === "error" && cl("status-dot-error"))} />
|
||||
<span>
|
||||
{status === "downloading_audio" && "Downloading Audio..."}
|
||||
{status === "processing_audio" && "Processing Audio..."}
|
||||
{status === "loading" && "Loading Model..."}
|
||||
{status === "transcribing" && "Transcribing..."}
|
||||
{status === "complete" && "Transcription"}
|
||||
{status === "error" && "Transcription Error"}
|
||||
</span>
|
||||
</div>
|
||||
<div className={cl("embed-actions")}>
|
||||
{result && (
|
||||
<>
|
||||
<Tooltip text={showTimestamps ? "Hide Timestamps" : "Show Timestamps"}>
|
||||
{props => (
|
||||
<button
|
||||
{...props}
|
||||
className={classes(cl("action-btn"), showTimestamps && cl("action-btn-active"))}
|
||||
onClick={e => {
|
||||
e.stopPropagation();
|
||||
setShowTimestamps(!showTimestamps);
|
||||
}}
|
||||
>
|
||||
<ClockIcon width={14} height={14} />
|
||||
</button>
|
||||
)}
|
||||
</Tooltip>
|
||||
<Tooltip text={copied ? "Copied!" : "Copy Text"}>
|
||||
{props => (
|
||||
<button
|
||||
{...props}
|
||||
className={cl("action-btn")}
|
||||
onClick={handleCopy}
|
||||
>
|
||||
{copied ? <CheckmarkIcon size={14} /> : <CopyIcon width={14} height={14} />}
|
||||
</button>
|
||||
)}
|
||||
</Tooltip>
|
||||
</>
|
||||
)}
|
||||
<Tooltip text="Close">
|
||||
{props => (
|
||||
<button
|
||||
{...props}
|
||||
className={cl("action-btn")}
|
||||
onClick={e => {
|
||||
e.stopPropagation();
|
||||
setIsOpen(false);
|
||||
}}
|
||||
>
|
||||
<CloseIcon size={14} />
|
||||
</button>
|
||||
)}
|
||||
</Tooltip>
|
||||
</div>
|
||||
</div>
|
||||
<div className={cl("embed-body")}>
|
||||
{error ? (
|
||||
<Flex flexDirection="column" gap={8}>
|
||||
<Span size="xs" color="text-danger">{error}</Span>
|
||||
<Button
|
||||
size="small"
|
||||
variant="primary"
|
||||
onClick={startTranscription}
|
||||
style={{ alignSelf: "flex-start" }}
|
||||
>
|
||||
Retry
|
||||
</Button>
|
||||
</Flex>
|
||||
) : displayText ? (
|
||||
<ScrollerAuto className={cl("embed-text")}>
|
||||
<Span size="sm">{displayText}</Span>
|
||||
</ScrollerAuto>
|
||||
) : (
|
||||
<Span size="xs" color="text-muted">
|
||||
{isWorking ? "Transcribing in progress..." : "Initializing..."}
|
||||
</Span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
const VoiceMessageTranscriberWrapped = ErrorBoundary.wrap(VoiceMessageTranscriber, { noop: true });
|
||||
|
||||
export default definePlugin({
|
||||
name: "VoiceMessageTranscriber",
|
||||
authors: [Devs.TheSun],
|
||||
authors: [Devs.TheSun, EquicordDevs.tt],
|
||||
description: "On-device transcriptions for voice messages powered by Whisper v3",
|
||||
tags: ["Chat", "Media", "Utility", "Voice"],
|
||||
patches: [
|
||||
|
|
@ -341,20 +638,12 @@ export default definePlugin({
|
|||
}
|
||||
},
|
||||
],
|
||||
set ManaBaseRadioGroup(value: any) {
|
||||
set ManaBaseRadioGroup(value: typeof ManaBaseRadioGroup) {
|
||||
ManaBaseRadioGroup = value;
|
||||
},
|
||||
settings,
|
||||
|
||||
button(src: string) {
|
||||
return <button
|
||||
className={cl("button")}
|
||||
style={{ backgroundColor: "transparent" }}
|
||||
onClick={e => {
|
||||
e.stopPropagation();
|
||||
openModal(modalProps => <LanguageSelectionModal modalProps={modalProps} src={src} />);
|
||||
}}>
|
||||
<ChannelListIcon colorClass={cl("icon")} />
|
||||
</button>;
|
||||
return <VoiceMessageTranscriberWrapped src={src} />;
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -24,4 +24,175 @@
|
|||
user-select: text;
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.vc-transcription-model-option {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.vc-transcription-model-size {
|
||||
font-size: 12px;
|
||||
color: var(--text-muted);
|
||||
opacity: 0.75;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.vc-transcription-container:has(> .vc-transcription-embed),
|
||||
.vc-transcription-container.vc-transcription-has-embed {
|
||||
flex-wrap: wrap !important;
|
||||
}
|
||||
|
||||
.vc-transcription-container:has(> .vc-transcription-embed-maintain-horizontal),
|
||||
.vc-transcription-container.vc-transcription-maintain-horizontal {
|
||||
flex-wrap: wrap !important;
|
||||
width: fit-content !important;
|
||||
max-width: 100% !important;
|
||||
}
|
||||
|
||||
.vc-transcription-embed {
|
||||
order: 9999 !important;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
width: 100%;
|
||||
flex-basis: 100%;
|
||||
min-width: 0;
|
||||
margin-top: 8px;
|
||||
border-radius: 8px;
|
||||
background: rgb(0 0 0 / 25%);
|
||||
border: 1px solid var(--border-subtle, rgb(255 255 255 / 8%));
|
||||
backdrop-filter: blur(4px);
|
||||
box-sizing: border-box;
|
||||
overflow: hidden;
|
||||
user-select: text;
|
||||
}
|
||||
|
||||
.vc-transcription-embed-header {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 6px 10px;
|
||||
background: rgb(0 0 0 / 15%);
|
||||
border-bottom: 1px solid var(--border-subtle, rgb(255 255 255 / 6%));
|
||||
}
|
||||
|
||||
.vc-transcription-embed-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.vc-transcription-status-dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
border-radius: 50%;
|
||||
background-color: var(--text-muted);
|
||||
transition: background-color 0.2s ease;
|
||||
}
|
||||
|
||||
.vc-transcription-status-dot-active {
|
||||
background-color: var(--main-color, var(--brand-experiment, var(--brand-500, #5865f2)));
|
||||
animation: vc-transcription-pulse 1.5s infinite;
|
||||
}
|
||||
|
||||
.vc-transcription-status-dot-error {
|
||||
background-color: var(--text-feedback-critical, #ed4245);
|
||||
}
|
||||
|
||||
@keyframes vc-transcription-pulse {
|
||||
0%, 100% {
|
||||
opacity: 1;
|
||||
transform: scale(1);
|
||||
}
|
||||
|
||||
50% {
|
||||
opacity: 0.35;
|
||||
transform: scale(0.85);
|
||||
}
|
||||
}
|
||||
|
||||
.vc-transcription-embed-actions {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.vc-transcription-action-btn {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 22px;
|
||||
height: 22px;
|
||||
padding: 0;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
background: transparent;
|
||||
color: var(--interactive-icon-default, var(--text-muted));
|
||||
cursor: pointer;
|
||||
transition: color 0.15s ease, background-color 0.15s ease;
|
||||
}
|
||||
|
||||
.vc-transcription-action-btn:hover {
|
||||
color: var(--interactive-icon-hover, var(--text-normal));
|
||||
background-color: var(--background-modifier-hover, rgb(255 255 255 / 8%));
|
||||
}
|
||||
|
||||
.vc-transcription-action-btn-active {
|
||||
color: var(--main-color, var(--brand-experiment, var(--brand-500, #5865f2)));
|
||||
background-color: var(--background-modifier-selected, rgb(255 255 255 / 12%));
|
||||
}
|
||||
|
||||
.vc-transcription-embed-body {
|
||||
padding: 8px 12px;
|
||||
max-height: 180px;
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.vc-transcription-embed-text {
|
||||
white-space: pre-wrap;
|
||||
user-select: text;
|
||||
}
|
||||
|
||||
.vc-transcription-embed-maintain-horizontal {
|
||||
order: 9999 !important;
|
||||
width: 0 !important;
|
||||
min-width: 100% !important;
|
||||
max-width: 100% !important;
|
||||
flex-basis: 100% !important;
|
||||
flex-grow: 1 !important;
|
||||
box-sizing: border-box !important;
|
||||
margin-left: 0 !important;
|
||||
margin-right: 0 !important;
|
||||
}
|
||||
|
||||
.vc-transcription-embed-maintain-horizontal .vc-transcription-embed-header {
|
||||
width: 100% !important;
|
||||
min-width: 0 !important;
|
||||
max-width: 100% !important;
|
||||
box-sizing: border-box !important;
|
||||
}
|
||||
|
||||
.vc-transcription-embed-maintain-horizontal .vc-transcription-embed-body {
|
||||
width: 100% !important;
|
||||
min-width: 0 !important;
|
||||
max-width: 100% !important;
|
||||
max-height: 280px;
|
||||
box-sizing: border-box !important;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
|
||||
.vc-transcription-embed-maintain-horizontal .vc-transcription-embed-text {
|
||||
width: 100% !important;
|
||||
min-width: 0 !important;
|
||||
max-width: 100% !important;
|
||||
box-sizing: border-box !important;
|
||||
overflow-wrap: anywhere;
|
||||
display: block;
|
||||
}
|
||||
|
|
@ -112,7 +112,11 @@ export const LANGUAGES = {
|
|||
|
||||
export const cl = classNameFactory("vc-transcription-");
|
||||
|
||||
const getAudioContext = () => new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 });
|
||||
const AudioContextClass = window.AudioContext || (window as Window & { webkitAudioContext?: typeof AudioContext }).webkitAudioContext;
|
||||
const getAudioContext = () => {
|
||||
if (!AudioContextClass) throw new Error("AudioContext is not supported in this environment");
|
||||
return new AudioContextClass({ sampleRate: 16000 });
|
||||
};
|
||||
export async function decodeAudio(blob: Blob): Promise<Float32Array> {
|
||||
const arrayBuffer = await blob.arrayBuffer();
|
||||
const audioContext = getAudioContext();
|
||||
|
|
@ -258,18 +262,28 @@ async function runTranscription({ audio, model, quantized, language, task }) {
|
|||
}
|
||||
`;
|
||||
|
||||
export interface TranscriptionChunk {
|
||||
timestamp: [number, number];
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface TranscriptionResult {
|
||||
text: string;
|
||||
chunks: TranscriptionChunk[];
|
||||
}
|
||||
|
||||
export class TranscriptionWorker {
|
||||
private worker: Worker;
|
||||
private onStatus: (status: string) => void;
|
||||
private onComplete: (output: any) => void;
|
||||
private onError: (error: any) => void;
|
||||
private onPartial: (output: any) => void;
|
||||
private onComplete: (output: TranscriptionResult) => void;
|
||||
private onError: (error: unknown) => void;
|
||||
private onPartial: (output: TranscriptionResult) => void;
|
||||
|
||||
constructor(
|
||||
onStatus: (status: string) => void,
|
||||
onComplete: (output: any) => void,
|
||||
onError: (error: any) => void,
|
||||
onPartial: (output: any) => void
|
||||
onComplete: (output: TranscriptionResult) => void,
|
||||
onError: (error: unknown) => void,
|
||||
onPartial: (output: TranscriptionResult) => void
|
||||
) {
|
||||
this.onStatus = onStatus;
|
||||
this.onComplete = onComplete;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue