Add sound support to vcNarratorCustom feature (#1083)
* feat(vcNarratorCustom): add sound support * use useForceUpdater instead of React.useReducer Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * removed mp4 and webm because they aren't supported --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
This commit is contained in:
Quadrinex
GitHub
parent
a614f92e8c
commit
feff7cda4d
3 changed files with 240 additions and 47 deletions
|
|
@ -31,6 +31,7 @@ import {
|
|||
SearchableSelect,
|
||||
SelectedChannelStore,
|
||||
SelectedGuildStore,
|
||||
showToast,
|
||||
useMemo,
|
||||
UserStore,
|
||||
VoiceStateStore,
|
||||
|
|
@ -60,6 +61,8 @@ import {
|
|||
const cl = classNameFactory("vc-narrator-");
|
||||
|
||||
const API_BASE = "https://tiktok-tts-aio.exampleuser.workers.dev";
|
||||
const AUDIO_EXTENSIONS = ["mp3", "wav", "ogg", "m4a", "aac", "flac", "wma"];
|
||||
const SOUND_PLACEHOLDER = "{{SOUND}}";
|
||||
|
||||
type LastApiCallStatus = {
|
||||
at: number;
|
||||
|
|
@ -166,6 +169,98 @@ function TroubleshootingSettings() {
|
|||
);
|
||||
}
|
||||
|
||||
function CustomSoundSettings({ soundKey, nameKey }: { soundKey: string; nameKey: string; }) {
|
||||
const fileInputRef = React.useRef<HTMLInputElement>(null);
|
||||
const [busy, setBusy] = React.useState(false);
|
||||
const forceUpdate = useForceUpdater();
|
||||
const customSound = settings.store[soundKey];
|
||||
const customSoundName = settings.store[nameKey] ?? "";
|
||||
const hasCustomSound = typeof customSound === "string" && customSound.startsWith("data:audio/");
|
||||
|
||||
const uploadSound = async (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const file = event.currentTarget.files?.[0];
|
||||
if (!file) return;
|
||||
|
||||
const extension = file.name.split(".").pop()?.toLowerCase();
|
||||
if (!extension || !AUDIO_EXTENSIONS.includes(extension)) {
|
||||
showToast("Please choose an audio file.");
|
||||
event.currentTarget.value = "";
|
||||
return;
|
||||
}
|
||||
|
||||
setBusy(true);
|
||||
try {
|
||||
const dataUri = await new Promise<string>((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => resolve(String(reader.result ?? ""));
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
|
||||
if (!dataUri.startsWith("data:audio/")) {
|
||||
showToast("Please choose an audio file.");
|
||||
return;
|
||||
}
|
||||
|
||||
settings.store[soundKey] = dataUri;
|
||||
settings.store[nameKey] = file.name;
|
||||
forceUpdate();
|
||||
showToast("Custom sound saved.");
|
||||
} catch {
|
||||
showToast("Could not load that audio file.");
|
||||
} finally {
|
||||
event.currentTarget.value = "";
|
||||
setBusy(false);
|
||||
}
|
||||
};
|
||||
|
||||
const previewSound = () => {
|
||||
if (!hasCustomSound) return;
|
||||
|
||||
setBusy(true);
|
||||
const audio = new Audio(customSound);
|
||||
audio.volume = settings.store.volume;
|
||||
audio.onended = () => setBusy(false);
|
||||
audio.onerror = () => setBusy(false);
|
||||
audio.play().catch(() => setBusy(false));
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={cl("custom-sound")}>
|
||||
<Forms.FormText className={cl("custom-sound-text")}>
|
||||
{hasCustomSound ? `Selected: ${customSoundName || "Custom audio"}` : "No custom sound selected."}
|
||||
</Forms.FormText>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".mp3,.wav,.ogg,.m4a,.aac,.flac,.webm,.wma,.mp4"
|
||||
className={cl("file-input")}
|
||||
onChange={uploadSound}
|
||||
/>
|
||||
<div className={cl("buttons")}>
|
||||
<Button variant="secondary" size="small" disabled={busy} onClick={() => fileInputRef.current?.click()}>
|
||||
Choose file
|
||||
</Button>
|
||||
<Button variant="secondary" size="small" disabled={busy || !hasCustomSound} onClick={previewSound}>
|
||||
Preview
|
||||
</Button>
|
||||
<Button
|
||||
variant="dangerPrimary"
|
||||
size="small"
|
||||
disabled={busy || !hasCustomSound}
|
||||
onClick={() => {
|
||||
settings.store[soundKey] = "";
|
||||
settings.store[nameKey] = "";
|
||||
forceUpdate();
|
||||
}}
|
||||
>
|
||||
Clear
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface VoiceState {
|
||||
userId: string;
|
||||
channelId?: string;
|
||||
|
|
@ -183,6 +278,7 @@ interface QueueItem {
|
|||
userId?: string;
|
||||
interruptKey?: string;
|
||||
useDefaultVoice?: boolean;
|
||||
sound?: string;
|
||||
}
|
||||
const mainQueue: QueueItem[] = [];
|
||||
const stateQueue: QueueItem[] = [];
|
||||
|
|
@ -265,6 +361,7 @@ async function processQueue() {
|
|||
userId: intro.userId,
|
||||
interruptKey: action.interruptKey,
|
||||
useDefaultVoice: false,
|
||||
sound: intro.sound ?? action.sound,
|
||||
};
|
||||
mainQueue.push(combined);
|
||||
}
|
||||
|
|
@ -275,7 +372,7 @@ async function processQueue() {
|
|||
onQueueChange?.();
|
||||
|
||||
try {
|
||||
await speak(item.text, item.userId, item.interruptKey, item.useDefaultVoice);
|
||||
await speak(item.text, item.userId, item.interruptKey, item.useDefaultVoice, item.sound);
|
||||
} catch (e) {
|
||||
console.error("TTS Error:", e);
|
||||
}
|
||||
|
|
@ -288,7 +385,7 @@ async function processQueue() {
|
|||
}, delay);
|
||||
}
|
||||
|
||||
function queueSpeak(text: string, userId?: string, interruptKey?: string, queue: "main" | "state" = "main", useDefaultVoice?: boolean) {
|
||||
function queueSpeak(text: string, userId?: string, interruptKey?: string, queue: "main" | "state" = "main", useDefaultVoice?: boolean, sound?: string) {
|
||||
if (text.trim().length === 0) return;
|
||||
|
||||
const targetQueue = queue === "state" ? stateQueue : mainQueue;
|
||||
|
|
@ -305,48 +402,45 @@ function queueSpeak(text: string, userId?: string, interruptKey?: string, queue:
|
|||
interruptPlayback(interruptKey);
|
||||
}
|
||||
|
||||
targetQueue.push({ text, userId, interruptKey, useDefaultVoice });
|
||||
targetQueue.push({ text, userId, interruptKey, useDefaultVoice, sound });
|
||||
onQueueChange?.();
|
||||
processQueue();
|
||||
}
|
||||
|
||||
async function speak(text: string, userId?: string, interruptKey?: string, useDefaultVoice?: boolean): Promise<void> {
|
||||
async function speak(text: string, userId?: string, interruptKey?: string, useDefaultVoice?: boolean, sound?: string): Promise<void> {
|
||||
return new Promise(resolve => {
|
||||
const onEnd = () => {
|
||||
if (currentStop === onEnd) {
|
||||
currentAudio = null;
|
||||
currentInterruptKey = undefined;
|
||||
currentStop = null;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
|
||||
const playAudio = (url: string) => {
|
||||
const playAudio = (url: string, playbackRate: number) => new Promise<boolean>(resolveAudio => {
|
||||
const audio = new Audio(url);
|
||||
let settled = false;
|
||||
|
||||
const finish = (completed: boolean) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (currentStop === stop) {
|
||||
currentAudio = null;
|
||||
currentInterruptKey = undefined;
|
||||
currentStop = null;
|
||||
}
|
||||
resolveAudio(completed);
|
||||
};
|
||||
|
||||
const stop = () => finish(false);
|
||||
|
||||
audio.volume = settings.store.volume;
|
||||
audio.playbackRate = settings.store.rate;
|
||||
audio.onended = onEnd;
|
||||
audio.onerror = onEnd;
|
||||
audio.playbackRate = playbackRate;
|
||||
audio.onended = () => finish(true);
|
||||
audio.onerror = stop;
|
||||
currentAudio = audio;
|
||||
currentInterruptKey = interruptKey;
|
||||
currentStop = onEnd;
|
||||
audio.play().catch(onEnd);
|
||||
};
|
||||
currentStop = stop;
|
||||
audio.play().catch(stop);
|
||||
});
|
||||
|
||||
void (async () => {
|
||||
const voice = useDefaultVoice
|
||||
? DEFAULT_VOICE
|
||||
: getVoiceForUser(userId, {
|
||||
userVoiceMap: settings.store.userVoiceMap,
|
||||
customVoice: settings.store.customVoice,
|
||||
defaultVoice: DEFAULT_VOICE,
|
||||
});
|
||||
|
||||
const cacheKey = `${voice}_${text}`;
|
||||
const getTtsUrl = async (speechText: string, voice: string) => {
|
||||
const cacheKey = `${voice}_${speechText}`;
|
||||
|
||||
if (ttsCache.has(cacheKey)) {
|
||||
playAudio(ttsCache.get(cacheKey)!);
|
||||
return;
|
||||
return ttsCache.get(cacheKey)!;
|
||||
}
|
||||
|
||||
try {
|
||||
|
|
@ -354,8 +448,7 @@ async function speak(text: string, userId?: string, interruptKey?: string, useDe
|
|||
if (cachedBlob) {
|
||||
const url = URL.createObjectURL(cachedBlob);
|
||||
ttsCache.set(cacheKey, url);
|
||||
playAudio(url);
|
||||
return;
|
||||
return url;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error("Error accessing IndexedDB:", err);
|
||||
|
|
@ -369,7 +462,7 @@ async function speak(text: string, userId?: string, interruptKey?: string, useDe
|
|||
headers: { "Content-Type": "application/json" },
|
||||
referrerPolicy: "no-referrer",
|
||||
body: JSON.stringify({
|
||||
text: text,
|
||||
text: speechText,
|
||||
voice: voice,
|
||||
base64: true,
|
||||
}),
|
||||
|
|
@ -378,8 +471,7 @@ async function speak(text: string, userId?: string, interruptKey?: string, useDe
|
|||
|
||||
if (!response.ok) {
|
||||
console.error(`TTS failed: ${response.status}`);
|
||||
resolve();
|
||||
return;
|
||||
return null;
|
||||
}
|
||||
|
||||
const audioData = atob((await response.text()).trim());
|
||||
|
|
@ -394,13 +486,50 @@ async function speak(text: string, userId?: string, interruptKey?: string, useDe
|
|||
ttsCache.set(cacheKey, url);
|
||||
setCachedVoiceInDB(cacheKey, blob).catch(console.error);
|
||||
|
||||
playAudio(url);
|
||||
return url;
|
||||
} catch (e) {
|
||||
recordApiError(e);
|
||||
console.error("TTS Network Error:", e);
|
||||
resolve();
|
||||
return null;
|
||||
}
|
||||
})().catch(onEnd);
|
||||
};
|
||||
|
||||
const playSpeech = async (speechText: string, voice: string) => {
|
||||
const trimmedText = speechText.trim();
|
||||
if (!trimmedText) return true;
|
||||
|
||||
const url = await getTtsUrl(trimmedText, voice);
|
||||
return !url || await playAudio(url, settings.store.rate);
|
||||
};
|
||||
|
||||
void (async () => {
|
||||
const voice = useDefaultVoice
|
||||
? DEFAULT_VOICE
|
||||
: getVoiceForUser(userId, {
|
||||
userVoiceMap: settings.store.userVoiceMap,
|
||||
customVoice: settings.store.customVoice,
|
||||
defaultVoice: DEFAULT_VOICE,
|
||||
});
|
||||
|
||||
const soundUrl = typeof sound === "string" && sound.startsWith("data:audio/") ? sound : "";
|
||||
const soundIndex = text.indexOf(SOUND_PLACEHOLDER);
|
||||
|
||||
if (!soundUrl || soundIndex === -1) {
|
||||
await playSpeech(text.replaceAll(SOUND_PLACEHOLDER, ""), voice);
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
const beforeSound = text.slice(0, soundIndex);
|
||||
const afterSound = text.slice(soundIndex + SOUND_PLACEHOLDER.length).replaceAll(SOUND_PLACEHOLDER, "");
|
||||
|
||||
if (await playSpeech(beforeSound, voice)) {
|
||||
const soundCompleted = await playAudio(soundUrl, 1);
|
||||
if (soundCompleted) await playSpeech(afterSound, voice);
|
||||
}
|
||||
|
||||
resolve();
|
||||
})().catch(() => resolve());
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -552,7 +681,12 @@ function playSample(tempSettings: any, type: string) {
|
|||
currentUser.globalName ?? currentUser.username,
|
||||
(myGuildId ? GuildMemberStore.getNick(myGuildId, currentUser.id) : null) ?? currentUser.username,
|
||||
settingsobj.latinOnly
|
||||
)
|
||||
),
|
||||
undefined,
|
||||
undefined,
|
||||
"main",
|
||||
undefined,
|
||||
settingsobj[type + "Sound"]
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -595,19 +729,52 @@ const settings = definePluginSettings({
|
|||
},
|
||||
joinMessage: {
|
||||
type: OptionType.STRING,
|
||||
description: "Placeholders: {{USER}}, {{DISPLAY_NAME}}, {{NICKNAME}}, {{CHANNEL}}",
|
||||
description: "Placeholders: {{USER}}, {{DISPLAY_NAME}}, {{NICKNAME}}, {{CHANNEL}}, {{SOUND}}.",
|
||||
default: "{{DISPLAY_NAME}} joined",
|
||||
},
|
||||
joinSound: {
|
||||
type: OptionType.COMPONENT,
|
||||
component: () => <CustomSoundSettings soundKey="joinSound" nameKey="joinSoundName" />,
|
||||
default: "",
|
||||
},
|
||||
joinSoundName: {
|
||||
type: OptionType.STRING,
|
||||
description: "Join sound file name.",
|
||||
default: "",
|
||||
hidden: true,
|
||||
},
|
||||
leaveMessage: {
|
||||
type: OptionType.STRING,
|
||||
description: "Leave Message",
|
||||
description: "Placeholders: {{USER}}, {{DISPLAY_NAME}}, {{NICKNAME}}, {{CHANNEL}}, {{SOUND}}.",
|
||||
default: "{{DISPLAY_NAME}} left",
|
||||
},
|
||||
leaveSound: {
|
||||
type: OptionType.COMPONENT,
|
||||
component: () => <CustomSoundSettings soundKey="leaveSound" nameKey="leaveSoundName" />,
|
||||
default: "",
|
||||
},
|
||||
leaveSoundName: {
|
||||
type: OptionType.STRING,
|
||||
description: "Leave sound file name.",
|
||||
default: "",
|
||||
hidden: true,
|
||||
},
|
||||
moveMessage: {
|
||||
type: OptionType.STRING,
|
||||
description: "Move Message",
|
||||
description: "Placeholders: {{USER}}, {{DISPLAY_NAME}}, {{NICKNAME}}, {{CHANNEL}}, {{SOUND}}.",
|
||||
default: "{{DISPLAY_NAME}} moved to {{CHANNEL}}",
|
||||
},
|
||||
moveSound: {
|
||||
type: OptionType.COMPONENT,
|
||||
component: () => <CustomSoundSettings soundKey="moveSound" nameKey="moveSoundName" />,
|
||||
default: "",
|
||||
},
|
||||
moveSoundName: {
|
||||
type: OptionType.STRING,
|
||||
description: "Move sound file name.",
|
||||
default: "",
|
||||
hidden: true,
|
||||
},
|
||||
announceOthersMute: {
|
||||
description: "Announce when other users mute/unmute in your current VC",
|
||||
type: OptionType.BOOLEAN,
|
||||
|
|
@ -879,7 +1046,7 @@ export default definePlugin({
|
|||
name: "VcNarratorCustom",
|
||||
description: "Announces when users join, leave, or move voice channels via narrator using TikTok TTS. Revamped and back from the dead.",
|
||||
tags: ["Accessibility", "Voice"],
|
||||
authors: [Devs.Ven, Devs.Nyako, EquicordDevs.Loukios, EquicordDevs.examplegit],
|
||||
authors: [Devs.Ven, Devs.Nyako, EquicordDevs.Loukios, EquicordDevs.examplegit, EquicordDevs.qdnx],
|
||||
settings,
|
||||
contextMenus: {
|
||||
"user-context": UserContextMenuPatch,
|
||||
|
|
@ -979,7 +1146,14 @@ export default definePlugin({
|
|||
const nickname = u && ((myGuildId ? GuildMemberStore.getNick(myGuildId, userId) : null) ?? displayName);
|
||||
const channel = ChannelStore.getChannel(id)?.name ?? "channel";
|
||||
|
||||
queueSpeak(formatText(template, u, channel, displayName, nickname, settings.store.latinOnly), userId);
|
||||
queueSpeak(
|
||||
formatText(template, u, channel, displayName, nickname, settings.store.latinOnly),
|
||||
userId,
|
||||
undefined,
|
||||
"main",
|
||||
undefined,
|
||||
settings.store[type + "Sound"]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -53,6 +53,21 @@
|
|||
color: var(--text-muted);
|
||||
}
|
||||
|
||||
.vc-narrator-custom-sound {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.vc-narrator-custom-sound-text {
|
||||
margin: 0;
|
||||
opacity: 0.85;
|
||||
}
|
||||
|
||||
.vc-narrator-file-input {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Voice select modal */
|
||||
.vc-narrator-modal-title {
|
||||
flex-grow: 1;
|
||||
|
|
@ -137,4 +152,4 @@
|
|||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
justify-content: center;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1041,6 +1041,10 @@ export const EquicordDevs = Object.freeze({
|
|||
name: "Loukios",
|
||||
id: 211461918127292416n
|
||||
},
|
||||
qdnx: {
|
||||
name: "qdnx",
|
||||
id: 1374803023506702508n
|
||||
},
|
||||
vappstar: {
|
||||
name: "vappstar",
|
||||
id: 747192967311261748n
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue