diff --git a/src/equicordplugins/triviaAI/index.tsx b/src/equicordplugins/triviaAI/index.tsx index f2932e93e..fb1d373f0 100644 --- a/src/equicordplugins/triviaAI/index.tsx +++ b/src/equicordplugins/triviaAI/index.tsx @@ -15,8 +15,7 @@ import { settings } from "./settings"; import { getPayload, getResponse, handleResponse } from "./utils"; const messageCtxPatch: NavContextMenuPatchCallback = (children, { message }: { message: Message; }) => { - const payload = getPayload(message); - if (!payload) return; + if (!message.content.trim() && !message.embeds.length && (!settings.store.supportImages || !message.attachments.some(att => att.content_type?.startsWith("image/")))) return; const group = findGroupChildrenByChildId("copy-text", children); if (!group) return; @@ -27,6 +26,9 @@ const messageCtxPatch: NavContextMenuPatchCallback = (children, { message }: { m label="Answer With AI" icon={RobotIcon} action={async () => { + const payload = await getPayload(message); + if (!payload) return; + const ans = await getResponse(payload); handleResponse(message, ans); }} @@ -47,8 +49,7 @@ export default definePlugin({ messagePopoverButton: { icon: RobotIcon, render(message: Message) { - const payload = getPayload(message); - if (!payload) return null; + if (!message.content.trim() && !message.embeds.length && (!settings.store.supportImages || !message.attachments.some(att => att.content_type?.startsWith("image/")))) return null; return { label: "Answer With AI", @@ -56,6 +57,9 @@ export default definePlugin({ message, channel: ChannelStore.getChannel(message.channel_id), onClick: async () => { + const payload = await getPayload(message); + if (!payload) return; + const ans = await getResponse(payload); handleResponse(message, ans); } diff --git a/src/equicordplugins/triviaAI/settings.ts b/src/equicordplugins/triviaAI/settings.ts index 1729a05aa..3a887d9d6 100644 --- a/src/equicordplugins/triviaAI/settings.ts +++ b/src/equicordplugins/triviaAI/settings.ts @@ -69,5 +69,10 @@ export const settings = definePluginSettings({ type: OptionType.BOOLEAN, description: "Pass images to the AI for context (if any). This is not supported by all models.", default: true + }, + sendImagesAsBase64: { + type: OptionType.BOOLEAN, + description: "Send images to the AI as base64.", + default: false } }); diff --git a/src/equicordplugins/triviaAI/utils.ts b/src/equicordplugins/triviaAI/utils.ts index b13386e30..cda8b3483 100644 --- a/src/equicordplugins/triviaAI/utils.ts +++ b/src/equicordplugins/triviaAI/utils.ts @@ -34,7 +34,7 @@ export type ApiMessage = { content: ContentPayload; }; -export function getPayload(message: Message): ApiMessage[] | null { +export async function getPayload(message: Message): Promise { const prevMessages = getPreviousMessages(message, settings.store.context); const allMessages = [...prevMessages, message]; @@ -79,7 +79,17 @@ export function getPayload(message: Message): ApiMessage[] | null { payload.push({ role, content }); } - return payload.length > 0 ? payload : null; + if (payload.length === 0) return null; + + if (!settings.store.sendImagesAsBase64) return payload; + + return Promise.all(payload.map(async msg => { + if (typeof msg.content === "string") return msg; + + const content = await Promise.all(msg.content.map(part => part.type === "image_url" ? toBase64Image(part) : part)); + + return { ...msg, content: content.filter(part => part !== null) }; + })); } export function getPreviousMessages(message: Message, count: number): Message[] { @@ -125,7 +135,7 @@ export function parseMessageContent(message: Message): ContentPayload | null { message.attachments .filter(att => att.content_type?.startsWith("image/")) - .forEach(att => imageUrls.add(att.url)); + .forEach(att => imageUrls.add(att.proxy_url ?? att.url)); message.embeds.forEach(embed => { const potentialUrls = [ @@ -162,6 +172,27 @@ export function parseMessageContent(message: Message): ContentPayload | null { return payload; } +async function toBase64Image(part: ImagePart): Promise { + try { + const req = await fetch(part.image_url.url); + if (!req.ok) return null; + + let binary = ""; + const bytes = new Uint8Array(await req.arrayBuffer()); + for (let i = 0; i < bytes.length; i += 0x8000) { + binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000)); + } + + return { + type: "image_url", + image_url: { url: `data:${req.headers.get("content-type") ?? "image/png"};base64,${btoa(binary)}` } + }; + } catch (e) { + logger.warn("failed to convert image to base64", e); + return null; + } +} + export async function handleResponse(message: Message, response: string): Promise { switch (settings.store.mode) { case "autoreply":