feat(triviaAI): sending images as base64 (#1229)

* feat(triviaAI): sending images as base64

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

src/equicordplugins/triviaAI/index.tsx

@@ -15,8 +15,7 @@ import { settings } from "./settings";
 import { getPayload, getResponse, handleResponse } from "./utils";
 
 const messageCtxPatch: NavContextMenuPatchCallback = (children, { message }: { message: Message; }) => {
-    const payload = getPayload(message);
-    if (!payload) return;
+    if (!message.content.trim() && !message.embeds.length && (!settings.store.supportImages || !message.attachments.some(att => att.content_type?.startsWith("image/")))) return;
 
     const group = findGroupChildrenByChildId("copy-text", children);
     if (!group) return;
@@ -27,6 +26,9 @@ const messageCtxPatch: NavContextMenuPatchCallback = (children, { message }: { m
             label="Answer With AI"
             icon={RobotIcon}
             action={async () => {
+                const payload = await getPayload(message);
+                if (!payload) return;
+
                 const ans = await getResponse(payload);
                 handleResponse(message, ans);
             }}
@@ -47,8 +49,7 @@ export default definePlugin({
     messagePopoverButton: {
         icon: RobotIcon,
         render(message: Message) {
-            const payload = getPayload(message);
-            if (!payload) return null;
+            if (!message.content.trim() && !message.embeds.length && (!settings.store.supportImages || !message.attachments.some(att => att.content_type?.startsWith("image/")))) return null;
 
             return {
                 label: "Answer With AI",
@@ -56,6 +57,9 @@ export default definePlugin({
                 message,
                 channel: ChannelStore.getChannel(message.channel_id),
                 onClick: async () => {
+                    const payload = await getPayload(message);
+                    if (!payload) return;
+
                     const ans = await getResponse(payload);
                     handleResponse(message, ans);
                 }

src/equicordplugins/triviaAI/settings.ts

@@ -69,5 +69,10 @@ export const settings = definePluginSettings({
         type: OptionType.BOOLEAN,
         description: "Pass images to the AI for context (if any). This is not supported by all models.",
         default: true
+    },
+    sendImagesAsBase64: {
+        type: OptionType.BOOLEAN,
+        description: "Send images to the AI as base64.",
+        default: false
     }
 });

src/equicordplugins/triviaAI/utils.ts

@@ -34,7 +34,7 @@ export type ApiMessage = {
     content: ContentPayload;
 };
 
-export function getPayload(message: Message): ApiMessage[] | null {
+export async function getPayload(message: Message): Promise<ApiMessage[] | null> {
     const prevMessages = getPreviousMessages(message, settings.store.context);
     const allMessages = [...prevMessages, message];
 
@@ -79,7 +79,17 @@ export function getPayload(message: Message): ApiMessage[] | null {
         payload.push({ role, content });
     }
 
-    return payload.length > 0 ? payload : null;
+    if (payload.length === 0) return null;
+
+    if (!settings.store.sendImagesAsBase64) return payload;
+
+    return Promise.all(payload.map(async msg => {
+        if (typeof msg.content === "string") return msg;
+
+        const content = await Promise.all(msg.content.map(part => part.type === "image_url" ? toBase64Image(part) : part));
+
+        return { ...msg, content: content.filter(part => part !== null) };
+    }));
 }
 
 export function getPreviousMessages(message: Message, count: number): Message[] {
@@ -125,7 +135,7 @@ export function parseMessageContent(message: Message): ContentPayload | null {
 
     message.attachments
         .filter(att => att.content_type?.startsWith("image/"))
-        .forEach(att => imageUrls.add(att.url));
+        .forEach(att => imageUrls.add(att.proxy_url ?? att.url));
 
     message.embeds.forEach(embed => {
         const potentialUrls = [
@@ -162,6 +172,27 @@ export function parseMessageContent(message: Message): ContentPayload | null {
     return payload;
 }
 
+async function toBase64Image(part: ImagePart): Promise<ImagePart | null> {
+    try {
+        const req = await fetch(part.image_url.url);
+        if (!req.ok) return null;
+
+        let binary = "";
+        const bytes = new Uint8Array(await req.arrayBuffer());
+        for (let i = 0; i < bytes.length; i += 0x8000) {
+            binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+        }
+
+        return {
+            type: "image_url",
+            image_url: { url: `data:${req.headers.get("content-type") ?? "image/png"};base64,${btoa(binary)}` }
+        };
+    } catch (e) {
+        logger.warn("failed to convert image to base64", e);
+        return null;
+    }
+}
+
 export async function handleResponse(message: Message, response: string): Promise<string> {
     switch (settings.store.mode) {
         case "autoreply":