From b752707def34ad79fc3f98d7250cd0942cf9b339 Mon Sep 17 00:00:00 2001 From: Jake <134332787+justjxke@users.noreply.github.com> Date: Thu, 9 Apr 2026 23:09:16 +0100 Subject: [PATCH] More DB and miscellaneous fixes (#958) * Trim MessageLoggerEnhanced hot paths * Add in-memory DataStore fallback * Tighten DataStore memory store types --- src/api/DataStore/index.ts | 133 ++++++++++++++---- .../messageLoggerEnhanced/db.ts | 8 +- .../messageLoggerEnhanced/utils/misc.ts | 5 +- 3 files changed, 115 insertions(+), 31 deletions(-) diff --git a/src/api/DataStore/index.ts b/src/api/DataStore/index.ts index ea75c36ed..3cdb50c7e 100644 --- a/src/api/DataStore/index.ts +++ b/src/api/DataStore/index.ts @@ -22,6 +22,20 @@ import { Logger } from "@utils/Logger"; const logger = new Logger("DataStore"); let loggedFailure = false; +const memoryDatabases = new Map>>(); + +interface MemoryStore { + __memoryStore: true; + get(key: IDBValidKey): T | undefined; + put(value: unknown, key: IDBValidKey): void; + delete(key: IDBValidKey): void; + clear(): void; + getAllKeys(): KeyType[]; + getAll(): T[]; + entries(): [KeyType, ValueType][]; +} + +type StoreLike = IDBObjectStore | MemoryStore; function handleFailure(error: unknown) { if (loggedFailure) return; @@ -36,6 +50,35 @@ function withFailureFallback(promise: Promise, fallback: T): Promise { }); } +function getMemoryStore(dbName: string, storeName: string): MemoryStore { + let database = memoryDatabases.get(dbName); + if (!database) { + database = new Map(); + memoryDatabases.set(dbName, database); + } + + let data = database.get(storeName); + if (!data) { + data = new Map(); + database.set(storeName, data); + } + + return { + __memoryStore: true, + get: (key: IDBValidKey) => data.get(key) as T | undefined, + put: (value, key) => { data.set(key, value); }, + delete: key => { data.delete(key); }, + clear: () => { data.clear(); }, + getAllKeys: () => Array.from(data.keys()) as KeyType[], + getAll: () => Array.from(data.values()) as T[], + entries: () => Array.from(data.entries()) as [KeyType, ValueType][] + }; +} + +function isMemoryStore(store: StoreLike): store is MemoryStore { + return "__memoryStore" in store; +} + export function promisifyRequest( request: IDBRequest | IDBTransaction, ): Promise { @@ -49,22 +92,23 @@ export function promisifyRequest( export function createStore(dbName: string, storeName: string): UseStore { const request = indexedDB.open(dbName); request.onupgradeneeded = () => request.result.createObjectStore(storeName); - const dbp = promisifyRequest(request); + const dbp = promisifyRequest(request).catch(error => { + handleFailure(error); + return null; + }); + const memoryStore = getMemoryStore(dbName, storeName); return async (txMode, callback) => { - try { - const db = await dbp; - return await callback(db.transaction(storeName, txMode).objectStore(storeName)); - } catch (error) { - handleFailure(error); - throw error; - } + const db = await dbp; + if (!db) return callback(memoryStore); + + return callback(db.transaction(storeName, txMode).objectStore(storeName)); }; } export type UseStore = ( txMode: IDBTransactionMode, - callback: (store: IDBObjectStore) => T | PromiseLike, + callback: (store: StoreLike) => T | PromiseLike, ) => Promise; let defaultGetStoreFunc: UseStore | undefined; @@ -80,7 +124,10 @@ export function get( key: IDBValidKey, customStore = defaultGetStore(), ): Promise { - return withFailureFallback(customStore("readonly", store => promisifyRequest(store.get(key))), undefined); + return withFailureFallback(customStore("readonly", store => { + if (isMemoryStore(store)) return store.get(key) as T | undefined; + return promisifyRequest(store.get(key)); + }), undefined); } export function set( @@ -90,6 +137,8 @@ export function set( ): Promise { return withFailureFallback(customStore("readwrite", store => { store.put(value, key); + if (isMemoryStore(store)) return; + return promisifyRequest(store.transaction); }), undefined); } @@ -100,6 +149,8 @@ export function setMany( ): Promise { return withFailureFallback(customStore("readwrite", store => { entries.forEach(entry => store.put(entry[1], entry[0])); + if (isMemoryStore(store)) return; + return promisifyRequest(store.transaction); }), undefined); } @@ -108,9 +159,13 @@ export function getMany( keys: IDBValidKey[], customStore = defaultGetStore(), ): Promise { - return withFailureFallback(customStore("readonly", store => - Promise.all(keys.map(key => promisifyRequest(store.get(key)))), - ), []); + return withFailureFallback(customStore("readonly", store => { + if (isMemoryStore(store)) { + return keys.map(key => store.get(key) as T | undefined); + } + + return Promise.all(keys.map(key => promisifyRequest(store.get(key)))); + }), []); } export function update( @@ -120,8 +175,13 @@ export function update( ): Promise { return withFailureFallback(customStore( "readwrite", - store => - new Promise((resolve, reject) => { + store => { + if (isMemoryStore(store)) { + store.put(updater(store.get(key) as T | undefined), key); + return; + } + + return new Promise((resolve, reject) => { store.get(key).onsuccess = function () { try { store.put(updater(this.result), key); @@ -130,8 +190,9 @@ export function update( reject(err); } }; - }), - ), undefined as void); + }); + }, + ), undefined); } export function del( @@ -140,8 +201,10 @@ export function del( ): Promise { return withFailureFallback(customStore("readwrite", store => { store.delete(key); + if (isMemoryStore(store)) return; + return promisifyRequest(store.transaction); - }), undefined as void); + }), undefined); } export function delMany( @@ -150,15 +213,19 @@ export function delMany( ): Promise { return withFailureFallback(customStore("readwrite", store => { keys.forEach(key => store.delete(key)); + if (isMemoryStore(store)) return; + return promisifyRequest(store.transaction); - }), undefined as void); + }), undefined); } export function clear(customStore = defaultGetStore()): Promise { return withFailureFallback(customStore("readwrite", store => { store.clear(); + if (isMemoryStore(store)) return; + return promisifyRequest(store.transaction); - }), undefined as void); + }), undefined); } function eachCursor( @@ -177,23 +244,33 @@ export function keys( customStore = defaultGetStore(), ): Promise { return withFailureFallback(customStore("readonly", store => { + if (isMemoryStore(store)) return store.getAllKeys(); + if (store.getAllKeys) { - return promisifyRequest(store.getAllKeys() as unknown as IDBRequest); + return promisifyRequest( + store.getAllKeys() as unknown as IDBRequest, + ); } const items: KeyType[] = []; - return eachCursor(store, cursor => items.push(cursor.key as KeyType)).then(() => items); + return eachCursor(store, cursor => + items.push(cursor.key as KeyType), + ).then(() => items); }), [] as KeyType[]); } export function values(customStore = defaultGetStore()): Promise { return withFailureFallback(customStore("readonly", store => { + if (isMemoryStore(store)) return store.getAll(); + if (store.getAll) { return promisifyRequest(store.getAll() as IDBRequest); } const items: T[] = []; - return eachCursor(store, cursor => items.push(cursor.value as T)).then(() => items); + return eachCursor(store, cursor => items.push(cursor.value as T)).then( + () => items, + ); }), [] as T[]); } @@ -201,14 +278,20 @@ export function entries( customStore = defaultGetStore(), ): Promise<[KeyType, ValueType][]> { return withFailureFallback(customStore("readonly", store => { + if (isMemoryStore(store)) return store.entries(); + if (store.getAll && store.getAllKeys) { return Promise.all([ - promisifyRequest(store.getAllKeys() as unknown as IDBRequest), + promisifyRequest( + store.getAllKeys() as unknown as IDBRequest, + ), promisifyRequest(store.getAll() as IDBRequest), ]).then(([keys, values]) => keys.map((key, i) => [key, values[i]] as [KeyType, ValueType])); } const items: [KeyType, ValueType][] = []; - return eachCursor(store, cursor => items.push([cursor.key as KeyType, cursor.value])).then(() => items); + return eachCursor(store, cursor => + items.push([cursor.key as KeyType, cursor.value]), + ).then(() => items); }), [] as [KeyType, ValueType][]); } diff --git a/src/equicordplugins/messageLoggerEnhanced/db.ts b/src/equicordplugins/messageLoggerEnhanced/db.ts index 2a11d18dc..31c34ef45 100644 --- a/src/equicordplugins/messageLoggerEnhanced/db.ts +++ b/src/equicordplugins/messageLoggerEnhanced/db.ts @@ -89,10 +89,12 @@ async function withDb(fallback: T, callback: (database: IDBPDatabase) } } -async function cacheRecords(records: DBMessageRecord[]) { +async function cacheRecords(records: DBMessageRecord[], cacheAttachmentBlobs = true) { for (const r of records) { cacheRecord(r); + if (!cacheAttachmentBlobs) continue; + for (const att of r.message.attachments) { const blobUrl = await getAttachmentBlobUrl(att); if (blobUrl) { @@ -194,7 +196,7 @@ export async function getOlderThanTimestampIDB(timestamp: string) { messages.push(c.value); } - return cacheRecords(messages); + return cacheRecords(messages, false); }); } @@ -245,7 +247,7 @@ export async function getMessagesByChannelAndAfterTimestampIDB(channel_id: strin messages.push(c.value); } - return cacheRecords(messages); + return cacheRecords(messages, false); }); } diff --git a/src/equicordplugins/messageLoggerEnhanced/utils/misc.ts b/src/equicordplugins/messageLoggerEnhanced/utils/misc.ts index 66a264abf..43c5f4166 100644 --- a/src/equicordplugins/messageLoggerEnhanced/utils/misc.ts +++ b/src/equicordplugins/messageLoggerEnhanced/utils/misc.ts @@ -24,7 +24,6 @@ import { DBMessageStatus } from "../db"; import { LoggedMessageJSON } from "../types"; import { DEFAULT_IMAGE_CACHE_DIR } from "./constants"; import { DISCORD_EPOCH } from "./index"; -import { memoize } from "./memoize"; const MessageClass: any = findLazy(m => m?.prototype?.isEdited); const AuthorClass = findLazy(m => m?.prototype?.getAvatarURL); @@ -93,7 +92,7 @@ export const mapTimestamp = (m: any) => { return m; }; -export const messageJsonToMessageClass = memoize((log: { message: LoggedMessageJSON; }) => { +export const messageJsonToMessageClass = (log: { message: LoggedMessageJSON; }) => { // console.time("message populate"); if (!log?.message) return null; @@ -123,7 +122,7 @@ export const messageJsonToMessageClass = memoize((log: { message: LoggedMessageJ // console.timeEnd("message populate"); return message; -}); +}; export function parseJSON(json?: string | null) { try {