More DB and miscellaneous fixes (#958)
* Trim MessageLoggerEnhanced hot paths * Add in-memory DataStore fallback * Tighten DataStore memory store types
This commit is contained in:
Jake
GitHub
parent
2b9dd6c773
commit
b752707def
3 changed files with 115 additions and 31 deletions
|
|
@ -22,6 +22,20 @@ import { Logger } from "@utils/Logger";
|
|||
|
||||
const logger = new Logger("DataStore");
|
||||
let loggedFailure = false;
|
||||
const memoryDatabases = new Map<string, Map<string, Map<IDBValidKey, unknown>>>();
|
||||
|
||||
interface MemoryStore {
|
||||
__memoryStore: true;
|
||||
get<T>(key: IDBValidKey): T | undefined;
|
||||
put(value: unknown, key: IDBValidKey): void;
|
||||
delete(key: IDBValidKey): void;
|
||||
clear(): void;
|
||||
getAllKeys<KeyType extends IDBValidKey>(): KeyType[];
|
||||
getAll<T>(): T[];
|
||||
entries<KeyType extends IDBValidKey, ValueType>(): [KeyType, ValueType][];
|
||||
}
|
||||
|
||||
type StoreLike = IDBObjectStore | MemoryStore;
|
||||
|
||||
function handleFailure(error: unknown) {
|
||||
if (loggedFailure) return;
|
||||
|
|
@ -36,6 +50,35 @@ function withFailureFallback<T>(promise: Promise<T>, fallback: T): Promise<T> {
|
|||
});
|
||||
}
|
||||
|
||||
function getMemoryStore(dbName: string, storeName: string): MemoryStore {
|
||||
let database = memoryDatabases.get(dbName);
|
||||
if (!database) {
|
||||
database = new Map();
|
||||
memoryDatabases.set(dbName, database);
|
||||
}
|
||||
|
||||
let data = database.get(storeName);
|
||||
if (!data) {
|
||||
data = new Map<IDBValidKey, unknown>();
|
||||
database.set(storeName, data);
|
||||
}
|
||||
|
||||
return {
|
||||
__memoryStore: true,
|
||||
get: <T>(key: IDBValidKey) => data.get(key) as T | undefined,
|
||||
put: (value, key) => { data.set(key, value); },
|
||||
delete: key => { data.delete(key); },
|
||||
clear: () => { data.clear(); },
|
||||
getAllKeys: <KeyType extends IDBValidKey>() => Array.from(data.keys()) as KeyType[],
|
||||
getAll: <T>() => Array.from(data.values()) as T[],
|
||||
entries: <KeyType extends IDBValidKey, ValueType>() => Array.from(data.entries()) as [KeyType, ValueType][]
|
||||
};
|
||||
}
|
||||
|
||||
function isMemoryStore(store: StoreLike): store is MemoryStore {
|
||||
return "__memoryStore" in store;
|
||||
}
|
||||
|
||||
export function promisifyRequest<T = undefined>(
|
||||
request: IDBRequest<T> | IDBTransaction,
|
||||
): Promise<T> {
|
||||
|
|
@ -49,22 +92,23 @@ export function promisifyRequest<T = undefined>(
|
|||
export function createStore(dbName: string, storeName: string): UseStore {
|
||||
const request = indexedDB.open(dbName);
|
||||
request.onupgradeneeded = () => request.result.createObjectStore(storeName);
|
||||
const dbp = promisifyRequest(request);
|
||||
const dbp = promisifyRequest(request).catch(error => {
|
||||
handleFailure(error);
|
||||
return null;
|
||||
});
|
||||
const memoryStore = getMemoryStore(dbName, storeName);
|
||||
|
||||
return async (txMode, callback) => {
|
||||
try {
|
||||
const db = await dbp;
|
||||
return await callback(db.transaction(storeName, txMode).objectStore(storeName));
|
||||
} catch (error) {
|
||||
handleFailure(error);
|
||||
throw error;
|
||||
}
|
||||
const db = await dbp;
|
||||
if (!db) return callback(memoryStore);
|
||||
|
||||
return callback(db.transaction(storeName, txMode).objectStore(storeName));
|
||||
};
|
||||
}
|
||||
|
||||
export type UseStore = <T>(
|
||||
txMode: IDBTransactionMode,
|
||||
callback: (store: IDBObjectStore) => T | PromiseLike<T>,
|
||||
callback: (store: StoreLike) => T | PromiseLike<T>,
|
||||
) => Promise<T>;
|
||||
|
||||
let defaultGetStoreFunc: UseStore | undefined;
|
||||
|
|
@ -80,7 +124,10 @@ export function get<T = any>(
|
|||
key: IDBValidKey,
|
||||
customStore = defaultGetStore(),
|
||||
): Promise<T | undefined> {
|
||||
return withFailureFallback(customStore("readonly", store => promisifyRequest(store.get(key))), undefined);
|
||||
return withFailureFallback(customStore("readonly", store => {
|
||||
if (isMemoryStore(store)) return store.get(key) as T | undefined;
|
||||
return promisifyRequest(store.get(key));
|
||||
}), undefined);
|
||||
}
|
||||
|
||||
export function set(
|
||||
|
|
@ -90,6 +137,8 @@ export function set(
|
|||
): Promise<void> {
|
||||
return withFailureFallback(customStore("readwrite", store => {
|
||||
store.put(value, key);
|
||||
if (isMemoryStore(store)) return;
|
||||
|
||||
return promisifyRequest(store.transaction);
|
||||
}), undefined);
|
||||
}
|
||||
|
|
@ -100,6 +149,8 @@ export function setMany(
|
|||
): Promise<void> {
|
||||
return withFailureFallback(customStore("readwrite", store => {
|
||||
entries.forEach(entry => store.put(entry[1], entry[0]));
|
||||
if (isMemoryStore(store)) return;
|
||||
|
||||
return promisifyRequest(store.transaction);
|
||||
}), undefined);
|
||||
}
|
||||
|
|
@ -108,9 +159,13 @@ export function getMany<T = any>(
|
|||
keys: IDBValidKey[],
|
||||
customStore = defaultGetStore(),
|
||||
): Promise<T[]> {
|
||||
return withFailureFallback(customStore("readonly", store =>
|
||||
Promise.all(keys.map(key => promisifyRequest(store.get(key)))),
|
||||
), []);
|
||||
return withFailureFallback(customStore("readonly", store => {
|
||||
if (isMemoryStore(store)) {
|
||||
return keys.map(key => store.get(key) as T | undefined);
|
||||
}
|
||||
|
||||
return Promise.all(keys.map(key => promisifyRequest(store.get(key))));
|
||||
}), []);
|
||||
}
|
||||
|
||||
export function update<T = any>(
|
||||
|
|
@ -120,8 +175,13 @@ export function update<T = any>(
|
|||
): Promise<void> {
|
||||
return withFailureFallback(customStore(
|
||||
"readwrite",
|
||||
store =>
|
||||
new Promise((resolve, reject) => {
|
||||
store => {
|
||||
if (isMemoryStore(store)) {
|
||||
store.put(updater(store.get(key) as T | undefined), key);
|
||||
return;
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
store.get(key).onsuccess = function () {
|
||||
try {
|
||||
store.put(updater(this.result), key);
|
||||
|
|
@ -130,8 +190,9 @@ export function update<T = any>(
|
|||
reject(err);
|
||||
}
|
||||
};
|
||||
}),
|
||||
), undefined as void);
|
||||
});
|
||||
},
|
||||
), undefined);
|
||||
}
|
||||
|
||||
export function del(
|
||||
|
|
@ -140,8 +201,10 @@ export function del(
|
|||
): Promise<void> {
|
||||
return withFailureFallback(customStore("readwrite", store => {
|
||||
store.delete(key);
|
||||
if (isMemoryStore(store)) return;
|
||||
|
||||
return promisifyRequest(store.transaction);
|
||||
}), undefined as void);
|
||||
}), undefined);
|
||||
}
|
||||
|
||||
export function delMany(
|
||||
|
|
@ -150,15 +213,19 @@ export function delMany(
|
|||
): Promise<void> {
|
||||
return withFailureFallback(customStore("readwrite", store => {
|
||||
keys.forEach(key => store.delete(key));
|
||||
if (isMemoryStore(store)) return;
|
||||
|
||||
return promisifyRequest(store.transaction);
|
||||
}), undefined as void);
|
||||
}), undefined);
|
||||
}
|
||||
|
||||
export function clear(customStore = defaultGetStore()): Promise<void> {
|
||||
return withFailureFallback(customStore("readwrite", store => {
|
||||
store.clear();
|
||||
if (isMemoryStore(store)) return;
|
||||
|
||||
return promisifyRequest(store.transaction);
|
||||
}), undefined as void);
|
||||
}), undefined);
|
||||
}
|
||||
|
||||
function eachCursor(
|
||||
|
|
@ -177,23 +244,33 @@ export function keys<KeyType extends IDBValidKey>(
|
|||
customStore = defaultGetStore(),
|
||||
): Promise<KeyType[]> {
|
||||
return withFailureFallback(customStore("readonly", store => {
|
||||
if (isMemoryStore(store)) return store.getAllKeys<KeyType>();
|
||||
|
||||
if (store.getAllKeys) {
|
||||
return promisifyRequest(store.getAllKeys() as unknown as IDBRequest<KeyType[]>);
|
||||
return promisifyRequest(
|
||||
store.getAllKeys() as unknown as IDBRequest<KeyType[]>,
|
||||
);
|
||||
}
|
||||
|
||||
const items: KeyType[] = [];
|
||||
return eachCursor(store, cursor => items.push(cursor.key as KeyType)).then(() => items);
|
||||
return eachCursor(store, cursor =>
|
||||
items.push(cursor.key as KeyType),
|
||||
).then(() => items);
|
||||
}), [] as KeyType[]);
|
||||
}
|
||||
|
||||
export function values<T = any>(customStore = defaultGetStore()): Promise<T[]> {
|
||||
return withFailureFallback(customStore("readonly", store => {
|
||||
if (isMemoryStore(store)) return store.getAll<T>();
|
||||
|
||||
if (store.getAll) {
|
||||
return promisifyRequest(store.getAll() as IDBRequest<T[]>);
|
||||
}
|
||||
|
||||
const items: T[] = [];
|
||||
return eachCursor(store, cursor => items.push(cursor.value as T)).then(() => items);
|
||||
return eachCursor(store, cursor => items.push(cursor.value as T)).then(
|
||||
() => items,
|
||||
);
|
||||
}), [] as T[]);
|
||||
}
|
||||
|
||||
|
|
@ -201,14 +278,20 @@ export function entries<KeyType extends IDBValidKey, ValueType = any>(
|
|||
customStore = defaultGetStore(),
|
||||
): Promise<[KeyType, ValueType][]> {
|
||||
return withFailureFallback(customStore("readonly", store => {
|
||||
if (isMemoryStore(store)) return store.entries<KeyType, ValueType>();
|
||||
|
||||
if (store.getAll && store.getAllKeys) {
|
||||
return Promise.all([
|
||||
promisifyRequest(store.getAllKeys() as unknown as IDBRequest<KeyType[]>),
|
||||
promisifyRequest(
|
||||
store.getAllKeys() as unknown as IDBRequest<KeyType[]>,
|
||||
),
|
||||
promisifyRequest(store.getAll() as IDBRequest<ValueType[]>),
|
||||
]).then(([keys, values]) => keys.map((key, i) => [key, values[i]] as [KeyType, ValueType]));
|
||||
}
|
||||
|
||||
const items: [KeyType, ValueType][] = [];
|
||||
return eachCursor(store, cursor => items.push([cursor.key as KeyType, cursor.value])).then(() => items);
|
||||
return eachCursor(store, cursor =>
|
||||
items.push([cursor.key as KeyType, cursor.value]),
|
||||
).then(() => items);
|
||||
}), [] as [KeyType, ValueType][]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,10 +89,12 @@ async function withDb<T>(fallback: T, callback: (database: IDBPDatabase<MLIDB>)
|
|||
}
|
||||
}
|
||||
|
||||
async function cacheRecords(records: DBMessageRecord[]) {
|
||||
async function cacheRecords(records: DBMessageRecord[], cacheAttachmentBlobs = true) {
|
||||
for (const r of records) {
|
||||
cacheRecord(r);
|
||||
|
||||
if (!cacheAttachmentBlobs) continue;
|
||||
|
||||
for (const att of r.message.attachments) {
|
||||
const blobUrl = await getAttachmentBlobUrl(att);
|
||||
if (blobUrl) {
|
||||
|
|
@ -194,7 +196,7 @@ export async function getOlderThanTimestampIDB(timestamp: string) {
|
|||
messages.push(c.value);
|
||||
}
|
||||
|
||||
return cacheRecords(messages);
|
||||
return cacheRecords(messages, false);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -245,7 +247,7 @@ export async function getMessagesByChannelAndAfterTimestampIDB(channel_id: strin
|
|||
messages.push(c.value);
|
||||
}
|
||||
|
||||
return cacheRecords(messages);
|
||||
return cacheRecords(messages, false);
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ import { DBMessageStatus } from "../db";
|
|||
import { LoggedMessageJSON } from "../types";
|
||||
import { DEFAULT_IMAGE_CACHE_DIR } from "./constants";
|
||||
import { DISCORD_EPOCH } from "./index";
|
||||
import { memoize } from "./memoize";
|
||||
|
||||
const MessageClass: any = findLazy(m => m?.prototype?.isEdited);
|
||||
const AuthorClass = findLazy(m => m?.prototype?.getAvatarURL);
|
||||
|
|
@ -93,7 +92,7 @@ export const mapTimestamp = (m: any) => {
|
|||
return m;
|
||||
};
|
||||
|
||||
export const messageJsonToMessageClass = memoize((log: { message: LoggedMessageJSON; }) => {
|
||||
export const messageJsonToMessageClass = (log: { message: LoggedMessageJSON; }) => {
|
||||
// console.time("message populate");
|
||||
if (!log?.message) return null;
|
||||
|
||||
|
|
@ -123,7 +122,7 @@ export const messageJsonToMessageClass = memoize((log: { message: LoggedMessageJ
|
|||
|
||||
// console.timeEnd("message populate");
|
||||
return message;
|
||||
});
|
||||
};
|
||||
|
||||
export function parseJSON(json?: string | null) {
|
||||
try {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue