Revert "chore: remove diffs + discord webhook announcements"

This reverts commit 7208174617.

package.json

@@ -10,7 +10,8 @@
 	},
 	"dependencies": {
 		"@napi-rs/canvas": "^0.1.97",
-		"color": "^5.0.3"
+		"color": "^5.0.3",
+		"discord-api-types": "^0.38.46"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^2.4.11",

src/update/runner.ts

@@ -1,16 +1,21 @@
+import mock from "../mock";
+import type { Differs } from "../types";
 import type { ChannelContext } from "./channel";
 import { commit } from "./git";
 import { log, makeProgress, wrapPromise } from "./progress";
-import { isMock } from "./shared";
+import { isMock, isQuiet, oprevFiles, prevFiles } from "./shared";
 import codeTask from "./tasks/code";
 import colorsTask from "./tasks/colors";
 import decompile from "./tasks/decompile";
+import diffs from "./tasks/diffs";
 import { formatError, join } from "./utils";
 
 export async function runTasks(channel: ChannelContext) {
 	log(`\nRunning tasks for ${channel.cuteVersion}...`);
 	const progress = makeProgress(
 		{
+			preinit: "Preinit",
+			preinit_save: "Caching original files",
 			decompile: "Decompilation",
 			decompile_downloading: "Downloading decompiler",
 			decompile_decompiling: "Decompiling index.android.bundle",
@@ -25,13 +30,40 @@ export async function runTasks(channel: ChannelContext) {
 			icons: "Icon parser",
 			icons_getting: "Writing icons.json",
 			icons_copying: "Copying images",
+			diff: "Diffs",
+			diff_code: "Diffing code",
+			diff_raw: "Diffing raw colors",
+			diff_semantic: "Diffing semantic colors",
+			diff_icons: "Diffing icons",
 		},
 		true,
 	);
+	let differs: Differs = { raw: new Map(), semantic: new Map(), icons: new Map(), code: new Map() };
 
 	let gzipDone: Promise<void> | undefined;
 
 	if (!isMock) {
+		try {
+			progress.start("preinit");
+
+			progress.start("preinit_save");
+			prevFiles.clear();
+			for (const oprev of oprevFiles) {
+				for (const base of [channel.dataDir, channel.canvasDir] as const) {
+					const file = Bun.file(join(base, oprev));
+					if (await file.exists()) {
+						prevFiles.set(oprev, await file.arrayBuffer());
+						break;
+					}
+				}
+			}
+			progress.update("preinit_save", true);
+			progress.update("preinit", true);
+		} catch (e) {
+			progress.update("preinit", false);
+			throw new Error(`Failed to cache original files!\n${e}`);
+		}
+
 		try {
 			gzipDone = (await wrapPromise(
 				decompile(channel, progress, join(channel.apksFolder, "base", "assets", "index.android.bundle")),
@@ -65,6 +97,7 @@ export async function runTasks(channel: ChannelContext) {
 		// 	console.warn("Icons task failed (non-critical):", e);
 		// });
 		progress.update("icons", null);
+		progress.update("diff_icons", null);
 
 		if (gzipDone) {
 			try {
@@ -78,6 +111,30 @@ export async function runTasks(channel: ChannelContext) {
 
 		if (progress.someFailed("decompile_gzip"))
 			throw new Error(`Failed at the decompile gzip task!\n${progress.prettyErrors("decompile_gzip")}`);
+
+		try {
+			const result = await diffs(channel, progress);
+			if (result) {
+				differs = result;
+				progress.update("diff", true);
+			}
+		} catch (e: any) {
+			progress.update("diff", false, formatError(e));
+			throw new Error(`Failed to generate diffs!\n${formatError(e)}`);
+		}
+	} else {
+		differs = mock;
+	}
+
+	if (!isQuiet) {
+		try {
+			const { webhook } = await import("./tasks/webhook");
+			await wrapPromise(webhook(differs), progress, "webhook");
+		} catch (e: any) {
+			throw new Error(`Failed to send webhook messages!\n${formatError(e)}`);
+		}
+	} else {
+		progress.update("webhook", null);
 	}
 
 	await commit(["version.txt"], `chore: bump app version to ${channel.cuteVersion}`, channel.dataDir);

src/update/shared.ts

@@ -1,7 +1,13 @@
 export const trackerUserAgent = "Themelings <https://github.com/nexpid/Themelings/>";
+export const maxDiffChanges = 20;
 
 export const apkSplits = ["base", "config.hdpi", "config.xxhdpi"] as const;
 
+export const oprevFiles = ["source.jsonl", "raw.json", "semantic.json", "semantic_simple.json", "icons.json"] as const;
+export const prevFiles = new Map<(typeof oprevFiles)[number], ArrayBuffer>();
+
 export const force = process.argv.includes("--force");
 export const isMock = process.argv.includes("--mock");
+export const diffAnyway = process.argv.includes("--diff-anyway");
 export const commitAnyway = process.argv.includes("--commit-anyway");
+export const isQuiet = process.argv.includes("--quiet");

src/update/tasks/diffs.ts

@@ -0,0 +1,265 @@
+import {
+	type CodeDiff,
+	type Diff,
+	type Differs,
+	DiffType,
+	type Icons,
+	type RawColors,
+	type SemanticColors,
+} from "../../types";
+import type { ChannelContext } from "../channel";
+import { fetchGitChanged, gitChanged } from "../git";
+import { type Progress, wrapPromise } from "../progress";
+import { diffAnyway, prevFiles } from "../shared";
+import { join, parseJsonl } from "../utils";
+
+interface DiffOpts<TOld, TNew> {
+	getSource: (key: string, val: TNew) => string;
+	getLabel?: (key: string, val: TNew) => string;
+	getOldSource?: (key: string, val: TOld) => string;
+	getOldLabel?: (key: string, val: TOld) => string;
+	isRename: (newKey: string, oldVal: TOld, newVal: TNew) => boolean;
+	isChanged?: (oldVal: TOld, newVal: TNew) => boolean;
+}
+
+function diffEntries<TOld, TNew>(
+	oldEntries: Record<string, TOld>,
+	newEntries: Record<string, TNew>,
+	opts: DiffOpts<TOld, TNew>,
+): Map<string, Diff> {
+	const changes = new Map<string, Diff>();
+	const renamed = new Set<string>();
+
+	for (const [name, newVal] of Object.entries(newEntries)) {
+		const oldVal = oldEntries[name];
+		if (oldVal === undefined) {
+			const oldName = Object.entries(oldEntries).find(
+				([key, val]) => !(key in newEntries) && opts.isRename(name, val, newVal),
+			)?.[0];
+
+			if (!oldName) {
+				changes.set(name, {
+					type: DiffType.Added,
+					source: opts.getSource(name, newVal),
+					label: opts.getLabel?.(name, newVal),
+				});
+			} else {
+				renamed.add(oldName);
+				changes.set(name, { type: DiffType.Renamed, oldName, source: opts.getSource(name, newVal) });
+			}
+		} else if (opts.isChanged?.(oldVal, newVal)) {
+			changes.set(name, {
+				type: DiffType.Changed,
+				source: opts.getSource(name, newVal),
+				label: opts.getLabel?.(name, newVal),
+				oldSource: opts.getOldSource?.(name, oldVal) ?? "",
+				oldLabel: opts.getOldLabel?.(name, oldVal),
+			});
+		}
+	}
+
+	for (const [name, oldVal] of Object.entries(oldEntries)) {
+		if (name in newEntries || renamed.has(name)) continue;
+		changes.set(name, {
+			type: DiffType.Removed,
+			source: opts.getOldSource?.(name, oldVal) ?? "",
+			label: opts.getOldLabel?.(name, oldVal),
+		});
+	}
+
+	return changes;
+}
+
+function diffEntriesCode(
+	oldEntries: Record<string, { size: number; compSize: number }>,
+	newEntries: Record<string, { size: number; compSize: number }>,
+): Map<string, CodeDiff> {
+	const changes = new Map<string, CodeDiff>();
+	const renamed = new Set<string>();
+
+	for (const [name, newVal] of Object.entries(newEntries)) {
+		if (!(name in oldEntries)) {
+			const oldName = Object.entries(oldEntries).find(
+				([key, val]) => !(key in newEntries) && val.compSize === newVal.compSize,
+			)?.[0];
+
+			if (!oldName) {
+				changes.set(name, { type: DiffType.Added, size: newVal.size });
+			} else {
+				renamed.add(oldName);
+				changes.set(name, { type: DiffType.Renamed, oldName, size: newVal.size });
+			}
+		}
+	}
+
+	for (const [name, oldVal] of Object.entries(oldEntries)) {
+		if (!(name in newEntries || renamed.has(name))) {
+			changes.set(name, { type: DiffType.Removed, size: oldVal.size });
+		}
+	}
+
+	return changes;
+}
+
+async function diffRaw(channel: ChannelContext) {
+	if (!prevFiles.has("raw.json")) throw new Error("Missing prevFile: raw.json");
+
+	const oldRaw = JSON.parse(new TextDecoder().decode(prevFiles.get("raw.json"))) as RawColors;
+	const newRaw: RawColors = await Bun.file(join(channel.canvasDir, "raw.json")).json();
+
+	return diffEntries(oldRaw, newRaw, {
+		getSource: (_, val) => val,
+		isRename: (newKey, oldVal, newVal) => !(newKey in newRaw) && oldVal === newVal,
+		isChanged: (oldVal, newVal) => oldVal.toLowerCase() !== newVal.toLowerCase(),
+	});
+}
+
+function getSemanticLabel(semantic: Record<string, [string, unknown]>) {
+	const light = semantic.light;
+	const dark = semantic.darker ?? semantic.dark;
+	if (dark && light) return `${light[0]}, ${dark[0]}`;
+	return Object.entries(semantic)
+		.map(([name, [color]]) => `${name} ${color}`)
+		.join(", ");
+}
+
+async function diffSemantic(channel: ChannelContext) {
+	if (!prevFiles.has("semantic.json")) throw new Error("Missing prevFile: semantic.json");
+
+	const oldSemantic = JSON.parse(new TextDecoder().decode(prevFiles.get("semantic.json"))) as SemanticColors;
+	const newSemantic: SemanticColors = await Bun.file(join(channel.canvasDir, "semantic.json")).json();
+
+	const changes = diffEntries(oldSemantic, newSemantic, {
+		getSource: (_, val) =>
+			Object.values(val)
+				.map(([color]) => color)
+				.join(","),
+		getLabel: (_, val) => getSemanticLabel(val),
+		isRename: (newKey, oldVal, newVal) => {
+			const themes = Object.keys(newVal).length;
+			return (
+				!(newKey in newSemantic) &&
+				Object.keys(oldVal).length === themes &&
+				Object.entries(oldVal).every(([skey, sval]) => sval[0] === newVal[skey]?.[0])
+			);
+		},
+	});
+
+	for (const [name, semantic] of Object.entries(newSemantic)) {
+		if (oldSemantic[name]) {
+			for (const [theme, [color]] of Object.entries(semantic)) {
+				if (!oldSemantic[name][theme]) {
+					changes.set(`${name}.${theme}`, { type: DiffType.Added, source: color });
+				} else if (oldSemantic[name][theme][0].toLowerCase() !== color.toLowerCase()) {
+					changes.set(`${name}.${theme}`, {
+						type: DiffType.Changed,
+						source: color,
+						oldSource: oldSemantic[name][theme][0],
+					});
+				}
+			}
+		}
+	}
+
+	for (const [name, semantic] of Object.entries(oldSemantic)) {
+		if (name in newSemantic) {
+			for (const [theme, [source]] of Object.entries(semantic)) {
+				if (!newSemantic[name][theme]) {
+					changes.set(`${name}.${theme}`, { type: DiffType.Removed, source });
+				}
+			}
+		}
+	}
+
+	return changes;
+}
+
+async function diffIcons(channel: ChannelContext) {
+	if (!prevFiles.has("icons.json")) throw new Error("Missing prevFile: icons.json");
+
+	const oldIcons = JSON.parse(new TextDecoder().decode(prevFiles.get("icons.json"))) as Icons;
+	const newIcons: Icons = await Bun.file(join(channel.canvasDir, "icons.json")).json();
+
+	const iconDir = {
+		old: join(channel.canvasDir, "oldicons"),
+		new: join(channel.canvasDir, "icons"),
+	};
+
+	return diffEntries(oldIcons, newIcons, {
+		getSource: (_, val) => join(iconDir.new, val.file),
+		getLabel: (_, val) => val.hash.slice(0, 8),
+		getOldSource: (_, val) => join(iconDir.old, val.file),
+		getOldLabel: (_, val) => val.hash.slice(0, 8),
+		isRename: (newKey, oldVal, newVal) => !(newKey in newIcons) && oldVal.hash === newVal.hash,
+		isChanged: (oldVal, newVal) => oldVal.hash !== newVal.hash,
+	});
+}
+
+function parseSource(text: string) {
+	const source = parseJsonl(text) as { file: string; size: number }[];
+	return Object.fromEntries(
+		source.map(({ file, size }) => [file, { size, compSize: size - JSON.stringify(file).length }]),
+	) as Record<string, { size: number; compSize: number }>;
+}
+
+async function diffCode(channel: ChannelContext) {
+	if (!prevFiles.has("source.jsonl")) throw new Error("Missing prevFile: source.jsonl");
+
+	const oldCode = parseSource(new TextDecoder().decode(prevFiles.get("source.jsonl")));
+	const newCode = parseSource(await Bun.file(join(channel.dataDir, "source.jsonl")).text());
+
+	return diffEntriesCode(oldCode, newCode);
+}
+
+export default async function diffs(channel: ChannelContext, progress: Progress) {
+	await fetchGitChanged();
+
+	const prefix = `${channel.channel}/`;
+	const changed = (name: string) => gitChanged.has(prefix + name);
+	const hasBaseline = (name: string) => prevFiles.has(name as typeof prevFiles extends Map<infer K, any> ? K : never);
+
+	if (!(changed("version.txt") || diffAnyway)) {
+		progress.update("diff", null);
+		progress.update("diff_raw", null);
+		progress.update("diff_semantic", null);
+		progress.update("diff_icons", null);
+		progress.update("diff_code", null);
+		return;
+	}
+
+	const differs: Differs = { raw: new Map(), semantic: new Map(), icons: new Map(), code: new Map() };
+
+	await Promise.all([
+		changed("raw.json") && hasBaseline("raw.json")
+			? wrapPromise(
+					diffRaw(channel).then((x) => (differs.raw = x)),
+					progress,
+					"diff_raw",
+				)
+			: progress.update("diff_raw", null),
+		changed("semantic.json") && hasBaseline("semantic.json")
+			? wrapPromise(
+					diffSemantic(channel).then((x) => (differs.semantic = x)),
+					progress,
+					"diff_semantic",
+				)
+			: progress.update("diff_semantic", null),
+		changed("icons.json") && hasBaseline("icons.json")
+			? wrapPromise(
+					diffIcons(channel).then((x) => (differs.icons = x)),
+					progress,
+					"diff_icons",
+				)
+			: progress.update("diff_icons", null),
+		changed("source.jsonl") && hasBaseline("source.jsonl")
+			? wrapPromise(
+					diffCode(channel).then((x) => (differs.code = x)),
+					progress,
+					"diff_code",
+				)
+			: progress.update("diff_code", null),
+	]);
+
+	progress.update("diff", true);
+	return differs;
+}

src/update/tasks/webhook.ts

@@ -0,0 +1,285 @@
+import { basename } from "node:path";
+import type { Canvas } from "@napi-rs/canvas";
+import {
+	type APIMessageTopLevelComponent,
+	ComponentType,
+	MessageFlags,
+	type RESTPostAPIWebhookWithTokenJSONBody,
+	RouteBases,
+	Routes,
+} from "discord-api-types/v10";
+import { drawSections } from "../../canvas";
+import { makeSections } from "../../canvas/factory";
+import { type CodeDiff, type Diff, type Differs, DiffType } from "../../types";
+import { ctx } from "../channel";
+import { maxDiffChanges } from "../shared";
+import { formatBytes, sortEntries } from "../utils";
+
+function fileBase(path: string, other?: string) {
+	if (!other) return `... ${basename(path)}`;
+	const paths = path.split("/"),
+		others = other.split("/");
+
+	let offset = 0;
+	while (offset < Math.min(paths.length, others.length) - 1 && paths[offset] === others[offset]) {
+		offset++;
+	}
+
+	return `... ${paths.slice(offset).join("/")}`;
+}
+
+function makeFooter(size: number, item: string) {
+	return `**${size.toLocaleString("en-US")}** ${item} change${size !== 1 ? "s" : ""} total`;
+}
+
+function getThreshold(lines: string[]) {
+	return lines.some((x) => x.includes("\n")) ? Math.floor(maxDiffChanges / 2) : maxDiffChanges;
+}
+
+function cap(lines: string[], item: string) {
+	const threshold = getThreshold(lines);
+	if (lines.length > threshold)
+		return [...lines.slice(0, threshold), `(+${(lines.length - threshold).toLocaleString("en-US")} ${item})`];
+	return lines;
+}
+
+interface FormattedDiff {
+	embed: string[];
+	file: string;
+	capped: boolean;
+}
+
+function formatDiff(diffs: Map<string, Diff | CodeDiff>): FormattedDiff {
+	const entries = sortEntries([...diffs.entries()]);
+
+	const sections = {
+		Added: entries
+			.filter(([, diff]) => diff.type === DiffType.Added)
+			.map(([name, diff]) => {
+				if ("size" in diff) return `+ ${fileBase(name)} (${formatBytes(diff.size)})`;
+				const d = diff as Extract<Diff, { type: DiffType.Added | DiffType.Removed }>;
+				return `+ ${name}: ${d.label || d.source}`;
+			}),
+		Changed: entries
+			.filter(([, diff]) => diff.type === DiffType.Changed)
+			.map(([name, diff]) => {
+				const d = diff as Extract<Diff, { type: DiffType.Changed }>;
+				return `- ${name}: ${d.oldLabel || d.oldSource}\n+ ${name}: ${d.label || d.source}`;
+			}),
+		Renamed: entries
+			.filter(([, diff]) => diff.type === DiffType.Renamed)
+			.map(([name, diff]) => {
+				if ("size" in diff) {
+					const cd = diff as Extract<CodeDiff, { type: DiffType.Renamed }>;
+					return `- ${fileBase(cd.oldName, name)}\n+ ${fileBase(name, cd.oldName)}`;
+				}
+				const d = diff as Extract<Diff, { type: DiffType.Renamed }>;
+				return `- ${d.oldName}\n+ ${name}`;
+			}),
+		Removed: entries
+			.filter(([, diff]) => diff.type === DiffType.Removed)
+			.map(([name, diff]) => {
+				if ("size" in diff) return `- ${fileBase(name)} (${formatBytes(diff.size)})`;
+				const d = diff as Extract<Diff, { type: DiffType.Added | DiffType.Removed }>;
+				return `- ${name}: ${d.label || d.source}`;
+			}),
+	};
+
+	const text = Object.entries(sections).filter(([, lines]) => lines.length);
+	return {
+		embed: text.map(
+			([title, lines]) => `### ${title}\n\`\`\`diff\n${cap(lines, title.toLowerCase()).join("\n")}\`\`\``,
+		),
+		file: text.map(([title, lines]) => `# ${title}\n\n${lines.join("\n")}`).join("\n\n"),
+		capped: text.some(([, lines]) => lines.length > getThreshold(lines)),
+	};
+}
+
+interface WebhookEmbed {
+	title: string;
+	body: FormattedDiff;
+	image?: Canvas;
+	footer: string;
+	key: string;
+}
+
+async function sendWebhook(webhook: string, role: string, embeds: WebhookEmbed[], color?: number) {
+	const body = new FormData();
+	const images = new Map<number, string>(),
+		diffs = new Map<number, string>();
+
+	let fi = 0;
+	for (let i = 0; i < embeds.length; i++) {
+		const embed = embeds[i];
+		if (embed.image) {
+			const key = fi++;
+			const filename = `${embed.key}.png`;
+			images.set(i, filename);
+
+			body.append(`files[${key}]`, new Blob([embed.image.toBuffer("image/png")], { type: "image/png" }), filename);
+		}
+		if (embed.body.capped) {
+			const key = fi++;
+			const filename = `${embed.key}-changes.diff`;
+			diffs.set(i, filename);
+
+			body.append(`files[${key}]`, new Blob([embed.body.file], { type: "text/plain" }), filename);
+		}
+	}
+
+	body.append(
+		"payload_json",
+		JSON.stringify({
+			flags: MessageFlags.IsComponentsV2,
+			components: [
+				{
+					type: ComponentType.TextDisplay,
+					content: `<@&${role}>`,
+				},
+				...embeds.map(
+					({ title, body, image, footer }, i) =>
+						({
+							type: ComponentType.Container,
+							accent_color: color,
+							components: [
+								{
+									type: ComponentType.TextDisplay,
+									content: `## ${title}\n-# ${ctx().version} (${ctx().cuteVersion})`,
+								},
+								...body.embed.map((content) => ({
+									type: ComponentType.TextDisplay,
+									content,
+								})),
+								body.capped && {
+									type: ComponentType.File,
+									file: {
+										url: `attachment://${diffs.get(i)}`,
+									},
+								},
+								body.capped && {
+									type: ComponentType.Separator,
+								},
+								image && {
+									type: ComponentType.MediaGallery,
+									items: [
+										{
+											media: {
+												url: `attachment://${images.get(i)}`,
+											},
+										},
+									],
+								},
+								{
+									type: ComponentType.TextDisplay,
+									content: footer,
+								},
+							].filter((x) => !!x),
+						}) as APIMessageTopLevelComponent,
+				),
+			],
+			allowed_mentions: process.env.NODE_ENV === "test" ? { parse: [] } : { roles: [role] },
+		} as RESTPostAPIWebhookWithTokenJSONBody),
+	);
+
+	const url = new URL(webhook);
+	url.searchParams.set("wait", "true");
+	url.searchParams.set("with_components", "true");
+
+	const res = await fetch(url, {
+		method: "POST",
+		body,
+	});
+
+	const data = await res.text();
+	if (!res.ok)
+		throw new Error(`Failed to send webhook message with embeds ${embeds.map((x) => x.title).join(", ")}: ${data}`);
+
+	const discordToken = process.env.bot_token;
+	if (!discordToken) return;
+
+	try {
+		const message = JSON.parse(data);
+
+		const route = `${RouteBases.api}/${Routes.channelMessageCrosspost(message.channel_id, message.id)}`;
+		await fetch(route, {
+			method: "POST",
+			headers: {
+				Authorization: discordToken,
+			},
+		});
+	} catch {}
+}
+
+export async function webhook(diffs: Differs) {
+	if (diffs.raw.size || diffs.semantic.size) {
+		const wh = process.env.color_webhook;
+		if (!wh) {
+			console.warn("Missing color_webhook, skipping color webhook");
+		} else {
+			await sendWebhook(
+				wh,
+				"1227327297795657850",
+				[
+					diffs.raw.size && {
+						title: "Raw colors",
+						body: formatDiff(diffs.raw),
+						image: drawSections(await makeSections(diffs.raw)),
+						footer: makeFooter(diffs.raw.size, "raw color"),
+						key: "raw-colors",
+					},
+					diffs.semantic.size && {
+						title: "Semantic colors",
+						body: formatDiff(diffs.semantic),
+						image: drawSections(await makeSections(diffs.semantic)),
+						footer: makeFooter(diffs.semantic.size, "semantic color"),
+						key: "semantic-colors",
+					},
+				].filter((x) => !!x) as WebhookEmbed[],
+				0xd8ef9e,
+			);
+		}
+	}
+
+	if (diffs.icons.size) {
+		const wh = process.env.icons_webhook;
+		if (!wh) {
+			console.warn("Missing icons_webhook, skipping");
+		} else {
+			await sendWebhook(
+				wh,
+				"1227327765079003217",
+				[
+					{
+						title: "Icons",
+						body: formatDiff(diffs.icons),
+						image: drawSections(await makeSections(diffs.icons, true)),
+						footer: makeFooter(diffs.icons.size, "icon"),
+						key: "icons",
+					},
+				],
+				0xf4b8f7,
+			);
+		}
+	}
+
+	if (diffs.code.size) {
+		const wh = process.env.code_webhook;
+		if (!wh) {
+			console.warn("Missing code_webhook, skipping");
+			return;
+		}
+		await sendWebhook(
+			wh,
+			"1233861867059941387",
+			[
+				{
+					title: "Code",
+					body: formatDiff(diffs.code),
+					footer: makeFooter(diffs.code.size, "code"),
+					key: "code",
+				},
+			],
+			0x86faf3,
+		);
+	}
+}