diff --git a/package.json b/package.json index d2c4ff287af..d93348ace2f 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,8 @@ }, "dependencies": { "@napi-rs/canvas": "^0.1.97", - "color": "^5.0.3" + "color": "^5.0.3", + "discord-api-types": "^0.38.46" }, "devDependencies": { "@biomejs/biome": "^2.4.11", diff --git a/src/update/runner.ts b/src/update/runner.ts index 64a2db9ad92..7a4072ee866 100644 --- a/src/update/runner.ts +++ b/src/update/runner.ts @@ -1,16 +1,21 @@ +import mock from "../mock"; +import type { Differs } from "../types"; import type { ChannelContext } from "./channel"; import { commit } from "./git"; import { log, makeProgress, wrapPromise } from "./progress"; -import { isMock } from "./shared"; +import { isMock, isQuiet, oprevFiles, prevFiles } from "./shared"; import codeTask from "./tasks/code"; import colorsTask from "./tasks/colors"; import decompile from "./tasks/decompile"; +import diffs from "./tasks/diffs"; import { formatError, join } from "./utils"; export async function runTasks(channel: ChannelContext) { log(`\nRunning tasks for ${channel.cuteVersion}...`); const progress = makeProgress( { + preinit: "Preinit", + preinit_save: "Caching original files", decompile: "Decompilation", decompile_downloading: "Downloading decompiler", decompile_decompiling: "Decompiling index.android.bundle", @@ -25,13 +30,40 @@ export async function runTasks(channel: ChannelContext) { icons: "Icon parser", icons_getting: "Writing icons.json", icons_copying: "Copying images", + diff: "Diffs", + diff_code: "Diffing code", + diff_raw: "Diffing raw colors", + diff_semantic: "Diffing semantic colors", + diff_icons: "Diffing icons", }, true, ); + let differs: Differs = { raw: new Map(), semantic: new Map(), icons: new Map(), code: new Map() }; let gzipDone: Promise | undefined; if (!isMock) { + try { + progress.start("preinit"); + + progress.start("preinit_save"); + prevFiles.clear(); + for (const oprev of oprevFiles) { + for (const base of [channel.dataDir, channel.canvasDir] as const) { + const file = Bun.file(join(base, oprev)); + if (await file.exists()) { + prevFiles.set(oprev, await file.arrayBuffer()); + break; + } + } + } + progress.update("preinit_save", true); + progress.update("preinit", true); + } catch (e) { + progress.update("preinit", false); + throw new Error(`Failed to cache original files!\n${e}`); + } + try { gzipDone = (await wrapPromise( decompile(channel, progress, join(channel.apksFolder, "base", "assets", "index.android.bundle")), @@ -65,6 +97,7 @@ export async function runTasks(channel: ChannelContext) { // console.warn("Icons task failed (non-critical):", e); // }); progress.update("icons", null); + progress.update("diff_icons", null); if (gzipDone) { try { @@ -78,6 +111,30 @@ export async function runTasks(channel: ChannelContext) { if (progress.someFailed("decompile_gzip")) throw new Error(`Failed at the decompile gzip task!\n${progress.prettyErrors("decompile_gzip")}`); + + try { + const result = await diffs(channel, progress); + if (result) { + differs = result; + progress.update("diff", true); + } + } catch (e: any) { + progress.update("diff", false, formatError(e)); + throw new Error(`Failed to generate diffs!\n${formatError(e)}`); + } + } else { + differs = mock; + } + + if (!isQuiet) { + try { + const { webhook } = await import("./tasks/webhook"); + await wrapPromise(webhook(differs), progress, "webhook"); + } catch (e: any) { + throw new Error(`Failed to send webhook messages!\n${formatError(e)}`); + } + } else { + progress.update("webhook", null); } await commit(["version.txt"], `chore: bump app version to ${channel.cuteVersion}`, channel.dataDir); diff --git a/src/update/shared.ts b/src/update/shared.ts index b7dd4350f96..710cbefad8a 100644 --- a/src/update/shared.ts +++ b/src/update/shared.ts @@ -1,7 +1,13 @@ export const trackerUserAgent = "Themelings "; +export const maxDiffChanges = 20; export const apkSplits = ["base", "config.hdpi", "config.xxhdpi"] as const; +export const oprevFiles = ["source.jsonl", "raw.json", "semantic.json", "semantic_simple.json", "icons.json"] as const; +export const prevFiles = new Map<(typeof oprevFiles)[number], ArrayBuffer>(); + export const force = process.argv.includes("--force"); export const isMock = process.argv.includes("--mock"); +export const diffAnyway = process.argv.includes("--diff-anyway"); export const commitAnyway = process.argv.includes("--commit-anyway"); +export const isQuiet = process.argv.includes("--quiet"); diff --git a/src/update/tasks/diffs.ts b/src/update/tasks/diffs.ts new file mode 100644 index 00000000000..d3685ba924e --- /dev/null +++ b/src/update/tasks/diffs.ts @@ -0,0 +1,265 @@ +import { + type CodeDiff, + type Diff, + type Differs, + DiffType, + type Icons, + type RawColors, + type SemanticColors, +} from "../../types"; +import type { ChannelContext } from "../channel"; +import { fetchGitChanged, gitChanged } from "../git"; +import { type Progress, wrapPromise } from "../progress"; +import { diffAnyway, prevFiles } from "../shared"; +import { join, parseJsonl } from "../utils"; + +interface DiffOpts { + getSource: (key: string, val: TNew) => string; + getLabel?: (key: string, val: TNew) => string; + getOldSource?: (key: string, val: TOld) => string; + getOldLabel?: (key: string, val: TOld) => string; + isRename: (newKey: string, oldVal: TOld, newVal: TNew) => boolean; + isChanged?: (oldVal: TOld, newVal: TNew) => boolean; +} + +function diffEntries( + oldEntries: Record, + newEntries: Record, + opts: DiffOpts, +): Map { + const changes = new Map(); + const renamed = new Set(); + + for (const [name, newVal] of Object.entries(newEntries)) { + const oldVal = oldEntries[name]; + if (oldVal === undefined) { + const oldName = Object.entries(oldEntries).find( + ([key, val]) => !(key in newEntries) && opts.isRename(name, val, newVal), + )?.[0]; + + if (!oldName) { + changes.set(name, { + type: DiffType.Added, + source: opts.getSource(name, newVal), + label: opts.getLabel?.(name, newVal), + }); + } else { + renamed.add(oldName); + changes.set(name, { type: DiffType.Renamed, oldName, source: opts.getSource(name, newVal) }); + } + } else if (opts.isChanged?.(oldVal, newVal)) { + changes.set(name, { + type: DiffType.Changed, + source: opts.getSource(name, newVal), + label: opts.getLabel?.(name, newVal), + oldSource: opts.getOldSource?.(name, oldVal) ?? "", + oldLabel: opts.getOldLabel?.(name, oldVal), + }); + } + } + + for (const [name, oldVal] of Object.entries(oldEntries)) { + if (name in newEntries || renamed.has(name)) continue; + changes.set(name, { + type: DiffType.Removed, + source: opts.getOldSource?.(name, oldVal) ?? "", + label: opts.getOldLabel?.(name, oldVal), + }); + } + + return changes; +} + +function diffEntriesCode( + oldEntries: Record, + newEntries: Record, +): Map { + const changes = new Map(); + const renamed = new Set(); + + for (const [name, newVal] of Object.entries(newEntries)) { + if (!(name in oldEntries)) { + const oldName = Object.entries(oldEntries).find( + ([key, val]) => !(key in newEntries) && val.compSize === newVal.compSize, + )?.[0]; + + if (!oldName) { + changes.set(name, { type: DiffType.Added, size: newVal.size }); + } else { + renamed.add(oldName); + changes.set(name, { type: DiffType.Renamed, oldName, size: newVal.size }); + } + } + } + + for (const [name, oldVal] of Object.entries(oldEntries)) { + if (!(name in newEntries || renamed.has(name))) { + changes.set(name, { type: DiffType.Removed, size: oldVal.size }); + } + } + + return changes; +} + +async function diffRaw(channel: ChannelContext) { + if (!prevFiles.has("raw.json")) throw new Error("Missing prevFile: raw.json"); + + const oldRaw = JSON.parse(new TextDecoder().decode(prevFiles.get("raw.json"))) as RawColors; + const newRaw: RawColors = await Bun.file(join(channel.canvasDir, "raw.json")).json(); + + return diffEntries(oldRaw, newRaw, { + getSource: (_, val) => val, + isRename: (newKey, oldVal, newVal) => !(newKey in newRaw) && oldVal === newVal, + isChanged: (oldVal, newVal) => oldVal.toLowerCase() !== newVal.toLowerCase(), + }); +} + +function getSemanticLabel(semantic: Record) { + const light = semantic.light; + const dark = semantic.darker ?? semantic.dark; + if (dark && light) return `${light[0]}, ${dark[0]}`; + return Object.entries(semantic) + .map(([name, [color]]) => `${name} ${color}`) + .join(", "); +} + +async function diffSemantic(channel: ChannelContext) { + if (!prevFiles.has("semantic.json")) throw new Error("Missing prevFile: semantic.json"); + + const oldSemantic = JSON.parse(new TextDecoder().decode(prevFiles.get("semantic.json"))) as SemanticColors; + const newSemantic: SemanticColors = await Bun.file(join(channel.canvasDir, "semantic.json")).json(); + + const changes = diffEntries(oldSemantic, newSemantic, { + getSource: (_, val) => + Object.values(val) + .map(([color]) => color) + .join(","), + getLabel: (_, val) => getSemanticLabel(val), + isRename: (newKey, oldVal, newVal) => { + const themes = Object.keys(newVal).length; + return ( + !(newKey in newSemantic) && + Object.keys(oldVal).length === themes && + Object.entries(oldVal).every(([skey, sval]) => sval[0] === newVal[skey]?.[0]) + ); + }, + }); + + for (const [name, semantic] of Object.entries(newSemantic)) { + if (oldSemantic[name]) { + for (const [theme, [color]] of Object.entries(semantic)) { + if (!oldSemantic[name][theme]) { + changes.set(`${name}.${theme}`, { type: DiffType.Added, source: color }); + } else if (oldSemantic[name][theme][0].toLowerCase() !== color.toLowerCase()) { + changes.set(`${name}.${theme}`, { + type: DiffType.Changed, + source: color, + oldSource: oldSemantic[name][theme][0], + }); + } + } + } + } + + for (const [name, semantic] of Object.entries(oldSemantic)) { + if (name in newSemantic) { + for (const [theme, [source]] of Object.entries(semantic)) { + if (!newSemantic[name][theme]) { + changes.set(`${name}.${theme}`, { type: DiffType.Removed, source }); + } + } + } + } + + return changes; +} + +async function diffIcons(channel: ChannelContext) { + if (!prevFiles.has("icons.json")) throw new Error("Missing prevFile: icons.json"); + + const oldIcons = JSON.parse(new TextDecoder().decode(prevFiles.get("icons.json"))) as Icons; + const newIcons: Icons = await Bun.file(join(channel.canvasDir, "icons.json")).json(); + + const iconDir = { + old: join(channel.canvasDir, "oldicons"), + new: join(channel.canvasDir, "icons"), + }; + + return diffEntries(oldIcons, newIcons, { + getSource: (_, val) => join(iconDir.new, val.file), + getLabel: (_, val) => val.hash.slice(0, 8), + getOldSource: (_, val) => join(iconDir.old, val.file), + getOldLabel: (_, val) => val.hash.slice(0, 8), + isRename: (newKey, oldVal, newVal) => !(newKey in newIcons) && oldVal.hash === newVal.hash, + isChanged: (oldVal, newVal) => oldVal.hash !== newVal.hash, + }); +} + +function parseSource(text: string) { + const source = parseJsonl(text) as { file: string; size: number }[]; + return Object.fromEntries( + source.map(({ file, size }) => [file, { size, compSize: size - JSON.stringify(file).length }]), + ) as Record; +} + +async function diffCode(channel: ChannelContext) { + if (!prevFiles.has("source.jsonl")) throw new Error("Missing prevFile: source.jsonl"); + + const oldCode = parseSource(new TextDecoder().decode(prevFiles.get("source.jsonl"))); + const newCode = parseSource(await Bun.file(join(channel.dataDir, "source.jsonl")).text()); + + return diffEntriesCode(oldCode, newCode); +} + +export default async function diffs(channel: ChannelContext, progress: Progress) { + await fetchGitChanged(); + + const prefix = `${channel.channel}/`; + const changed = (name: string) => gitChanged.has(prefix + name); + const hasBaseline = (name: string) => prevFiles.has(name as typeof prevFiles extends Map ? K : never); + + if (!(changed("version.txt") || diffAnyway)) { + progress.update("diff", null); + progress.update("diff_raw", null); + progress.update("diff_semantic", null); + progress.update("diff_icons", null); + progress.update("diff_code", null); + return; + } + + const differs: Differs = { raw: new Map(), semantic: new Map(), icons: new Map(), code: new Map() }; + + await Promise.all([ + changed("raw.json") && hasBaseline("raw.json") + ? wrapPromise( + diffRaw(channel).then((x) => (differs.raw = x)), + progress, + "diff_raw", + ) + : progress.update("diff_raw", null), + changed("semantic.json") && hasBaseline("semantic.json") + ? wrapPromise( + diffSemantic(channel).then((x) => (differs.semantic = x)), + progress, + "diff_semantic", + ) + : progress.update("diff_semantic", null), + changed("icons.json") && hasBaseline("icons.json") + ? wrapPromise( + diffIcons(channel).then((x) => (differs.icons = x)), + progress, + "diff_icons", + ) + : progress.update("diff_icons", null), + changed("source.jsonl") && hasBaseline("source.jsonl") + ? wrapPromise( + diffCode(channel).then((x) => (differs.code = x)), + progress, + "diff_code", + ) + : progress.update("diff_code", null), + ]); + + progress.update("diff", true); + return differs; +} diff --git a/src/update/tasks/webhook.ts b/src/update/tasks/webhook.ts new file mode 100644 index 00000000000..fdaf69e8c93 --- /dev/null +++ b/src/update/tasks/webhook.ts @@ -0,0 +1,285 @@ +import { basename } from "node:path"; +import type { Canvas } from "@napi-rs/canvas"; +import { + type APIMessageTopLevelComponent, + ComponentType, + MessageFlags, + type RESTPostAPIWebhookWithTokenJSONBody, + RouteBases, + Routes, +} from "discord-api-types/v10"; +import { drawSections } from "../../canvas"; +import { makeSections } from "../../canvas/factory"; +import { type CodeDiff, type Diff, type Differs, DiffType } from "../../types"; +import { ctx } from "../channel"; +import { maxDiffChanges } from "../shared"; +import { formatBytes, sortEntries } from "../utils"; + +function fileBase(path: string, other?: string) { + if (!other) return `... ${basename(path)}`; + const paths = path.split("/"), + others = other.split("/"); + + let offset = 0; + while (offset < Math.min(paths.length, others.length) - 1 && paths[offset] === others[offset]) { + offset++; + } + + return `... ${paths.slice(offset).join("/")}`; +} + +function makeFooter(size: number, item: string) { + return `**${size.toLocaleString("en-US")}** ${item} change${size !== 1 ? "s" : ""} total`; +} + +function getThreshold(lines: string[]) { + return lines.some((x) => x.includes("\n")) ? Math.floor(maxDiffChanges / 2) : maxDiffChanges; +} + +function cap(lines: string[], item: string) { + const threshold = getThreshold(lines); + if (lines.length > threshold) + return [...lines.slice(0, threshold), `(+${(lines.length - threshold).toLocaleString("en-US")} ${item})`]; + return lines; +} + +interface FormattedDiff { + embed: string[]; + file: string; + capped: boolean; +} + +function formatDiff(diffs: Map): FormattedDiff { + const entries = sortEntries([...diffs.entries()]); + + const sections = { + Added: entries + .filter(([, diff]) => diff.type === DiffType.Added) + .map(([name, diff]) => { + if ("size" in diff) return `+ ${fileBase(name)} (${formatBytes(diff.size)})`; + const d = diff as Extract; + return `+ ${name}: ${d.label || d.source}`; + }), + Changed: entries + .filter(([, diff]) => diff.type === DiffType.Changed) + .map(([name, diff]) => { + const d = diff as Extract; + return `- ${name}: ${d.oldLabel || d.oldSource}\n+ ${name}: ${d.label || d.source}`; + }), + Renamed: entries + .filter(([, diff]) => diff.type === DiffType.Renamed) + .map(([name, diff]) => { + if ("size" in diff) { + const cd = diff as Extract; + return `- ${fileBase(cd.oldName, name)}\n+ ${fileBase(name, cd.oldName)}`; + } + const d = diff as Extract; + return `- ${d.oldName}\n+ ${name}`; + }), + Removed: entries + .filter(([, diff]) => diff.type === DiffType.Removed) + .map(([name, diff]) => { + if ("size" in diff) return `- ${fileBase(name)} (${formatBytes(diff.size)})`; + const d = diff as Extract; + return `- ${name}: ${d.label || d.source}`; + }), + }; + + const text = Object.entries(sections).filter(([, lines]) => lines.length); + return { + embed: text.map( + ([title, lines]) => `### ${title}\n\`\`\`diff\n${cap(lines, title.toLowerCase()).join("\n")}\`\`\``, + ), + file: text.map(([title, lines]) => `# ${title}\n\n${lines.join("\n")}`).join("\n\n"), + capped: text.some(([, lines]) => lines.length > getThreshold(lines)), + }; +} + +interface WebhookEmbed { + title: string; + body: FormattedDiff; + image?: Canvas; + footer: string; + key: string; +} + +async function sendWebhook(webhook: string, role: string, embeds: WebhookEmbed[], color?: number) { + const body = new FormData(); + const images = new Map(), + diffs = new Map(); + + let fi = 0; + for (let i = 0; i < embeds.length; i++) { + const embed = embeds[i]; + if (embed.image) { + const key = fi++; + const filename = `${embed.key}.png`; + images.set(i, filename); + + body.append(`files[${key}]`, new Blob([embed.image.toBuffer("image/png")], { type: "image/png" }), filename); + } + if (embed.body.capped) { + const key = fi++; + const filename = `${embed.key}-changes.diff`; + diffs.set(i, filename); + + body.append(`files[${key}]`, new Blob([embed.body.file], { type: "text/plain" }), filename); + } + } + + body.append( + "payload_json", + JSON.stringify({ + flags: MessageFlags.IsComponentsV2, + components: [ + { + type: ComponentType.TextDisplay, + content: `<@&${role}>`, + }, + ...embeds.map( + ({ title, body, image, footer }, i) => + ({ + type: ComponentType.Container, + accent_color: color, + components: [ + { + type: ComponentType.TextDisplay, + content: `## ${title}\n-# ${ctx().version} (${ctx().cuteVersion})`, + }, + ...body.embed.map((content) => ({ + type: ComponentType.TextDisplay, + content, + })), + body.capped && { + type: ComponentType.File, + file: { + url: `attachment://${diffs.get(i)}`, + }, + }, + body.capped && { + type: ComponentType.Separator, + }, + image && { + type: ComponentType.MediaGallery, + items: [ + { + media: { + url: `attachment://${images.get(i)}`, + }, + }, + ], + }, + { + type: ComponentType.TextDisplay, + content: footer, + }, + ].filter((x) => !!x), + }) as APIMessageTopLevelComponent, + ), + ], + allowed_mentions: process.env.NODE_ENV === "test" ? { parse: [] } : { roles: [role] }, + } as RESTPostAPIWebhookWithTokenJSONBody), + ); + + const url = new URL(webhook); + url.searchParams.set("wait", "true"); + url.searchParams.set("with_components", "true"); + + const res = await fetch(url, { + method: "POST", + body, + }); + + const data = await res.text(); + if (!res.ok) + throw new Error(`Failed to send webhook message with embeds ${embeds.map((x) => x.title).join(", ")}: ${data}`); + + const discordToken = process.env.bot_token; + if (!discordToken) return; + + try { + const message = JSON.parse(data); + + const route = `${RouteBases.api}/${Routes.channelMessageCrosspost(message.channel_id, message.id)}`; + await fetch(route, { + method: "POST", + headers: { + Authorization: discordToken, + }, + }); + } catch {} +} + +export async function webhook(diffs: Differs) { + if (diffs.raw.size || diffs.semantic.size) { + const wh = process.env.color_webhook; + if (!wh) { + console.warn("Missing color_webhook, skipping color webhook"); + } else { + await sendWebhook( + wh, + "1227327297795657850", + [ + diffs.raw.size && { + title: "Raw colors", + body: formatDiff(diffs.raw), + image: drawSections(await makeSections(diffs.raw)), + footer: makeFooter(diffs.raw.size, "raw color"), + key: "raw-colors", + }, + diffs.semantic.size && { + title: "Semantic colors", + body: formatDiff(diffs.semantic), + image: drawSections(await makeSections(diffs.semantic)), + footer: makeFooter(diffs.semantic.size, "semantic color"), + key: "semantic-colors", + }, + ].filter((x) => !!x) as WebhookEmbed[], + 0xd8ef9e, + ); + } + } + + if (diffs.icons.size) { + const wh = process.env.icons_webhook; + if (!wh) { + console.warn("Missing icons_webhook, skipping"); + } else { + await sendWebhook( + wh, + "1227327765079003217", + [ + { + title: "Icons", + body: formatDiff(diffs.icons), + image: drawSections(await makeSections(diffs.icons, true)), + footer: makeFooter(diffs.icons.size, "icon"), + key: "icons", + }, + ], + 0xf4b8f7, + ); + } + } + + if (diffs.code.size) { + const wh = process.env.code_webhook; + if (!wh) { + console.warn("Missing code_webhook, skipping"); + return; + } + await sendWebhook( + wh, + "1233861867059941387", + [ + { + title: "Code", + body: formatDiff(diffs.code), + footer: makeFooter(diffs.code.size, "code"), + key: "code", + }, + ], + 0x86faf3, + ); + } +}