diff --git a/src/update/runner.ts b/src/update/runner.ts index 7a1af8885d1..0d7d8919782 100644 --- a/src/update/runner.ts +++ b/src/update/runner.ts @@ -8,6 +8,7 @@ import codeTask from "./tasks/code"; import colorsTask from "./tasks/colors"; import decompile from "./tasks/decompile"; import diffs from "./tasks/diffs"; +import iconsTask from "./tasks/icons"; import { formatError, handleShellErr, join } from "./utils"; export async function runTasks() { @@ -28,10 +29,14 @@ export async function runTasks() { colors: "Colors", colors_getting: "Extracting colors", colors_pushing: "Committing colors", + icons: "Icon parser", + icons_getting: "Writing icons.json", + icons_copying: "Copying images", diff: "Diffs", diff_code: "Diffing code", diff_raw: "Diffing raw colors", diff_semantic: "Diffing semantic colors", + diff_icons: "Diffing icons", }, true, ); @@ -82,18 +87,13 @@ export async function runTasks() { const code = (await Bun.file(codePath).text()).replace(/\r/g, "").split("\n"); - await wrapPromise(codeTask(progress, code), progress, "code"); - if (progress.someFailed("code")) - throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code")}`); - - // colors (canvas) - uses same code array - try { - await wrapPromise(colorsTask(code), progress, "colors"); - } catch (e: any) { - progress.update("colors", false, formatError(e)); - // don't fail whole run if colors fails, just log - console.error("Colors task failed:", formatError(e)); - } + await Promise.allSettled([ + wrapPromise(codeTask(progress, code), progress, "code"), + wrapPromise(colorsTask(code), progress, "colors"), + wrapPromise(iconsTask(progress, code), progress, "icons"), + ]); + if (progress.someFailed("code", "colors", "icons")) + throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code", "colors", "icons")}`); while (!progress.isFinished("decompile_gzip")) { await Bun.sleep(1000); diff --git a/src/update/shared.ts b/src/update/shared.ts index e974faae91b..e0f5f0f4c9a 100644 --- a/src/update/shared.ts +++ b/src/update/shared.ts @@ -10,7 +10,7 @@ export const modulesPath = join(workFolder, "modules"); export const modulePathsDest = join("..", "data", "module-paths.json"); export const apkSplits = ["base", "config.hdpi", "config.xxhdpi"] as const; -export const oprevFiles = ["source.jsonl", "raw.json", "semantic.json", "semantic_simple.json"] as const; +export const oprevFiles = ["source.jsonl", "raw.json", "semantic.json", "semantic_simple.json", "icons.json"] as const; export const prevFiles = new Map<(typeof oprevFiles)[number], ArrayBuffer>(); export const version = await Bun.file("../data/version.txt").text(); diff --git a/src/update/tasks/diffs.ts b/src/update/tasks/diffs.ts index a04a8716aea..9827287ac8d 100644 --- a/src/update/tasks/diffs.ts +++ b/src/update/tasks/diffs.ts @@ -3,6 +3,7 @@ import { type Diff, type Differs, DiffType, + type Icons, type RawColors, type SemanticColors, } from "../../types"; @@ -138,6 +139,60 @@ async function diffSemantic() { return changes; } +async function diffIcons() { + if (!prevFiles.has("icons.json")) { + throw new Error("Missing prevFile: icons.json"); + } + + const oldIcons = JSON.parse(new TextDecoder().decode(prevFiles.get("icons.json"))) as Icons; + const newIcons: Icons = await Bun.file(join("../canvas", "icons.json")).json(); + + const iconDir = { + old: join("../canvas", "oldicons"), + new: join("../canvas", "icons"), + }; + + const renamed = new Set(); + const changes = new Map(); + for (const [name, icon] of Object.entries(newIcons)) { + const label = icon.hash.slice(0, 8); + if (!oldIcons[name]) { + const oldName = Object.entries(oldIcons).find(([key, val]) => !newIcons[key] && val.hash === icon.hash)?.[0]; + + if (!oldName) + changes.set(name, { + type: DiffType.Added, + source: join(iconDir.new, icon.file), + label, + }); + else { + renamed.add(oldName); + changes.set(name, { + type: DiffType.Renamed, + oldName, + source: join(iconDir.new, icon.file), + }); + } + } else if (oldIcons[name].hash !== icon.hash) + changes.set(name, { + type: DiffType.Changed, + source: join(iconDir.new, icon.file), + label, + oldSource: join(iconDir.old, oldIcons[name].file), + oldLabel: oldIcons[name].hash.slice(0, 8), + }); + } + for (const [name, icon] of Object.entries(oldIcons)) + if (!(newIcons[name] || renamed.has(name))) + changes.set(name, { + type: DiffType.Removed, + source: join(iconDir.old, icon.file), + label: icon.hash.slice(0, 8), + }); + + return changes; +} + function parseSource(text: string) { const source = parseJsonl(text) as { file: string; size: number }[]; return Object.fromEntries( @@ -191,6 +246,7 @@ export default async function diffs(progress: Progress) { progress.update("diff", null); progress.update("diff_raw", null); progress.update("diff_semantic", null); + progress.update("diff_icons", null); progress.update("diff_code", null); return; } @@ -212,6 +268,13 @@ export default async function diffs(progress: Progress) { "diff_semantic", ) : progress.update("diff_semantic", null), + gitChanged.has("icons.json") + ? wrapPromise( + diffIcons().then((x) => (differs.icons = x)), + progress, + "diff_icons", + ) + : progress.update("diff_icons", null), gitChanged.has("source.jsonl") ? wrapPromise( diffCode().then((x) => (differs.code = x)), diff --git a/src/update/tasks/icons.ts b/src/update/tasks/icons.ts new file mode 100644 index 00000000000..d8e6376fc31 --- /dev/null +++ b/src/update/tasks/icons.ts @@ -0,0 +1,115 @@ +import { copyFile, mkdir, readdir, rename } from "node:fs/promises"; +import { basename } from "node:path"; +import { runInNewContext } from "node:vm"; +import type { Icons } from "../../types"; +import { commit } from "../git"; +import type { Progress } from "../progress"; +import { apkSplits, apksFolder, cuteVersion } from "../shared"; +import { discordPath, join, listRequiredDirs, sortObj } from "../utils"; + +const infoObjRegex = /({.+?})/; +const scalesArrayRegex = /(\[.+?\])/; + +// canvas branch stores icons +const iconsJsonPath = "../canvas/icons.json"; +const iconsDir = "../canvas/icons"; +const oldIconsDir = "../canvas/oldicons"; + +export async function parseAssets(code: string[]) { + const retrievedAssets: { + httpServerLocation: string; + width: number; + height: number; + scales: number[]; + hash: string; + name: string; + type: "png" | "svg" | "lottie"; + }[] = []; + + for (let i = 0; i < code.length; i++) { + const line = code[i], + infoLine = code[i + 1], + scalesLine = code[i + 2]; + + if (line.includes(".registerAsset") && infoLine?.includes("'httpServerLocation'") && scalesLine?.includes("[")) { + const infoText = infoLine.match(infoObjRegex)?.[1]; + if (!infoText) throw new Error(`Failed to find infoText for ${infoLine} (line ${i + 1})`); + + const scalesText = scalesLine.match(scalesArrayRegex)?.[1]; + if (!scalesText) throw new Error(`Failed to find scalesText for ${scalesLine} (line ${i + 2})`); + + const info = runInNewContext(`(${infoText})`); + if (typeof info !== "object") continue; + + const scales = runInNewContext(`(${scalesText})`); + if (!Array.isArray(scales)) continue; + + info.scales = scales; + + if ( + ["httpServerLocation", "hash", "name", "type"].every((x) => x in info) && + ["svg", "png", "lottie"].includes(info.type) && + !info.httpServerLocation.includes("node_modules/.pnpm") + ) { + retrievedAssets.push(info); + } + } + } + + const apkPaths = new Map(); + for (const split of apkSplits) { + const folder = join(apksFolder, split); + for (const path of await readdir(folder, { recursive: true })) apkPaths.set(basename(path), join(folder, path)); + } + + const icons: Icons = {}; + const toCopy: { from: string; to: string }[] = []; + for (const asset of retrievedAssets) { + const baseName = [...asset.httpServerLocation.split("/").slice(2), asset.name] + .join("_") + .toLowerCase() + .replace(/\W+/g, ""); + const path = `${baseName}.${asset.type}`; + + const apkPath = apkPaths.get(path); + if (apkPath) { + const realPath = join( + discordPath(asset.httpServerLocation.split("/").slice(2).join("/")), + `${asset.name}.${asset.type}`, + ); + + icons[asset.name] = { + file: realPath, + hash: asset.hash, + scales: asset.scales, + width: asset.width ?? null, + height: asset.height ?? null, + }; + toCopy.push({ + from: apkPath, + to: join(iconsDir, realPath), + }); + } + } + + return { icons, toCopy }; +} + +export default async function icons(progress: Progress, code: string[]) { + progress.start("icons_getting"); + + const { icons, toCopy } = await parseAssets(code); + await Bun.write(iconsJsonPath, JSON.stringify(sortObj(icons), undefined, 4)); + progress.update("icons_getting", true); + + progress.start("icons_copying"); + + await rename(iconsDir, oldIconsDir).catch(() => {}); + const dirs = listRequiredDirs(toCopy.map((x) => x.to)); + + await Promise.all(dirs.map((dir) => mkdir(dir, { recursive: true }))); + await Promise.all(toCopy.map(({ from, to }) => copyFile(from, to))); + + await commit(["icons.json", "icons"], `chore: update icons for ${cuteVersion}`, "../canvas"); + progress.update("icons_copying", true); +}