diff --git a/src/update/runner.ts b/src/update/runner.ts index 7a4072ee866..9a130873e55 100644 --- a/src/update/runner.ts +++ b/src/update/runner.ts @@ -6,6 +6,7 @@ import { log, makeProgress, wrapPromise } from "./progress"; import { isMock, isQuiet, oprevFiles, prevFiles } from "./shared"; import codeTask from "./tasks/code"; import colorsTask from "./tasks/colors"; +import iconsTask from "./tasks/icons"; import decompile from "./tasks/decompile"; import diffs from "./tasks/diffs"; import { formatError, join } from "./utils"; @@ -84,20 +85,12 @@ export async function runTasks(channel: ChannelContext) { const code = (await Bun.file(channel.codePath).text()).replace(/\r/g, "").split("\n"); - await wrapPromise(codeTask(channel, progress, code), progress, "code"); + await Promise.allSettled([ + wrapPromise(codeTask(channel, progress, code), progress, "code"), + wrapPromise(colorsTask(channel, code), progress, "colors"), + wrapPromise(iconsTask(channel, progress, code), progress, "icons"), + ]); if (progress.someFailed("code")) throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code")}`); - // colors is non-critical, run in background - wrapPromise(colorsTask(channel, code), progress, "colors").catch((e) => { - progress.update("colors", false, String(e)); - console.warn("Colors task failed (non-critical):", e); - }); - // icons disabled for now (6h hang due to parseAssets), will be fixed separately - // wrapPromise(iconsTask(progress, code), progress, "icons").catch((e) => { - // progress.update("icons", false, String(e)); - // console.warn("Icons task failed (non-critical):", e); - // }); - progress.update("icons", null); - progress.update("diff_icons", null); if (gzipDone) { try { diff --git a/src/update/tasks/colors.ts b/src/update/tasks/colors.ts index 349c8435e35..a44193a1cea 100644 --- a/src/update/tasks/colors.ts +++ b/src/update/tasks/colors.ts @@ -1,4 +1,3 @@ -import { runInNewContext } from "node:vm"; import Color, { type ColorInstance } from "color"; import type { RawColors, SemanticColors } from "../../types"; import type { ChannelContext } from "../channel"; @@ -9,56 +8,66 @@ function hex(color: ColorInstance) { return (color.alpha() === 1 ? color.hex() : color.hexa()).toLowerCase(); } -// Support both old python decompiler (rX[rY] bug) and new Rust -function fixHermesDecIndex(code: string) { - return code.replace(/r(\d+)\[(\d+)\] =/g, "r$1[r$2] ="); -} - -function findModuleSnippet(code: string[], importPath: string): string | null { - const text = code.join("\n"); - const parts = text.split(/(?=\/\/ === Module \d+:)/); - for (const part of parts) if (part.includes(importPath)) return part; - return null; +function findModuleSnippet(code: string[] | string, importPath: string): string | null { + const text = Array.isArray(code) ? code.join("\n") : code; + const targetIdx = text.indexOf(importPath); + if (targetIdx === -1) return null; + const startIdx = text.lastIndexOf("// === Module ", targetIdx); + const fromIdx = startIdx === -1 ? 0 : startIdx; + const nextIdx = text.indexOf("// === Module ", targetIdx + importPath.length); + const toIdx = nextIdx === -1 ? text.length : nextIdx; + return text.slice(fromIdx, toIdx); } function evalRawModule(snippet: string): any { - let code = snippet; - code = code.replace(/^import set from "set".*$/m, "const __importSet = { fileFinishedImporting: () => {} };"); - code = code.replace(/^import.*$/gm, ""); - code = code.replace(/set\.fileFinishedImporting/g, "__importSet.fileFinishedImporting"); - code = code.replace(/export\s+const\s+_private/, "const _private"); - code = code + "\n; _private;"; - const result = runInNewContext(code, {}); - if (!result || typeof result !== "object" || !result.RawColors || typeof result.RawColors !== "object") { + let code = snippet.replace(/^import.*$/gm, ""); + const preamble = ` + const size = { fileFinishedImporting: () => {} }; + const set = size; + \n`; + code = preamble + code; + code = code.replace(/export\s+(?:const|var|let)\s+_private/, "const _private"); + code = code + "\nreturn _private;"; + try { + const fn = new Function(code); + const result = fn(); + if (!result || typeof result !== "object" || !result.RawColors || typeof result.RawColors !== "object") { + throw new Error("RawColors object not found in evaluated module."); + } + return result; + } catch (e: any) { throw new Error( - "VM evaluation failed: RawColors not found or invalid shape. The regex transforms may not match the current decompiler output.", + `VM evaluation failed for RawColors: ${e.message}. Decompiler output may have changed structure.`, ); } - return result; } function evalSemanticModule(snippet: string): any { - let code = snippet; - code = code.replace(/^import set from "set".*$/m, "const __importSet = { fileFinishedImporting: () => {} };"); - code = code.replace( - /^import items from "items".*$/m, - "const items = { _private: { Themes: new Proxy({}, { get(_, k){ return k.toString().toLowerCase(); } }) } };", - ); - code = code.replace(/^import.*$/gm, ""); - code = code.replace(/set\.fileFinishedImporting/g, "__importSet.fileFinishedImporting"); - code = code.replace(/export\s+const\s+_private/, "const _private"); - code = code + "\n; _private;"; - code = fixHermesDecIndex(code); - const result = runInNewContext(code, {}); - if (!result || typeof result !== "object" || !result.SemanticColors || typeof result.SemanticColors !== "object") { + let code = snippet.replace(/^import.*$/gm, ""); + const preamble = ` + const size = { fileFinishedImporting: () => {} }; + const set = size; + const ThemeTypes = { _private: { Themes: new Proxy({}, { get(_, k) { return k.toString().toLowerCase(); } }) } }; + const items = ThemeTypes; + \n`; + code = preamble + code; + code = code.replace(/export\s+(?:const|var|let)\s+_private/, "const _private"); + code = code + "\nreturn _private;"; + try { + const fn = new Function(code); + const result = fn(); + if (!result || typeof result !== "object" || !result.SemanticColors || typeof result.SemanticColors !== "object") { + throw new Error("SemanticColors object not found in evaluated module."); + } + return result; + } catch (e: any) { throw new Error( - "VM evaluation failed: SemanticColors not found or invalid shape. The regex transforms may not match the current decompiler output.", + `VM evaluation failed for SemanticColors: ${e.message}. Decompiler output may have changed structure.`, ); } - return result; } -export function getInternalRawColors(code: string[]) { +export function getInternalRawColors(code: string[] | string) { const snippet = findModuleSnippet(code, "raw-color-definitions.tsx"); if (!snippet) throw new Error("Cannot find raw-color-definitions module"); const mod = evalRawModule(snippet); @@ -69,7 +78,7 @@ export function getInternalRawColors(code: string[]) { return raw; } -export function getInternalSemanticColors(code: string[], raw: Record) { +export function getInternalSemanticColors(code: string[] | string, raw: Record) { const snippet = findModuleSnippet(code, "native/generated-definitions"); if (!snippet) throw new Error("Cannot find semantic definitions module"); const mod = evalSemanticModule(snippet); @@ -97,7 +106,7 @@ export function convertSimpleSemantic(semantic: SemanticColors) { return simpleSemantic; } -export default async function colors(channel: ChannelContext, code: string[]) { +export default async function colors(channel: ChannelContext, code: string[] | string) { const raw = getInternalRawColors(code); await Bun.write(join(channel.canvasDir, "raw.json"), JSON.stringify(sortObj(raw), null, 4)); diff --git a/src/update/tasks/icons.ts b/src/update/tasks/icons.ts index fb2895cc016..c63a7ae2848 100644 --- a/src/update/tasks/icons.ts +++ b/src/update/tasks/icons.ts @@ -1,17 +1,16 @@ import { copyFile, mkdir, readdir, rename } from "node:fs/promises"; import { basename } from "node:path"; -import { runInNewContext } from "node:vm"; import type { Icons } from "../../types"; import type { ChannelContext } from "../channel"; import { commit } from "../git"; import type { Progress } from "../progress"; import { apkSplits } from "../shared"; import { discordPath, join, listRequiredDirs, sortObj } from "../utils"; +import { exists } from "node:fs/promises"; -const infoObjRegex = /({.+?})/; -const scalesArrayRegex = /(\[.+?\])/; +const singleLineAssetRegex = /\.registerAsset\((\{.+?\})\)/; -export async function parseAssets(channel: ChannelContext, code: string[]) { +export async function parseAssets(channel: ChannelContext, code: string[] | string) { const retrievedAssets: { httpServerLocation: string; width: number; @@ -22,40 +21,65 @@ export async function parseAssets(channel: ChannelContext, code: string[]) { type: "png" | "svg" | "lottie"; }[] = []; - for (let i = 0; i < code.length; i++) { - const line = code[i], - infoLine = code[i + 1], - scalesLine = code[i + 2]; + const lines = Array.isArray(code) ? code : code.split("\n"); - if (line.includes(".registerAsset") && infoLine?.includes("'httpServerLocation'") && scalesLine?.includes("[")) { - const infoText = infoLine.match(infoObjRegex)?.[1]; - if (!infoText) throw new Error(`Failed to find infoText for ${infoLine} (line ${i + 1})`); + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (!line.includes(".registerAsset")) continue; - const scalesText = scalesLine.match(scalesArrayRegex)?.[1]; - if (!scalesText) throw new Error(`Failed to find scalesText for ${scalesLine} (line ${i + 2})`); + // Pattern 1: Modern single-line format (hermes-decomp / Rust) + const singleMatch = line.match(singleLineAssetRegex); + if (singleMatch) { + try { + const fn = new Function(`return (${singleMatch[1]})`); + const info = fn(); + if ( + info && + ["httpServerLocation", "hash", "name", "type"].every((x) => x in info) && + ["svg", "png", "lottie"].includes(info.type) && + !info.httpServerLocation.includes("node_modules/.pnpm") + ) { + retrievedAssets.push(info); + continue; + } + } catch {} + } - const info = runInNewContext(`(${infoText})`); - if (typeof info !== "object") continue; - - const scales = runInNewContext(`(${scalesText})`); - if (!Array.isArray(scales)) continue; - - info.scales = scales; - - if ( - ["httpServerLocation", "hash", "name", "type"].every((x) => x in info) && - ["svg", "png", "lottie"].includes(info.type) && - !info.httpServerLocation.includes("node_modules/.pnpm") - ) { - retrievedAssets.push(info); - } + // Pattern 2: Legacy multi-line format + const infoLine = lines[i + 1]; + const scalesLine = lines[i + 2]; + if (infoLine?.includes("'httpServerLocation'") && scalesLine?.includes("[")) { + try { + const infoText = infoLine.match(/({.+?})/)?.[1]; + const scalesText = scalesLine.match(/(\[.+?\])/)?.[1]; + if (infoText && scalesText) { + const info = new Function(`return (${infoText})`)(); + const scales = new Function(`return (${scalesText})`)(); + if (info && typeof info === "object" && Array.isArray(scales)) { + info.scales = scales; + if ( + ["httpServerLocation", "hash", "name", "type"].every((x) => x in info) && + ["svg", "png", "lottie"].includes(info.type) && + !info.httpServerLocation.includes("node_modules/.pnpm") + ) { + retrievedAssets.push(info); + } + } + } + } catch {} } } const apkPaths = new Map(); for (const split of apkSplits) { const folder = join(channel.apksFolder, split); - for (const path of await readdir(folder, { recursive: true })) apkPaths.set(basename(path), join(folder, path)); + if (await exists(folder)) { + try { + for (const path of await readdir(folder, { recursive: true })) { + apkPaths.set(basename(path), join(folder, path)); + } + } catch {} + } } const iconsDir = join(channel.canvasDir, "icons"); @@ -93,7 +117,7 @@ export async function parseAssets(channel: ChannelContext, code: string[]) { return { icons, toCopy }; } -export default async function icons(channel: ChannelContext, progress: Progress, code: string[]) { +export default async function icons(channel: ChannelContext, progress: Progress, code: string[] | string) { progress.start("icons_getting"); const { icons, toCopy } = await parseAssets(channel, code); @@ -108,7 +132,21 @@ export default async function icons(channel: ChannelContext, progress: Progress, const dirs = listRequiredDirs(toCopy.map((x) => x.to)); await Promise.all(dirs.map((dir) => mkdir(dir, { recursive: true }))); - await Promise.all(toCopy.map(({ from, to }) => copyFile(from, to))); + + // Batch file copying in chunks of 50 to avoid file descriptor starvation or timeouts + const BATCH_SIZE = 50; + for (let i = 0; i < toCopy.length; i += BATCH_SIZE) { + const batch = toCopy.slice(i, i + BATCH_SIZE); + await Promise.all( + batch.map(async ({ from, to }) => { + try { + if (await exists(from)) { + await copyFile(from, to); + } + } catch {} + }), + ); + } await commit(["icons.json", "icons"], `chore: update icons for ${channel.cuteVersion}`, channel.canvasDir); progress.update("icons_copying", true);