fix: restore fast non-blocking colors and icons tasks
This commit is contained in:
Rosie Val
parent
4ffe412464
commit
4a65973d93
3 changed files with 123 additions and 83 deletions
|
|
@ -6,6 +6,7 @@ import { log, makeProgress, wrapPromise } from "./progress";
|
|||
import { isMock, isQuiet, oprevFiles, prevFiles } from "./shared";
|
||||
import codeTask from "./tasks/code";
|
||||
import colorsTask from "./tasks/colors";
|
||||
import iconsTask from "./tasks/icons";
|
||||
import decompile from "./tasks/decompile";
|
||||
import diffs from "./tasks/diffs";
|
||||
import { formatError, join } from "./utils";
|
||||
|
|
@ -84,20 +85,12 @@ export async function runTasks(channel: ChannelContext) {
|
|||
|
||||
const code = (await Bun.file(channel.codePath).text()).replace(/\r/g, "").split("\n");
|
||||
|
||||
await wrapPromise(codeTask(channel, progress, code), progress, "code");
|
||||
await Promise.allSettled([
|
||||
wrapPromise(codeTask(channel, progress, code), progress, "code"),
|
||||
wrapPromise(colorsTask(channel, code), progress, "colors"),
|
||||
wrapPromise(iconsTask(channel, progress, code), progress, "icons"),
|
||||
]);
|
||||
if (progress.someFailed("code")) throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code")}`);
|
||||
// colors is non-critical, run in background
|
||||
wrapPromise(colorsTask(channel, code), progress, "colors").catch((e) => {
|
||||
progress.update("colors", false, String(e));
|
||||
console.warn("Colors task failed (non-critical):", e);
|
||||
});
|
||||
// icons disabled for now (6h hang due to parseAssets), will be fixed separately
|
||||
// wrapPromise(iconsTask(progress, code), progress, "icons").catch((e) => {
|
||||
// progress.update("icons", false, String(e));
|
||||
// console.warn("Icons task failed (non-critical):", e);
|
||||
// });
|
||||
progress.update("icons", null);
|
||||
progress.update("diff_icons", null);
|
||||
|
||||
if (gzipDone) {
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
import { runInNewContext } from "node:vm";
|
||||
import Color, { type ColorInstance } from "color";
|
||||
import type { RawColors, SemanticColors } from "../../types";
|
||||
import type { ChannelContext } from "../channel";
|
||||
|
|
@ -9,56 +8,66 @@ function hex(color: ColorInstance) {
|
|||
return (color.alpha() === 1 ? color.hex() : color.hexa()).toLowerCase();
|
||||
}
|
||||
|
||||
// Support both old python decompiler (rX[rY] bug) and new Rust
|
||||
function fixHermesDecIndex(code: string) {
|
||||
return code.replace(/r(\d+)\[(\d+)\] =/g, "r$1[r$2] =");
|
||||
}
|
||||
|
||||
function findModuleSnippet(code: string[], importPath: string): string | null {
|
||||
const text = code.join("\n");
|
||||
const parts = text.split(/(?=\/\/ === Module \d+:)/);
|
||||
for (const part of parts) if (part.includes(importPath)) return part;
|
||||
return null;
|
||||
function findModuleSnippet(code: string[] | string, importPath: string): string | null {
|
||||
const text = Array.isArray(code) ? code.join("\n") : code;
|
||||
const targetIdx = text.indexOf(importPath);
|
||||
if (targetIdx === -1) return null;
|
||||
const startIdx = text.lastIndexOf("// === Module ", targetIdx);
|
||||
const fromIdx = startIdx === -1 ? 0 : startIdx;
|
||||
const nextIdx = text.indexOf("// === Module ", targetIdx + importPath.length);
|
||||
const toIdx = nextIdx === -1 ? text.length : nextIdx;
|
||||
return text.slice(fromIdx, toIdx);
|
||||
}
|
||||
|
||||
function evalRawModule(snippet: string): any {
|
||||
let code = snippet;
|
||||
code = code.replace(/^import set from "set".*$/m, "const __importSet = { fileFinishedImporting: () => {} };");
|
||||
code = code.replace(/^import.*$/gm, "");
|
||||
code = code.replace(/set\.fileFinishedImporting/g, "__importSet.fileFinishedImporting");
|
||||
code = code.replace(/export\s+const\s+_private/, "const _private");
|
||||
code = code + "\n; _private;";
|
||||
const result = runInNewContext(code, {});
|
||||
if (!result || typeof result !== "object" || !result.RawColors || typeof result.RawColors !== "object") {
|
||||
let code = snippet.replace(/^import.*$/gm, "");
|
||||
const preamble = `
|
||||
const size = { fileFinishedImporting: () => {} };
|
||||
const set = size;
|
||||
\n`;
|
||||
code = preamble + code;
|
||||
code = code.replace(/export\s+(?:const|var|let)\s+_private/, "const _private");
|
||||
code = code + "\nreturn _private;";
|
||||
try {
|
||||
const fn = new Function(code);
|
||||
const result = fn();
|
||||
if (!result || typeof result !== "object" || !result.RawColors || typeof result.RawColors !== "object") {
|
||||
throw new Error("RawColors object not found in evaluated module.");
|
||||
}
|
||||
return result;
|
||||
} catch (e: any) {
|
||||
throw new Error(
|
||||
"VM evaluation failed: RawColors not found or invalid shape. The regex transforms may not match the current decompiler output.",
|
||||
`VM evaluation failed for RawColors: ${e.message}. Decompiler output may have changed structure.`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function evalSemanticModule(snippet: string): any {
|
||||
let code = snippet;
|
||||
code = code.replace(/^import set from "set".*$/m, "const __importSet = { fileFinishedImporting: () => {} };");
|
||||
code = code.replace(
|
||||
/^import items from "items".*$/m,
|
||||
"const items = { _private: { Themes: new Proxy({}, { get(_, k){ return k.toString().toLowerCase(); } }) } };",
|
||||
);
|
||||
code = code.replace(/^import.*$/gm, "");
|
||||
code = code.replace(/set\.fileFinishedImporting/g, "__importSet.fileFinishedImporting");
|
||||
code = code.replace(/export\s+const\s+_private/, "const _private");
|
||||
code = code + "\n; _private;";
|
||||
code = fixHermesDecIndex(code);
|
||||
const result = runInNewContext(code, {});
|
||||
if (!result || typeof result !== "object" || !result.SemanticColors || typeof result.SemanticColors !== "object") {
|
||||
let code = snippet.replace(/^import.*$/gm, "");
|
||||
const preamble = `
|
||||
const size = { fileFinishedImporting: () => {} };
|
||||
const set = size;
|
||||
const ThemeTypes = { _private: { Themes: new Proxy({}, { get(_, k) { return k.toString().toLowerCase(); } }) } };
|
||||
const items = ThemeTypes;
|
||||
\n`;
|
||||
code = preamble + code;
|
||||
code = code.replace(/export\s+(?:const|var|let)\s+_private/, "const _private");
|
||||
code = code + "\nreturn _private;";
|
||||
try {
|
||||
const fn = new Function(code);
|
||||
const result = fn();
|
||||
if (!result || typeof result !== "object" || !result.SemanticColors || typeof result.SemanticColors !== "object") {
|
||||
throw new Error("SemanticColors object not found in evaluated module.");
|
||||
}
|
||||
return result;
|
||||
} catch (e: any) {
|
||||
throw new Error(
|
||||
"VM evaluation failed: SemanticColors not found or invalid shape. The regex transforms may not match the current decompiler output.",
|
||||
`VM evaluation failed for SemanticColors: ${e.message}. Decompiler output may have changed structure.`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export function getInternalRawColors(code: string[]) {
|
||||
export function getInternalRawColors(code: string[] | string) {
|
||||
const snippet = findModuleSnippet(code, "raw-color-definitions.tsx");
|
||||
if (!snippet) throw new Error("Cannot find raw-color-definitions module");
|
||||
const mod = evalRawModule(snippet);
|
||||
|
|
@ -69,7 +78,7 @@ export function getInternalRawColors(code: string[]) {
|
|||
return raw;
|
||||
}
|
||||
|
||||
export function getInternalSemanticColors(code: string[], raw: Record<string, string>) {
|
||||
export function getInternalSemanticColors(code: string[] | string, raw: Record<string, string>) {
|
||||
const snippet = findModuleSnippet(code, "native/generated-definitions");
|
||||
if (!snippet) throw new Error("Cannot find semantic definitions module");
|
||||
const mod = evalSemanticModule(snippet);
|
||||
|
|
@ -97,7 +106,7 @@ export function convertSimpleSemantic(semantic: SemanticColors) {
|
|||
return simpleSemantic;
|
||||
}
|
||||
|
||||
export default async function colors(channel: ChannelContext, code: string[]) {
|
||||
export default async function colors(channel: ChannelContext, code: string[] | string) {
|
||||
const raw = getInternalRawColors(code);
|
||||
await Bun.write(join(channel.canvasDir, "raw.json"), JSON.stringify(sortObj(raw), null, 4));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +1,16 @@
|
|||
import { copyFile, mkdir, readdir, rename } from "node:fs/promises";
|
||||
import { basename } from "node:path";
|
||||
import { runInNewContext } from "node:vm";
|
||||
import type { Icons } from "../../types";
|
||||
import type { ChannelContext } from "../channel";
|
||||
import { commit } from "../git";
|
||||
import type { Progress } from "../progress";
|
||||
import { apkSplits } from "../shared";
|
||||
import { discordPath, join, listRequiredDirs, sortObj } from "../utils";
|
||||
import { exists } from "node:fs/promises";
|
||||
|
||||
const infoObjRegex = /({.+?})/;
|
||||
const scalesArrayRegex = /(\[.+?\])/;
|
||||
const singleLineAssetRegex = /\.registerAsset\((\{.+?\})\)/;
|
||||
|
||||
export async function parseAssets(channel: ChannelContext, code: string[]) {
|
||||
export async function parseAssets(channel: ChannelContext, code: string[] | string) {
|
||||
const retrievedAssets: {
|
||||
httpServerLocation: string;
|
||||
width: number;
|
||||
|
|
@ -22,40 +21,65 @@ export async function parseAssets(channel: ChannelContext, code: string[]) {
|
|||
type: "png" | "svg" | "lottie";
|
||||
}[] = [];
|
||||
|
||||
for (let i = 0; i < code.length; i++) {
|
||||
const line = code[i],
|
||||
infoLine = code[i + 1],
|
||||
scalesLine = code[i + 2];
|
||||
const lines = Array.isArray(code) ? code : code.split("\n");
|
||||
|
||||
if (line.includes(".registerAsset") && infoLine?.includes("'httpServerLocation'") && scalesLine?.includes("[")) {
|
||||
const infoText = infoLine.match(infoObjRegex)?.[1];
|
||||
if (!infoText) throw new Error(`Failed to find infoText for ${infoLine} (line ${i + 1})`);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (!line.includes(".registerAsset")) continue;
|
||||
|
||||
const scalesText = scalesLine.match(scalesArrayRegex)?.[1];
|
||||
if (!scalesText) throw new Error(`Failed to find scalesText for ${scalesLine} (line ${i + 2})`);
|
||||
// Pattern 1: Modern single-line format (hermes-decomp / Rust)
|
||||
const singleMatch = line.match(singleLineAssetRegex);
|
||||
if (singleMatch) {
|
||||
try {
|
||||
const fn = new Function(`return (${singleMatch[1]})`);
|
||||
const info = fn();
|
||||
if (
|
||||
info &&
|
||||
["httpServerLocation", "hash", "name", "type"].every((x) => x in info) &&
|
||||
["svg", "png", "lottie"].includes(info.type) &&
|
||||
!info.httpServerLocation.includes("node_modules/.pnpm")
|
||||
) {
|
||||
retrievedAssets.push(info);
|
||||
continue;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const info = runInNewContext(`(${infoText})`);
|
||||
if (typeof info !== "object") continue;
|
||||
|
||||
const scales = runInNewContext(`(${scalesText})`);
|
||||
if (!Array.isArray(scales)) continue;
|
||||
|
||||
info.scales = scales;
|
||||
|
||||
if (
|
||||
["httpServerLocation", "hash", "name", "type"].every((x) => x in info) &&
|
||||
["svg", "png", "lottie"].includes(info.type) &&
|
||||
!info.httpServerLocation.includes("node_modules/.pnpm")
|
||||
) {
|
||||
retrievedAssets.push(info);
|
||||
}
|
||||
// Pattern 2: Legacy multi-line format
|
||||
const infoLine = lines[i + 1];
|
||||
const scalesLine = lines[i + 2];
|
||||
if (infoLine?.includes("'httpServerLocation'") && scalesLine?.includes("[")) {
|
||||
try {
|
||||
const infoText = infoLine.match(/({.+?})/)?.[1];
|
||||
const scalesText = scalesLine.match(/(\[.+?\])/)?.[1];
|
||||
if (infoText && scalesText) {
|
||||
const info = new Function(`return (${infoText})`)();
|
||||
const scales = new Function(`return (${scalesText})`)();
|
||||
if (info && typeof info === "object" && Array.isArray(scales)) {
|
||||
info.scales = scales;
|
||||
if (
|
||||
["httpServerLocation", "hash", "name", "type"].every((x) => x in info) &&
|
||||
["svg", "png", "lottie"].includes(info.type) &&
|
||||
!info.httpServerLocation.includes("node_modules/.pnpm")
|
||||
) {
|
||||
retrievedAssets.push(info);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const apkPaths = new Map<string, string>();
|
||||
for (const split of apkSplits) {
|
||||
const folder = join(channel.apksFolder, split);
|
||||
for (const path of await readdir(folder, { recursive: true })) apkPaths.set(basename(path), join(folder, path));
|
||||
if (await exists(folder)) {
|
||||
try {
|
||||
for (const path of await readdir(folder, { recursive: true })) {
|
||||
apkPaths.set(basename(path), join(folder, path));
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const iconsDir = join(channel.canvasDir, "icons");
|
||||
|
|
@ -93,7 +117,7 @@ export async function parseAssets(channel: ChannelContext, code: string[]) {
|
|||
return { icons, toCopy };
|
||||
}
|
||||
|
||||
export default async function icons(channel: ChannelContext, progress: Progress, code: string[]) {
|
||||
export default async function icons(channel: ChannelContext, progress: Progress, code: string[] | string) {
|
||||
progress.start("icons_getting");
|
||||
|
||||
const { icons, toCopy } = await parseAssets(channel, code);
|
||||
|
|
@ -108,7 +132,21 @@ export default async function icons(channel: ChannelContext, progress: Progress,
|
|||
const dirs = listRequiredDirs(toCopy.map((x) => x.to));
|
||||
|
||||
await Promise.all(dirs.map((dir) => mkdir(dir, { recursive: true })));
|
||||
await Promise.all(toCopy.map(({ from, to }) => copyFile(from, to)));
|
||||
|
||||
// Batch file copying in chunks of 50 to avoid file descriptor starvation or timeouts
|
||||
const BATCH_SIZE = 50;
|
||||
for (let i = 0; i < toCopy.length; i += BATCH_SIZE) {
|
||||
const batch = toCopy.slice(i, i + BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(async ({ from, to }) => {
|
||||
try {
|
||||
if (await exists(from)) {
|
||||
await copyFile(from, to);
|
||||
}
|
||||
} catch {}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
await commit(["icons.json", "icons"], `chore: update icons for ${channel.cuteVersion}`, channel.canvasDir);
|
||||
progress.update("icons_copying", true);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue