From 14c5a5d00586ca7fb9d7c17da44cd0a31f2663cc Mon Sep 17 00:00:00 2001 From: lvwmwm Date: Fri, 4 Sep 2026 14:33:27 -0300 Subject: [PATCH] fix(update): watchdog decompile and skip release on failure - hermes-decomp killed by SIGTERM (exit 143) on pathological 380MB module this release - replace Bun shell (no .timeout/.kill support) with Bun.spawn watchdog; kills decompile/extract after 20min - on decompile failure, restore version.txt, mark run skipped (.update-skipped), exit cleanly so hourly run retries - gate Publish/Tag/canvas steps on skip marker so a skipped release isn't pushed or tagged --- .github/workflows/update.yml | 24 +++++++++++++++++++----- src/update/runner.ts | 25 ++++++++++++++++++++++++- src/update/tasks/decompile.ts | 25 ++++++++++++++++++++----- 3 files changed, 63 insertions(+), 11 deletions(-) diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index df45747aca8..ac2f3f3ea38 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -70,16 +70,30 @@ jobs: - name: Publish changes if: ${{ steps.check.outputs.version != 'false' }} working-directory: data - run: git push + run: | + if [ -f .update-skipped ]; then + echo "Update skipped (decompile failed); not publishing." + exit 0 + fi + git push - name: Publish canvas if: ${{ steps.check.outputs.version != 'false' }} - working-directory: canvas - run: git push + working-directory: data + run: | + if [ -f .update-skipped ]; then + echo "Update skipped (decompile failed); not publishing canvas." + exit 0 + fi + git -C ../canvas push - name: Tag changes if: ${{ steps.check.outputs.version != 'false' }} working-directory: data run: | - git tag -f ${{ steps.check.outputs.version }} - git push origin -f ${{ steps.check.outputs.version }} + if [ -f .update-skipped ]; then + echo "Update skipped (decompile failed); not tagging." + exit 0 + fi + git tag -f ${{ steps.check.outputs.version }} + git push origin -f ${{ steps.check.outputs.version }} diff --git a/src/update/runner.ts b/src/update/runner.ts index 8e02bae9c31..44cd2d9c976 100644 --- a/src/update/runner.ts +++ b/src/update/runner.ts @@ -47,6 +47,11 @@ export async function runTasks() { progress.start("preinit"); progress.start("preinit_discard"); + await Bun.$`git clean -f -- .update-skipped` + .cwd("../data") + .nothrow() + .quiet() + .catch(() => {}); await Bun.$`git reset --hard`.cwd("../data").nothrow().quiet().then(handleShellErr); await Bun.write("../data/version.txt", version); // canvas branch may not exist in test @@ -90,7 +95,25 @@ export async function runTasks() { "decompile", )) as Promise | undefined; } catch (e) { - throw new Error(`Failed to decompile!\n${e}`); + const detail = (formatError(e) || String(e)).split("\n")[0]; + console.warn( + `\nDecompile failed or timed out (${detail}).` + + `\nSkipping update for this release so the next run can retry.\n`, + ); + // restore version.txt so the next check sees the release as still pending and re-runs + await Bun.$`git checkout -- version.txt` + .cwd("../data") + .nothrow() + .quiet() + .catch(() => {}); + await Bun.$`git checkout -- version.txt` + .cwd("../canvas") + .nothrow() + .quiet() + .catch(() => {}); + // mark the run as skipped so the workflow won't publish/tag a release that never decompiled + await Bun.write("../data/.update-skipped", version).catch(() => {}); + return; } const code = (await Bun.file(codePath).text()).replace(/\r/g, "").split("\n"); diff --git a/src/update/tasks/decompile.ts b/src/update/tasks/decompile.ts index 501eeb218a5..8116eeb4745 100644 --- a/src/update/tasks/decompile.ts +++ b/src/update/tasks/decompile.ts @@ -1,4 +1,5 @@ import { exists } from "node:fs/promises"; +import { spawn } from "bun"; import { commit } from "../git"; import type { Progress } from "../progress"; import { codePath, commitAnyway, cuteVersion, modulePathsDest, modulesPath, workFolder } from "../shared"; @@ -8,6 +9,23 @@ const bundleDest = join("..", "data", "index.android.bundle"); const gzipWorkerURL = new URL("decompile-gzip.ts", import.meta.url).href; +// generous cap so a hanging decompiler is aborted deterministically instead of wedging the job +const DECOMPILE_TIMEOUT_MS = 20 * 60 * 1000; + +async function runDecompiler(cmd: string[], label: string) { + const proc = spawn({ cmd, stdout: "pipe", stderr: "pipe", stdin: "ignore" }); + const timer = setTimeout(() => { + console.warn(`${label} exceeded ${DECOMPILE_TIMEOUT_MS / 1000}s, killing it`); + proc.kill(); + }, DECOMPILE_TIMEOUT_MS); + const exitCode = await proc.exited; + clearTimeout(timer); + const out = `${await new Response(proc.stdout).text().catch(() => "")}${await new Response(proc.stderr) + .text() + .catch(() => "")}`.trim(); + if (exitCode !== 0 && exitCode !== 11) throw new Error(`${label} failed (exit ${exitCode})\n${out}`.trim()); +} + export default async function decompile(progress: Progress, pathToBundle: string) { const pathToDecompiler = join(workFolder, "decompiler"); @@ -30,14 +48,11 @@ export default async function decompile(progress: Progress, pathToBundle: string } if (!(await Bun.file(codePath).exists())) { - await Bun.$`${decompilerBin} decompile ${pathToBundle} --output ${codePath}`.quiet().nothrow().then(handleShellErr); + await runDecompiler([decompilerBin, "decompile", pathToBundle, "--output", codePath], "decompile"); } if (!(await exists(modulesPath))) { - await Bun.$`${decompilerBin} extract ${pathToBundle} --output ${modulesPath}` - .quiet() - .nothrow() - .then(handleShellErr); + await runDecompiler([decompilerBin, "extract", pathToBundle, "--output", modulesPath], "extract"); } progress.update("decompile_decompiling", true);