fix(update): watchdog decompile and skip release on failure

- hermes-decomp killed by SIGTERM (exit 143) on pathological 380MB module this release
- replace Bun shell (no .timeout/.kill support) with Bun.spawn watchdog; kills decompile/extract after 20min
- on decompile failure, restore version.txt, mark run skipped (.update-skipped), exit cleanly so hourly run retries
- gate Publish/Tag/canvas steps on skip marker so a skipped release isn't pushed or tagged

.github/workflows/update.yml

@@ -70,16 +70,30 @@ jobs:
             - name: Publish changes
               if: ${{ steps.check.outputs.version != 'false' }}
               working-directory: data
-              run: git push
+              run: |
+                  if [ -f .update-skipped ]; then
+                      echo "Update skipped (decompile failed); not publishing."
+                      exit 0
+                  fi
+                  git push
 
             - name: Publish canvas
               if: ${{ steps.check.outputs.version != 'false' }}
-              working-directory: canvas
-              run: git push
+              working-directory: data
+              run: |
+                  if [ -f .update-skipped ]; then
+                      echo "Update skipped (decompile failed); not publishing canvas."
+                      exit 0
+                  fi
+                  git -C ../canvas push
 
             - name: Tag changes
               if: ${{ steps.check.outputs.version != 'false' }}
               working-directory: data
               run: |
+                  if [ -f .update-skipped ]; then
+                      echo "Update skipped (decompile failed); not tagging."
+                      exit 0
+                  fi
                   git tag -f ${{ steps.check.outputs.version }}
                   git push origin -f ${{ steps.check.outputs.version }}

src/update/runner.ts

@@ -47,6 +47,11 @@ export async function runTasks() {
 			progress.start("preinit");
 
 			progress.start("preinit_discard");
+			await Bun.$`git clean -f -- .update-skipped`
+				.cwd("../data")
+				.nothrow()
+				.quiet()
+				.catch(() => {});
 			await Bun.$`git reset --hard`.cwd("../data").nothrow().quiet().then(handleShellErr);
 			await Bun.write("../data/version.txt", version);
 			// canvas branch may not exist in test
@@ -90,7 +95,25 @@ export async function runTasks() {
 				"decompile",
 			)) as Promise<void> | undefined;
 		} catch (e) {
-			throw new Error(`Failed to decompile!\n${e}`);
+			const detail = (formatError(e) || String(e)).split("\n")[0];
+			console.warn(
+				`\nDecompile failed or timed out (${detail}).` +
+					`\nSkipping update for this release so the next run can retry.\n`,
+			);
+			// restore version.txt so the next check sees the release as still pending and re-runs
+			await Bun.$`git checkout -- version.txt`
+				.cwd("../data")
+				.nothrow()
+				.quiet()
+				.catch(() => {});
+			await Bun.$`git checkout -- version.txt`
+				.cwd("../canvas")
+				.nothrow()
+				.quiet()
+				.catch(() => {});
+			// mark the run as skipped so the workflow won't publish/tag a release that never decompiled
+			await Bun.write("../data/.update-skipped", version).catch(() => {});
+			return;
 		}
 
 		const code = (await Bun.file(codePath).text()).replace(/\r/g, "").split("\n");

src/update/tasks/decompile.ts

@@ -1,4 +1,5 @@
 import { exists } from "node:fs/promises";
+import { spawn } from "bun";
 import { commit } from "../git";
 import type { Progress } from "../progress";
 import { codePath, commitAnyway, cuteVersion, modulePathsDest, modulesPath, workFolder } from "../shared";
@@ -8,6 +9,23 @@ const bundleDest = join("..", "data", "index.android.bundle");
 
 const gzipWorkerURL = new URL("decompile-gzip.ts", import.meta.url).href;
 
+// generous cap so a hanging decompiler is aborted deterministically instead of wedging the job
+const DECOMPILE_TIMEOUT_MS = 20 * 60 * 1000;
+
+async function runDecompiler(cmd: string[], label: string) {
+	const proc = spawn({ cmd, stdout: "pipe", stderr: "pipe", stdin: "ignore" });
+	const timer = setTimeout(() => {
+		console.warn(`${label} exceeded ${DECOMPILE_TIMEOUT_MS / 1000}s, killing it`);
+		proc.kill();
+	}, DECOMPILE_TIMEOUT_MS);
+	const exitCode = await proc.exited;
+	clearTimeout(timer);
+	const out = `${await new Response(proc.stdout).text().catch(() => "")}${await new Response(proc.stderr)
+		.text()
+		.catch(() => "")}`.trim();
+	if (exitCode !== 0 && exitCode !== 11) throw new Error(`${label} failed (exit ${exitCode})\n${out}`.trim());
+}
+
 export default async function decompile(progress: Progress, pathToBundle: string) {
 	const pathToDecompiler = join(workFolder, "decompiler");
 
@@ -30,14 +48,11 @@ export default async function decompile(progress: Progress, pathToBundle: string
 	}
 
 	if (!(await Bun.file(codePath).exists())) {
-		await Bun.$`${decompilerBin} decompile ${pathToBundle} --output ${codePath}`.quiet().nothrow().then(handleShellErr);
+		await runDecompiler([decompilerBin, "decompile", pathToBundle, "--output", codePath], "decompile");
 	}
 
 	if (!(await exists(modulesPath))) {
-		await Bun.$`${decompilerBin} extract ${pathToBundle} --output ${modulesPath}`
-			.quiet()
-			.nothrow()
-			.then(handleShellErr);
+		await runDecompiler([decompilerBin, "extract", pathToBundle, "--output", modulesPath], "extract");
 	}
 
 	progress.update("decompile_decompiling", true);