diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index ac2f3f3ea38..e6c7c8034a8 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: inputs: force: - description: Force update regardless of "bun run check" + description: Force update regardless of whether versions changed default: false required: false type: boolean @@ -20,7 +20,7 @@ on: - cron: "30 * * * *" jobs: - check-update: + update: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -51,49 +51,27 @@ jobs: git -C ../canvas config user.name github-actions git -C ../canvas config user.email 41898282+github-actions[bot]@users.noreply.github.com - - name: Check for version - id: check - working-directory: main - run: | - version=$(bun run check ${{ github.event.inputs.force == 'true' && '--force' || '' }}) - echo "[DEBUG]: Check result: $version" - echo "version=$version" >> $GITHUB_OUTPUT - - name: Update - if: ${{ steps.check.outputs.version != 'false' }} env: code_webhook: ${{ secrets.CODE_WEBHOOK }} bot_token: ${{ secrets.BOT_TOKEN }} working-directory: main - run: bun run update:prod ${{ github.event.inputs.quiet == 'true' && '--quiet' || '' }} - - - name: Publish changes - if: ${{ steps.check.outputs.version != 'false' }} - working-directory: data run: | - if [ -f .update-skipped ]; then - echo "Update skipped (decompile failed); not publishing." - exit 0 - fi - git push + bun run src/update/index.ts ${{ github.event.inputs.force == 'true' && '--force' || '' }} ${{ github.event.inputs.quiet == 'true' && '--quiet' || '' }} + + - name: Publish changes + working-directory: data + run: git push - name: Publish canvas - if: ${{ steps.check.outputs.version != 'false' }} working-directory: data - run: | - if [ -f .update-skipped ]; then - echo "Update skipped (decompile failed); not publishing canvas." - exit 0 - fi - git -C ../canvas push + run: git -C ../canvas push - name: Tag changes - if: ${{ steps.check.outputs.version != 'false' }} working-directory: data run: | - if [ -f .update-skipped ]; then - echo "Update skipped (decompile failed); not tagging." - exit 0 - fi - git tag -f ${{ steps.check.outputs.version }} - git push origin -f ${{ steps.check.outputs.version }} + for ch in alpha beta stable; do + v=$(cat ".update-ok-$ch" 2>/dev/null) || continue + git tag -f "$ch-$v" + git push origin -f "$ch-$v" + done \ No newline at end of file diff --git a/package.json b/package.json index c6d1e29935a..d93348ace2f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "name": "themelings", "type": "module", "scripts": { - "check": "bun run src/check.ts", "update:test": "bun run --env-file local.env src/update/index.ts", "update:commit": "bun run --env-file local.env src/update/index.ts --commit-anyway", "update:mock": "bun run --env-file local.env src/update/index.ts --mock", diff --git a/src/check.ts b/src/check.ts deleted file mode 100644 index b819f30a8d0..00000000000 --- a/src/check.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { trackerUserAgent } from "./update/shared"; - -const tracker = await fetch("https://tracker.vendetta.rocks/tracker/index", { - headers: { - "User-Agent": trackerUserAgent, - }, -}); -if (!tracker.ok || tracker.status !== 200) throw new Error("Failed to get version from tracker!"); - -const data = (await tracker.json()) as { latest: { alpha: number } }; -const alpha = data.latest.alpha.toString(); - -const file = Bun.file("../data/version.txt"); -const latest = (await file.exists()) && (await file.text()); - -if (latest !== alpha || process.argv.includes("--force")) { - await file.write(alpha); - process.stdout.write(alpha); -} else process.stdout.write("false"); diff --git a/src/update/channel.ts b/src/update/channel.ts new file mode 100644 index 00000000000..5c894b4e33c --- /dev/null +++ b/src/update/channel.ts @@ -0,0 +1,38 @@ +export const channels = ["alpha", "beta", "stable"] as const; +export type Channel = (typeof channels)[number]; + +export interface ChannelContext { + channel: Channel; + dataDir: string; + canvasDir: string; + workFolder: string; + apksFolder: string; + codePath: string; + modulesPath: string; + modulePathsDest: string; + version: string; + cuteVersion: string; +} + +let current: ChannelContext | undefined; + +export function setChannel(channel: Channel, version: string): ChannelContext { + current = { + channel, + dataDir: `../data/${channel}`, + canvasDir: `../canvas/${channel}`, + workFolder: `tmp/${channel}`, + apksFolder: `tmp/${channel}/apks`, + codePath: `tmp/${channel}/code.js`, + modulesPath: `tmp/${channel}/modules`, + modulePathsDest: `../data/${channel}/module-paths.json`, + version, + cuteVersion: `${channel} ${version.slice(0, 3)}.${Number(version.slice(-2))}`, + }; + return current; +} + +export function ctx(): ChannelContext { + if (!current) throw new Error("Channel context is not set"); + return current; +} diff --git a/src/update/git.ts b/src/update/git.ts index 88591bf7c1d..438ce9ec6dd 100644 --- a/src/update/git.ts +++ b/src/update/git.ts @@ -1,3 +1,4 @@ +import { channels } from "./channel"; import { commitAnyway } from "./shared"; import { handleShellErr } from "./utils"; @@ -15,7 +16,7 @@ export async function fetchGitChanged() { .split("\x00") .filter((x) => x !== "") .map((x) => x.slice(3))) - if (changed) gitChanged.add(changed); + if (changed && channels.some((channel) => changed.startsWith(`${channel}/`))) gitChanged.add(changed); } catch {} } } @@ -28,7 +29,7 @@ export async function commit(files: string[], message: string, cwd = "../data") await fetchGitChanged(); } else { await Bun.$`git restore --staged .`.cwd(cwd).nothrow().quiet().then(handleShellErr); - if (cwd === "../data") await fetchGitChanged(); + if (cwd === "../data" || cwd.startsWith("../data/")) await fetchGitChanged(); await Bun.$`git add ${{ raw: files.map((x) => Bun.$.escape(x)).join(" ") }}` .cwd(cwd) .nothrow() diff --git a/src/update/index.ts b/src/update/index.ts index 1394f41ea0f..a35af89bc54 100644 --- a/src/update/index.ts +++ b/src/update/index.ts @@ -1,89 +1,144 @@ import { exists, mkdir, rm } from "node:fs/promises"; +import { type Channel, type ChannelContext, channels, setChannel } from "./channel"; import { log, makeProgress, wrapPromise } from "./progress"; import { runTasks } from "./runner"; -import { type apkSplits, apksFolder, isMock, trackerUserAgent, version, workFolder } from "./shared"; +import { type apkSplits, force, isMock, trackerUserAgent } from "./shared"; import { handleShellErr, join } from "./utils"; -const cdnUrl = `https://tracker.vendetta.rocks/tracker/download/${version}/`; +const tracker = await fetch("https://tracker.vendetta.rocks/tracker/index", { + headers: { + "User-Agent": trackerUserAgent, + }, +}); +if (!tracker.ok || tracker.status !== 200) throw new Error("Failed to get version from tracker!"); -const mediaFiles = Bun.$.braces("res/**/*.{png,jpg,lottie}"); -const apkAssets = { - base: ["assets/index.android.bundle", ...mediaFiles], - "config.hdpi": mediaFiles, - "config.xxhdpi": mediaFiles, -} satisfies Record<(typeof apkSplits)[number], string[]>; +const { latest } = (await tracker.json()) as { latest: Record }; -let reuseFolder = isMock; -if (!reuseFolder) { - const matches = await Bun.file("tmp/ver") - .text() - .then((x) => x === version) - .catch(() => false); - if (matches) { - const exist = await Promise.allSettled( - Object.keys(apkAssets).map((folder) => exists(join(apksFolder, folder, "res"))), +async function downloadApks(channel: ChannelContext) { + const cdnUrl = `https://tracker.vendetta.rocks/tracker/download/${channel.version}/`; + + const mediaFiles = Bun.$.braces("res/**/*.{png,jpg,lottie}"); + const apkAssets = { + base: ["assets/index.android.bundle", ...mediaFiles], + "config.hdpi": mediaFiles, + "config.xxhdpi": mediaFiles, + } satisfies Record<(typeof apkSplits)[number], string[]>; + + let reuseFolder = isMock; + if (!reuseFolder) { + const matches = await Bun.file(join(channel.workFolder, "ver")) + .text() + .then((x) => x === channel.version) + .catch(() => false); + if (matches) { + const exist = await Promise.allSettled( + Object.keys(apkAssets).map((folder) => exists(join(channel.apksFolder, folder, "res"))), + ); + if (exist.every((x) => x.status === "fulfilled" && x.value)) reuseFolder = true; + } + } + + if (!reuseFolder) await rm(channel.workFolder, { force: true, recursive: true }); + if (!(await exists(channel.apksFolder))) await mkdir(channel.apksFolder, { recursive: true }); + + if (!reuseFolder) { + log("Downloading APKs..."); + + const downloadProgress = makeProgress( + Object.fromEntries(Object.keys(apkAssets).map((apk) => [apk, `Downloading ${apk}.apk`])), ); - if (exist.every((x) => x.status === "fulfilled" && x.value)) reuseFolder = true; + await Promise.allSettled( + Object.keys(apkAssets).map((apk) => { + return wrapPromise( + fetch(cdnUrl + apk, { + headers: { + "User-Agent": trackerUserAgent, + "Cache-Control": "public, max-age=3600", + }, + }) + .then((res) => res.arrayBuffer()) + .then((data) => Bun.write(join(channel.workFolder, `${apk}.zip`), data)), + downloadProgress, + apk, + ); + }), + ); + if (downloadProgress.anyFailed()) + throw new Error(`Failed to download all APKs!\n${downloadProgress.prettyErrors()}`); + + log("\nUnzipping APKs..."); + + const unzipProgress = makeProgress( + Object.fromEntries(Object.keys(apkAssets).map((apk) => [apk, `Unzipping ${apk}.apk`])), + ); + + await Promise.allSettled( + Object.entries(apkAssets).map(([apk, assets]) => + wrapPromise( + Bun.$`unzip -o ${join(channel.workFolder, `${apk}.zip`)} ${{ + raw: assets.map((file) => Bun.$.escape(file)).join(" "), + }} -d ${channel.apksFolder} 2>/dev/null` + .quiet() + .nothrow() + .then(handleShellErr), + unzipProgress, + apk, + ), + ), + ); + if (unzipProgress.anyFailed()) throw new Error(`Failed to unzip all APKs!\n${unzipProgress.prettyErrors()}`); + + await Bun.write(join(channel.workFolder, "ver"), channel.version); + } else log("Reusing folder!"); +} + +async function resetRepos() { + for (const ch of channels) await rm(join("../data", `.update-ok-${ch}`), { force: true }).catch(() => {}); + for (const cwd of ["../data", "../canvas"] as const) { + await Bun.$`git reset --hard`.cwd(cwd).nothrow().quiet().then(handleShellErr); + await Bun.$`git restore --staged .` + .cwd(cwd) + .nothrow() + .quiet() + .catch(() => {}); } } -// Temp folder yippee! +async function processChannel(channelName: Channel, targetVersion: string) { + const channel = setChannel(channelName, targetVersion); + log(`\n${"ā•".repeat(30)}\n${channel.channel}: ${channel.version}\n${"ā•".repeat(30)}`); -if (!reuseFolder) await rm(workFolder, { force: true, recursive: true }); -if (!(await exists(apksFolder))) await mkdir(apksFolder, { recursive: true }); + try { + await downloadApks(channel); + await runTasks(channel); + await Bun.write(join("../data", `.update-ok-${channel.channel}`), channel.version); + } finally { + await rm(channel.workFolder, { force: true, recursive: true }).catch(() => {}); + } +} -if (!reuseFolder) { - // Download the APKs - log("Downloading APKs..."); +if (!isMock) { + await resetRepos(); - const downloadProgress = makeProgress( - Object.fromEntries(Object.keys(apkAssets).map((apk) => [apk, `Downloading ${apk}.apk`])), - ); - await Promise.allSettled( - Object.keys(apkAssets).map((apk) => { - return wrapPromise( - fetch(cdnUrl + apk, { - headers: { - "User-Agent": trackerUserAgent, - "Cache-Control": "public, max-age=3600", - }, - }) - .then((res) => res.arrayBuffer()) - .then((data) => Bun.write(join(workFolder, `${apk}.zip`), data)), - downloadProgress, - apk, - ); - }), - ); - if (downloadProgress.anyFailed()) throw new Error(`Failed to download all APKs!\n${downloadProgress.prettyErrors()}`); + for (const channelName of channels) { + const targetVersion = latest[channelName].toString(); + const local = await Bun.file(join("../data", channelName, "version.txt")) + .text() + .catch(() => ""); + if (!force && local === targetVersion) { + log(`${channelName}: up to date (${targetVersion})`); + continue; + } + await processChannel(channelName, targetVersion); + } +} else { + for (const channelName of channels) { + const version = + (await Bun.file(join("../data", channelName, "version.txt")) + .text() + .catch(() => "0")) || "0"; + await runTasks(setChannel(channelName, version)); + } +} - log("\nUnzipping APKs..."); - - const unzipProgress = makeProgress( - Object.fromEntries(Object.keys(apkAssets).map((apk) => [apk, `Unzipping ${apk}.apk`])), - ); - - await Promise.allSettled( - Object.entries(apkAssets).map(([apk, assets]) => - wrapPromise( - Bun.$`unzip -o ${join(workFolder, `${apk}.zip`)} ${{ - raw: assets.map((file) => Bun.$.escape(file)).join(" "), - }} -d ${join(apksFolder, apk)} 2>/dev/null` - .quiet() - .nothrow() - .then(handleShellErr), - unzipProgress, - apk, - ), - ), - ); - if (unzipProgress.anyFailed()) throw new Error(`Failed to unzip all APKs!\n${unzipProgress.prettyErrors()}`); - - await Bun.write(join(workFolder, "ver"), version); -} else log("Reusing folder!"); - -// Run tasks - -await runTasks(); - -log("āœ… Done"); +log("\nāœ… Done"); diff --git a/src/update/runner.ts b/src/update/runner.ts index 44cd2d9c976..7a4072ee866 100644 --- a/src/update/runner.ts +++ b/src/update/runner.ts @@ -1,20 +1,20 @@ import mock from "../mock"; import type { Differs } from "../types"; +import type { ChannelContext } from "./channel"; import { commit } from "./git"; import { log, makeProgress, wrapPromise } from "./progress"; -import { apksFolder, codePath, cuteVersion, isMock, isQuiet, oprevFiles, prevFiles, version } from "./shared"; +import { isMock, isQuiet, oprevFiles, prevFiles } from "./shared"; import codeTask from "./tasks/code"; import colorsTask from "./tasks/colors"; import decompile from "./tasks/decompile"; import diffs from "./tasks/diffs"; -import { formatError, handleShellErr, join } from "./utils"; +import { formatError, join } from "./utils"; -export async function runTasks() { - log("\nRunning tasks..."); +export async function runTasks(channel: ChannelContext) { + log(`\nRunning tasks for ${channel.cuteVersion}...`); const progress = makeProgress( { preinit: "Preinit", - preinit_discard: "Discarding changes", preinit_save: "Caching original files", decompile: "Decompilation", decompile_downloading: "Downloading decompiler", @@ -46,34 +46,10 @@ export async function runTasks() { try { progress.start("preinit"); - progress.start("preinit_discard"); - await Bun.$`git clean -f -- .update-skipped` - .cwd("../data") - .nothrow() - .quiet() - .catch(() => {}); - await Bun.$`git reset --hard`.cwd("../data").nothrow().quiet().then(handleShellErr); - await Bun.write("../data/version.txt", version); - // canvas branch may not exist in test - await Bun.$`git reset --hard` - .cwd("../canvas") - .nothrow() - .quiet() - .then(() => {}); - await Bun.write("../canvas/version.txt", version).catch(() => {}); - - await Bun.$`git restore --staged .`.cwd("../data").nothrow().quiet().then(handleShellErr); - await Bun.$`git restore --staged .` - .cwd("../canvas") - .nothrow() - .quiet() - .then(() => {}); - - progress.update("preinit_discard", true); - progress.start("preinit_save"); + prevFiles.clear(); for (const oprev of oprevFiles) { - for (const base of ["../data", "../canvas"] as const) { + for (const base of [channel.dataDir, channel.canvasDir] as const) { const file = Bun.file(join(base, oprev)); if (await file.exists()) { prevFiles.set(oprev, await file.arrayBuffer()); @@ -85,12 +61,12 @@ export async function runTasks() { progress.update("preinit", true); } catch (e) { progress.update("preinit", false); - throw new Error(`Failed to discard changes!\n${e}`); + throw new Error(`Failed to cache original files!\n${e}`); } try { gzipDone = (await wrapPromise( - decompile(progress, join(apksFolder, "base", "assets", "index.android.bundle")), + decompile(channel, progress, join(channel.apksFolder, "base", "assets", "index.android.bundle")), progress, "decompile", )) as Promise | undefined; @@ -98,30 +74,20 @@ export async function runTasks() { const detail = (formatError(e) || String(e)).split("\n")[0]; console.warn( `\nDecompile failed or timed out (${detail}).` + - `\nSkipping update for this release so the next run can retry.\n`, + `\nSkipping ${channel.channel} for this release so the next run can retry.\n`, ); - // restore version.txt so the next check sees the release as still pending and re-runs - await Bun.$`git checkout -- version.txt` - .cwd("../data") - .nothrow() - .quiet() - .catch(() => {}); - await Bun.$`git checkout -- version.txt` - .cwd("../canvas") - .nothrow() - .quiet() - .catch(() => {}); - // mark the run as skipped so the workflow won't publish/tag a release that never decompiled - await Bun.write("../data/.update-skipped", version).catch(() => {}); return; } - const code = (await Bun.file(codePath).text()).replace(/\r/g, "").split("\n"); + await Bun.write(join(channel.dataDir, "version.txt"), channel.version); + await Bun.write(join(channel.canvasDir, "version.txt"), channel.version).catch(() => {}); - await wrapPromise(codeTask(progress, code), progress, "code"); + const code = (await Bun.file(channel.codePath).text()).replace(/\r/g, "").split("\n"); + + await wrapPromise(codeTask(channel, progress, code), progress, "code"); if (progress.someFailed("code")) throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code")}`); // colors is non-critical, run in background - wrapPromise(colorsTask(code), progress, "colors").catch((e) => { + wrapPromise(colorsTask(channel, code), progress, "colors").catch((e) => { progress.update("colors", false, String(e)); console.warn("Colors task failed (non-critical):", e); }); @@ -147,7 +113,7 @@ export async function runTasks() { throw new Error(`Failed at the decompile gzip task!\n${progress.prettyErrors("decompile_gzip")}`); try { - const result = await diffs(progress); + const result = await diffs(channel, progress); if (result) { differs = result; progress.update("diff", true); @@ -171,9 +137,9 @@ export async function runTasks() { progress.update("webhook", null); } - await commit(["version.txt"], `chore: bump app version to ${cuteVersion}`); + await commit(["version.txt"], `chore: bump app version to ${channel.cuteVersion}`, channel.dataDir); // canvas version bump (if canvas checkout exists) try { - await commit(["version.txt"], `chore: bump app version to ${cuteVersion}`, "../canvas"); + await commit(["version.txt"], `chore: bump app version to ${channel.cuteVersion}`, channel.canvasDir); } catch {} } diff --git a/src/update/shared.ts b/src/update/shared.ts index e0f5f0f4c9a..710cbefad8a 100644 --- a/src/update/shared.ts +++ b/src/update/shared.ts @@ -1,21 +1,12 @@ -import { join } from "node:path"; - export const trackerUserAgent = "Themelings "; export const maxDiffChanges = 20; -export const workFolder = "tmp"; -export const apksFolder = join(workFolder, "apks"); -export const codePath = join(workFolder, "code.js"); -export const modulesPath = join(workFolder, "modules"); -export const modulePathsDest = join("..", "data", "module-paths.json"); export const apkSplits = ["base", "config.hdpi", "config.xxhdpi"] as const; export const oprevFiles = ["source.jsonl", "raw.json", "semantic.json", "semantic_simple.json", "icons.json"] as const; export const prevFiles = new Map<(typeof oprevFiles)[number], ArrayBuffer>(); -export const version = await Bun.file("../data/version.txt").text(); -export const cuteVersion = `${["stable", "beta", "alpha"][Number(version[3])] ?? "version"} ${version.slice(0, 3)}.${Number(version.slice(-2))}`; - +export const force = process.argv.includes("--force"); export const isMock = process.argv.includes("--mock"); export const diffAnyway = process.argv.includes("--diff-anyway"); export const commitAnyway = process.argv.includes("--commit-anyway"); diff --git a/src/update/tasks/code.ts b/src/update/tasks/code.ts index 894a3b92426..3eeb23b6312 100644 --- a/src/update/tasks/code.ts +++ b/src/update/tasks/code.ts @@ -1,8 +1,9 @@ import { mkdir, readdir, rm, stat } from "node:fs/promises"; import { join as pathJoin } from "node:path"; +import type { ChannelContext } from "../channel"; import { commit } from "../git"; import type { Progress } from "../progress"; -import { commitAnyway, cuteVersion, modulePathsDest, modulesPath } from "../shared"; +import { commitAnyway } from "../shared"; import { formatBytes, join, sortEntries } from "../utils"; const MAX_FILE_SIZE = 100 * 1024 * 1024; @@ -21,16 +22,16 @@ function sanitizePath(p: string): string { .replace(/[<>:"|?*]/g, "_"); } -export default async function code(progress: Progress, _code: string[]) { +export default async function code(channel: ChannelContext, progress: Progress, _code: string[]) { progress.start("code_getting"); - if (!(await stat(modulesPath).catch(() => false))) { - throw new Error(`Modules directory not found at ${modulesPath}`); + if (!(await stat(channel.modulesPath).catch(() => false))) { + throw new Error(`Modules directory not found at ${channel.modulesPath}`); } const modulePaths = new Map(); try { - const arr: { id: number; path: string }[] = await Bun.file(modulePathsDest).json(); + const arr: { id: number; path: string }[] = await Bun.file(channel.modulePathsDest).json(); for (const { id, path } of arr) modulePaths.set(id, path); console.log(`Loaded ${modulePaths.size} module path mappings`); } catch { @@ -54,7 +55,7 @@ export default async function code(progress: Progress, _code: string[]) { return files; } - const rawFiles = await scanDir(modulesPath, modulesPath); + const rawFiles = await scanDir(channel.modulesPath, channel.modulesPath); function classifyUnmapped(relativePath: string): string { if (/^module_\d+\.js$/.test(relativePath)) return "__polyfill"; @@ -88,7 +89,7 @@ export default async function code(progress: Progress, _code: string[]) { } await Bun.write( - "../data/source.jsonl", + join(channel.dataDir, "source.jsonl"), sortEntries([...files.entries()]) .map(([file, size]) => `{ "file": ${JSON.stringify(file)}, "size": ${size} }`) .join("\n"), @@ -99,7 +100,7 @@ export default async function code(progress: Progress, _code: string[]) { if (process.env.NODE_ENV !== "test" && !commitAnyway) { progress.start("code_remaking"); - const filePrefix = "../data/source"; + const filePrefix = join(channel.dataDir, "source"); await rm(filePrefix, { recursive: true, force: true }); await Promise.all( @@ -107,7 +108,7 @@ export default async function code(progress: Progress, _code: string[]) { if (fileSize > MAX_FILE_SIZE) return; const dest = join(filePrefix, resolveDest(relativePath)); await mkdir(pathJoin(dest, ".."), { recursive: true }); - await Bun.write(dest, Bun.file(join(modulesPath, relativePath))); + await Bun.write(dest, Bun.file(join(channel.modulesPath, relativePath))); }), ); @@ -119,9 +120,9 @@ export default async function code(progress: Progress, _code: string[]) { if (skipped.length) console.warn(`Skipped ${skipped.length} file(s) >100MB (GitHub limit):\n${skipped.join("\n")}`); const message = skipped.length - ? `chore: update source for ${cuteVersion}\n\nSkipped ${skipped.length} file(s) >100MB (GitHub limit): ${skipped.join(", ")}` - : `chore: update source for ${cuteVersion}`; - await commit(["source.jsonl", "source/*"], message); + ? `chore: update source for ${channel.cuteVersion}\n\nSkipped ${skipped.length} file(s) >100MB (GitHub limit): ${skipped.join(", ")}` + : `chore: update source for ${channel.cuteVersion}`; + await commit(["source.jsonl", "source/*"], message, channel.dataDir); progress.update("code_pushing", true); } else { progress.update("code_remaking", null); diff --git a/src/update/tasks/colors.ts b/src/update/tasks/colors.ts index 6d5aaa11635..349c8435e35 100644 --- a/src/update/tasks/colors.ts +++ b/src/update/tasks/colors.ts @@ -1,9 +1,9 @@ import { runInNewContext } from "node:vm"; import Color, { type ColorInstance } from "color"; import type { RawColors, SemanticColors } from "../../types"; +import type { ChannelContext } from "../channel"; import { commit } from "../git"; -import { cuteVersion } from "../shared"; -import { sortObj } from "../utils"; +import { join, sortObj } from "../utils"; function hex(color: ColorInstance) { return (color.alpha() === 1 ? color.hex() : color.hexa()).toLowerCase(); @@ -97,17 +97,20 @@ export function convertSimpleSemantic(semantic: SemanticColors) { return simpleSemantic; } -export default async function colors(code: string[]) { +export default async function colors(channel: ChannelContext, code: string[]) { const raw = getInternalRawColors(code); - await Bun.write("../canvas/raw.json", JSON.stringify(sortObj(raw), null, 4)); + await Bun.write(join(channel.canvasDir, "raw.json"), JSON.stringify(sortObj(raw), null, 4)); const semantic = getInternalSemanticColors(code, raw); - await Bun.write("../canvas/semantic.json", JSON.stringify(sortObj(semantic), null, 4)); - await Bun.write("../canvas/semantic_simple.json", JSON.stringify(sortObj(convertSimpleSemantic(semantic)), null, 4)); + await Bun.write(join(channel.canvasDir, "semantic.json"), JSON.stringify(sortObj(semantic), null, 4)); + await Bun.write( + join(channel.canvasDir, "semantic_simple.json"), + JSON.stringify(sortObj(convertSimpleSemantic(semantic)), null, 4), + ); await commit( ["raw.json", "semantic.json", "semantic_simple.json"], - `chore: update colors for ${cuteVersion}`, - "../canvas", + `chore: update colors for ${channel.cuteVersion}`, + channel.canvasDir, ); } diff --git a/src/update/tasks/decompile.ts b/src/update/tasks/decompile.ts index 8116eeb4745..e163eeed133 100644 --- a/src/update/tasks/decompile.ts +++ b/src/update/tasks/decompile.ts @@ -1,12 +1,11 @@ import { exists } from "node:fs/promises"; import { spawn } from "bun"; +import type { ChannelContext } from "../channel"; import { commit } from "../git"; import type { Progress } from "../progress"; -import { codePath, commitAnyway, cuteVersion, modulePathsDest, modulesPath, workFolder } from "../shared"; +import { commitAnyway } from "../shared"; import { handleShellErr, join } from "../utils"; -const bundleDest = join("..", "data", "index.android.bundle"); - const gzipWorkerURL = new URL("decompile-gzip.ts", import.meta.url).href; // generous cap so a hanging decompiler is aborted deterministically instead of wedging the job @@ -26,8 +25,9 @@ async function runDecompiler(cmd: string[], label: string) { if (exitCode !== 0 && exitCode !== 11) throw new Error(`${label} failed (exit ${exitCode})\n${out}`.trim()); } -export default async function decompile(progress: Progress, pathToBundle: string) { - const pathToDecompiler = join(workFolder, "decompiler"); +export default async function decompile(channel: ChannelContext, progress: Progress, pathToBundle: string) { + // shared across all channels within a run (built once) + const pathToDecompiler = join("tmp", "decompiler"); progress.start("decompile_downloading"); if (!(await exists(pathToDecompiler))) { @@ -47,18 +47,18 @@ export default async function decompile(progress: Progress, pathToBundle: string await Bun.$`cargo build --release -p hbc-decomp-cli`.cwd(pathToDecompiler).quiet().nothrow().then(handleShellErr); } - if (!(await Bun.file(codePath).exists())) { - await runDecompiler([decompilerBin, "decompile", pathToBundle, "--output", codePath], "decompile"); + if (!(await Bun.file(channel.codePath).exists())) { + await runDecompiler([decompilerBin, "decompile", pathToBundle, "--output", channel.codePath], "decompile"); } - if (!(await exists(modulesPath))) { - await runDecompiler([decompilerBin, "extract", pathToBundle, "--output", modulesPath], "extract"); + if (!(await exists(channel.modulesPath))) { + await runDecompiler([decompilerBin, "extract", pathToBundle, "--output", channel.modulesPath], "extract"); } progress.update("decompile_decompiling", true); progress.start("decompile_paths"); - const codeJs = await Bun.file(codePath).text(); + const codeJs = await Bun.file(channel.codePath).text(); const modulePaths: { id: number; path: string }[] = []; const moduleSections = codeJs.split(/\/\/ === Module (\d+): .+ ===\n?/); for (let i = 1; i < moduleSections.length; i += 2) { @@ -67,13 +67,14 @@ export default async function decompile(progress: Progress, pathToBundle: string const m = content.match(/fileFinishedImporting\("([^"]+)"\)/); if (m) modulePaths.push({ id: modId, path: m[1] }); } - await Bun.write(modulePathsDest, JSON.stringify(modulePaths)); + await Bun.write(channel.modulePathsDest, JSON.stringify(modulePaths)); console.log(`Extracted ${modulePaths.length} module paths`); progress.update("decompile_paths", true); if (process.env.NODE_ENV !== "test" && !commitAnyway) { const gzFile = "code.js.gz"; + const bundleDest = join("..", "data", channel.channel, "index.android.bundle"); await Bun.write(bundleDest, Bun.file(pathToBundle)); const gzipper = new Worker(gzipWorkerURL); @@ -83,14 +84,15 @@ export default async function decompile(progress: Progress, pathToBundle: string if (data === true) { await commit( [gzFile, "module-paths.json", "index.android.bundle"], - `chore: update decompiled code for ${cuteVersion}`, + `chore: update decompiled code for ${channel.cuteVersion}`, + channel.dataDir, ); progress.update("decompile_gzip", true); } gzipper.terminate(); resolveGzip(); }); - gzipper.postMessage({ path: codePath, target: join("../data", gzFile) }); + gzipper.postMessage({ path: channel.codePath, target: join("../data", channel.channel, gzFile) }); return gzipDone; } progress.update("decompile_gzip", null); diff --git a/src/update/tasks/diffs.ts b/src/update/tasks/diffs.ts index 0d72203719e..d3685ba924e 100644 --- a/src/update/tasks/diffs.ts +++ b/src/update/tasks/diffs.ts @@ -7,6 +7,7 @@ import { type RawColors, type SemanticColors, } from "../../types"; +import type { ChannelContext } from "../channel"; import { fetchGitChanged, gitChanged } from "../git"; import { type Progress, wrapPromise } from "../progress"; import { diffAnyway, prevFiles } from "../shared"; @@ -100,11 +101,11 @@ function diffEntriesCode( return changes; } -async function diffRaw() { +async function diffRaw(channel: ChannelContext) { if (!prevFiles.has("raw.json")) throw new Error("Missing prevFile: raw.json"); const oldRaw = JSON.parse(new TextDecoder().decode(prevFiles.get("raw.json"))) as RawColors; - const newRaw: RawColors = await Bun.file(join("../canvas", "raw.json")).json(); + const newRaw: RawColors = await Bun.file(join(channel.canvasDir, "raw.json")).json(); return diffEntries(oldRaw, newRaw, { getSource: (_, val) => val, @@ -122,11 +123,11 @@ function getSemanticLabel(semantic: Record) { .join(", "); } -async function diffSemantic() { +async function diffSemantic(channel: ChannelContext) { if (!prevFiles.has("semantic.json")) throw new Error("Missing prevFile: semantic.json"); const oldSemantic = JSON.parse(new TextDecoder().decode(prevFiles.get("semantic.json"))) as SemanticColors; - const newSemantic: SemanticColors = await Bun.file(join("../canvas", "semantic.json")).json(); + const newSemantic: SemanticColors = await Bun.file(join(channel.canvasDir, "semantic.json")).json(); const changes = diffEntries(oldSemantic, newSemantic, { getSource: (_, val) => @@ -173,13 +174,16 @@ async function diffSemantic() { return changes; } -async function diffIcons() { +async function diffIcons(channel: ChannelContext) { if (!prevFiles.has("icons.json")) throw new Error("Missing prevFile: icons.json"); const oldIcons = JSON.parse(new TextDecoder().decode(prevFiles.get("icons.json"))) as Icons; - const newIcons: Icons = await Bun.file(join("../canvas", "icons.json")).json(); + const newIcons: Icons = await Bun.file(join(channel.canvasDir, "icons.json")).json(); - const iconDir = { old: join("../canvas", "oldicons"), new: join("../canvas", "icons") }; + const iconDir = { + old: join(channel.canvasDir, "oldicons"), + new: join(channel.canvasDir, "icons"), + }; return diffEntries(oldIcons, newIcons, { getSource: (_, val) => join(iconDir.new, val.file), @@ -198,19 +202,23 @@ function parseSource(text: string) { ) as Record; } -async function diffCode() { +async function diffCode(channel: ChannelContext) { if (!prevFiles.has("source.jsonl")) throw new Error("Missing prevFile: source.jsonl"); const oldCode = parseSource(new TextDecoder().decode(prevFiles.get("source.jsonl"))); - const newCode = parseSource(await Bun.file(join("../data", "source.jsonl")).text()); + const newCode = parseSource(await Bun.file(join(channel.dataDir, "source.jsonl")).text()); return diffEntriesCode(oldCode, newCode); } -export default async function diffs(progress: Progress) { +export default async function diffs(channel: ChannelContext, progress: Progress) { await fetchGitChanged(); - if (!(gitChanged.has("version.txt") || diffAnyway)) { + const prefix = `${channel.channel}/`; + const changed = (name: string) => gitChanged.has(prefix + name); + const hasBaseline = (name: string) => prevFiles.has(name as typeof prevFiles extends Map ? K : never); + + if (!(changed("version.txt") || diffAnyway)) { progress.update("diff", null); progress.update("diff_raw", null); progress.update("diff_semantic", null); @@ -222,30 +230,30 @@ export default async function diffs(progress: Progress) { const differs: Differs = { raw: new Map(), semantic: new Map(), icons: new Map(), code: new Map() }; await Promise.all([ - gitChanged.has("raw.json") + changed("raw.json") && hasBaseline("raw.json") ? wrapPromise( - diffRaw().then((x) => (differs.raw = x)), + diffRaw(channel).then((x) => (differs.raw = x)), progress, "diff_raw", ) : progress.update("diff_raw", null), - gitChanged.has("semantic.json") + changed("semantic.json") && hasBaseline("semantic.json") ? wrapPromise( - diffSemantic().then((x) => (differs.semantic = x)), + diffSemantic(channel).then((x) => (differs.semantic = x)), progress, "diff_semantic", ) : progress.update("diff_semantic", null), - gitChanged.has("icons.json") + changed("icons.json") && hasBaseline("icons.json") ? wrapPromise( - diffIcons().then((x) => (differs.icons = x)), + diffIcons(channel).then((x) => (differs.icons = x)), progress, "diff_icons", ) : progress.update("diff_icons", null), - gitChanged.has("source.jsonl") + changed("source.jsonl") && hasBaseline("source.jsonl") ? wrapPromise( - diffCode().then((x) => (differs.code = x)), + diffCode(channel).then((x) => (differs.code = x)), progress, "diff_code", ) diff --git a/src/update/tasks/icons.ts b/src/update/tasks/icons.ts index d8e6376fc31..fb2895cc016 100644 --- a/src/update/tasks/icons.ts +++ b/src/update/tasks/icons.ts @@ -2,20 +2,16 @@ import { copyFile, mkdir, readdir, rename } from "node:fs/promises"; import { basename } from "node:path"; import { runInNewContext } from "node:vm"; import type { Icons } from "../../types"; +import type { ChannelContext } from "../channel"; import { commit } from "../git"; import type { Progress } from "../progress"; -import { apkSplits, apksFolder, cuteVersion } from "../shared"; +import { apkSplits } from "../shared"; import { discordPath, join, listRequiredDirs, sortObj } from "../utils"; const infoObjRegex = /({.+?})/; const scalesArrayRegex = /(\[.+?\])/; -// canvas branch stores icons -const iconsJsonPath = "../canvas/icons.json"; -const iconsDir = "../canvas/icons"; -const oldIconsDir = "../canvas/oldicons"; - -export async function parseAssets(code: string[]) { +export async function parseAssets(channel: ChannelContext, code: string[]) { const retrievedAssets: { httpServerLocation: string; width: number; @@ -58,10 +54,12 @@ export async function parseAssets(code: string[]) { const apkPaths = new Map(); for (const split of apkSplits) { - const folder = join(apksFolder, split); + const folder = join(channel.apksFolder, split); for (const path of await readdir(folder, { recursive: true })) apkPaths.set(basename(path), join(folder, path)); } + const iconsDir = join(channel.canvasDir, "icons"); + const icons: Icons = {}; const toCopy: { from: string; to: string }[] = []; for (const asset of retrievedAssets) { @@ -95,21 +93,23 @@ export async function parseAssets(code: string[]) { return { icons, toCopy }; } -export default async function icons(progress: Progress, code: string[]) { +export default async function icons(channel: ChannelContext, progress: Progress, code: string[]) { progress.start("icons_getting"); - const { icons, toCopy } = await parseAssets(code); - await Bun.write(iconsJsonPath, JSON.stringify(sortObj(icons), undefined, 4)); + const { icons, toCopy } = await parseAssets(channel, code); + await Bun.write(join(channel.canvasDir, "icons.json"), JSON.stringify(sortObj(icons), undefined, 4)); progress.update("icons_getting", true); progress.start("icons_copying"); + const iconsDir = join(channel.canvasDir, "icons"); + const oldIconsDir = join(channel.canvasDir, "oldicons"); await rename(iconsDir, oldIconsDir).catch(() => {}); const dirs = listRequiredDirs(toCopy.map((x) => x.to)); await Promise.all(dirs.map((dir) => mkdir(dir, { recursive: true }))); await Promise.all(toCopy.map(({ from, to }) => copyFile(from, to))); - await commit(["icons.json", "icons"], `chore: update icons for ${cuteVersion}`, "../canvas"); + await commit(["icons.json", "icons"], `chore: update icons for ${channel.cuteVersion}`, channel.canvasDir); progress.update("icons_copying", true); } diff --git a/src/update/tasks/webhook.ts b/src/update/tasks/webhook.ts index 80f77519a36..fdaf69e8c93 100644 --- a/src/update/tasks/webhook.ts +++ b/src/update/tasks/webhook.ts @@ -11,7 +11,8 @@ import { import { drawSections } from "../../canvas"; import { makeSections } from "../../canvas/factory"; import { type CodeDiff, type Diff, type Differs, DiffType } from "../../types"; -import { cuteVersion, maxDiffChanges, version } from "../shared"; +import { ctx } from "../channel"; +import { maxDiffChanges } from "../shared"; import { formatBytes, sortEntries } from "../utils"; function fileBase(path: string, other?: string) { @@ -143,7 +144,7 @@ async function sendWebhook(webhook: string, role: string, embeds: WebhookEmbed[] components: [ { type: ComponentType.TextDisplay, - content: `## ${title}\n-# ${version} (${cuteVersion})`, + content: `## ${title}\n-# ${ctx().version} (${ctx().cuteVersion})`, }, ...body.embed.map((content) => ({ type: ComponentType.TextDisplay,