feat: per-channel release processing (alpha/beta/stable) + single sequential workflow
This commit is contained in:
lvwmwm
parent
14c5a5d005
commit
9b2cf30e4f
14 changed files with 284 additions and 260 deletions
44
.github/workflows/update.yml
vendored
44
.github/workflows/update.yml
vendored
|
|
@ -7,7 +7,7 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
force:
|
||||
description: Force update regardless of "bun run check"
|
||||
description: Force update regardless of whether versions changed
|
||||
default: false
|
||||
required: false
|
||||
type: boolean
|
||||
|
|
@ -20,7 +20,7 @@ on:
|
|||
- cron: "30 * * * *"
|
||||
|
||||
jobs:
|
||||
check-update:
|
||||
update:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -51,49 +51,27 @@ jobs:
|
|||
git -C ../canvas config user.name github-actions
|
||||
git -C ../canvas config user.email 41898282+github-actions[bot]@users.noreply.github.com
|
||||
|
||||
- name: Check for version
|
||||
id: check
|
||||
working-directory: main
|
||||
run: |
|
||||
version=$(bun run check ${{ github.event.inputs.force == 'true' && '--force' || '' }})
|
||||
echo "[DEBUG]: Check result: $version"
|
||||
echo "version=$version" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Update
|
||||
if: ${{ steps.check.outputs.version != 'false' }}
|
||||
env:
|
||||
code_webhook: ${{ secrets.CODE_WEBHOOK }}
|
||||
bot_token: ${{ secrets.BOT_TOKEN }}
|
||||
working-directory: main
|
||||
run: bun run update:prod ${{ github.event.inputs.quiet == 'true' && '--quiet' || '' }}
|
||||
run: |
|
||||
bun run src/update/index.ts ${{ github.event.inputs.force == 'true' && '--force' || '' }} ${{ github.event.inputs.quiet == 'true' && '--quiet' || '' }}
|
||||
|
||||
- name: Publish changes
|
||||
if: ${{ steps.check.outputs.version != 'false' }}
|
||||
working-directory: data
|
||||
run: |
|
||||
if [ -f .update-skipped ]; then
|
||||
echo "Update skipped (decompile failed); not publishing."
|
||||
exit 0
|
||||
fi
|
||||
git push
|
||||
run: git push
|
||||
|
||||
- name: Publish canvas
|
||||
if: ${{ steps.check.outputs.version != 'false' }}
|
||||
working-directory: data
|
||||
run: |
|
||||
if [ -f .update-skipped ]; then
|
||||
echo "Update skipped (decompile failed); not publishing canvas."
|
||||
exit 0
|
||||
fi
|
||||
git -C ../canvas push
|
||||
run: git -C ../canvas push
|
||||
|
||||
- name: Tag changes
|
||||
if: ${{ steps.check.outputs.version != 'false' }}
|
||||
working-directory: data
|
||||
run: |
|
||||
if [ -f .update-skipped ]; then
|
||||
echo "Update skipped (decompile failed); not tagging."
|
||||
exit 0
|
||||
fi
|
||||
git tag -f ${{ steps.check.outputs.version }}
|
||||
git push origin -f ${{ steps.check.outputs.version }}
|
||||
for ch in alpha beta stable; do
|
||||
v=$(cat ".update-ok-$ch" 2>/dev/null) || continue
|
||||
git tag -f "$ch-$v"
|
||||
git push origin -f "$ch-$v"
|
||||
done
|
||||
|
|
@ -2,7 +2,6 @@
|
|||
"name": "themelings",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"check": "bun run src/check.ts",
|
||||
"update:test": "bun run --env-file local.env src/update/index.ts",
|
||||
"update:commit": "bun run --env-file local.env src/update/index.ts --commit-anyway",
|
||||
"update:mock": "bun run --env-file local.env src/update/index.ts --mock",
|
||||
|
|
|
|||
19
src/check.ts
19
src/check.ts
|
|
@ -1,19 +0,0 @@
|
|||
import { trackerUserAgent } from "./update/shared";
|
||||
|
||||
const tracker = await fetch("https://tracker.vendetta.rocks/tracker/index", {
|
||||
headers: {
|
||||
"User-Agent": trackerUserAgent,
|
||||
},
|
||||
});
|
||||
if (!tracker.ok || tracker.status !== 200) throw new Error("Failed to get version from tracker!");
|
||||
|
||||
const data = (await tracker.json()) as { latest: { alpha: number } };
|
||||
const alpha = data.latest.alpha.toString();
|
||||
|
||||
const file = Bun.file("../data/version.txt");
|
||||
const latest = (await file.exists()) && (await file.text());
|
||||
|
||||
if (latest !== alpha || process.argv.includes("--force")) {
|
||||
await file.write(alpha);
|
||||
process.stdout.write(alpha);
|
||||
} else process.stdout.write("false");
|
||||
38
src/update/channel.ts
Normal file
38
src/update/channel.ts
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
export const channels = ["alpha", "beta", "stable"] as const;
|
||||
export type Channel = (typeof channels)[number];
|
||||
|
||||
export interface ChannelContext {
|
||||
channel: Channel;
|
||||
dataDir: string;
|
||||
canvasDir: string;
|
||||
workFolder: string;
|
||||
apksFolder: string;
|
||||
codePath: string;
|
||||
modulesPath: string;
|
||||
modulePathsDest: string;
|
||||
version: string;
|
||||
cuteVersion: string;
|
||||
}
|
||||
|
||||
let current: ChannelContext | undefined;
|
||||
|
||||
export function setChannel(channel: Channel, version: string): ChannelContext {
|
||||
current = {
|
||||
channel,
|
||||
dataDir: `../data/${channel}`,
|
||||
canvasDir: `../canvas/${channel}`,
|
||||
workFolder: `tmp/${channel}`,
|
||||
apksFolder: `tmp/${channel}/apks`,
|
||||
codePath: `tmp/${channel}/code.js`,
|
||||
modulesPath: `tmp/${channel}/modules`,
|
||||
modulePathsDest: `../data/${channel}/module-paths.json`,
|
||||
version,
|
||||
cuteVersion: `${channel} ${version.slice(0, 3)}.${Number(version.slice(-2))}`,
|
||||
};
|
||||
return current;
|
||||
}
|
||||
|
||||
export function ctx(): ChannelContext {
|
||||
if (!current) throw new Error("Channel context is not set");
|
||||
return current;
|
||||
}
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import { channels } from "./channel";
|
||||
import { commitAnyway } from "./shared";
|
||||
import { handleShellErr } from "./utils";
|
||||
|
||||
|
|
@ -15,7 +16,7 @@ export async function fetchGitChanged() {
|
|||
.split("\x00")
|
||||
.filter((x) => x !== "")
|
||||
.map((x) => x.slice(3)))
|
||||
if (changed) gitChanged.add(changed);
|
||||
if (changed && channels.some((channel) => changed.startsWith(`${channel}/`))) gitChanged.add(changed);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
|
@ -28,7 +29,7 @@ export async function commit(files: string[], message: string, cwd = "../data")
|
|||
await fetchGitChanged();
|
||||
} else {
|
||||
await Bun.$`git restore --staged .`.cwd(cwd).nothrow().quiet().then(handleShellErr);
|
||||
if (cwd === "../data") await fetchGitChanged();
|
||||
if (cwd === "../data" || cwd.startsWith("../data/")) await fetchGitChanged();
|
||||
await Bun.$`git add ${{ raw: files.map((x) => Bun.$.escape(x)).join(" ") }}`
|
||||
.cwd(cwd)
|
||||
.nothrow()
|
||||
|
|
|
|||
|
|
@ -1,10 +1,21 @@
|
|||
import { exists, mkdir, rm } from "node:fs/promises";
|
||||
import { type Channel, type ChannelContext, channels, setChannel } from "./channel";
|
||||
import { log, makeProgress, wrapPromise } from "./progress";
|
||||
import { runTasks } from "./runner";
|
||||
import { type apkSplits, apksFolder, isMock, trackerUserAgent, version, workFolder } from "./shared";
|
||||
import { type apkSplits, force, isMock, trackerUserAgent } from "./shared";
|
||||
import { handleShellErr, join } from "./utils";
|
||||
|
||||
const cdnUrl = `https://tracker.vendetta.rocks/tracker/download/${version}/`;
|
||||
const tracker = await fetch("https://tracker.vendetta.rocks/tracker/index", {
|
||||
headers: {
|
||||
"User-Agent": trackerUserAgent,
|
||||
},
|
||||
});
|
||||
if (!tracker.ok || tracker.status !== 200) throw new Error("Failed to get version from tracker!");
|
||||
|
||||
const { latest } = (await tracker.json()) as { latest: Record<Channel, number> };
|
||||
|
||||
async function downloadApks(channel: ChannelContext) {
|
||||
const cdnUrl = `https://tracker.vendetta.rocks/tracker/download/${channel.version}/`;
|
||||
|
||||
const mediaFiles = Bun.$.braces("res/**/*.{png,jpg,lottie}");
|
||||
const apkAssets = {
|
||||
|
|
@ -15,25 +26,22 @@ const apkAssets = {
|
|||
|
||||
let reuseFolder = isMock;
|
||||
if (!reuseFolder) {
|
||||
const matches = await Bun.file("tmp/ver")
|
||||
const matches = await Bun.file(join(channel.workFolder, "ver"))
|
||||
.text()
|
||||
.then((x) => x === version)
|
||||
.then((x) => x === channel.version)
|
||||
.catch(() => false);
|
||||
if (matches) {
|
||||
const exist = await Promise.allSettled(
|
||||
Object.keys(apkAssets).map((folder) => exists(join(apksFolder, folder, "res"))),
|
||||
Object.keys(apkAssets).map((folder) => exists(join(channel.apksFolder, folder, "res"))),
|
||||
);
|
||||
if (exist.every((x) => x.status === "fulfilled" && x.value)) reuseFolder = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Temp folder yippee!
|
||||
|
||||
if (!reuseFolder) await rm(workFolder, { force: true, recursive: true });
|
||||
if (!(await exists(apksFolder))) await mkdir(apksFolder, { recursive: true });
|
||||
if (!reuseFolder) await rm(channel.workFolder, { force: true, recursive: true });
|
||||
if (!(await exists(channel.apksFolder))) await mkdir(channel.apksFolder, { recursive: true });
|
||||
|
||||
if (!reuseFolder) {
|
||||
// Download the APKs
|
||||
log("Downloading APKs...");
|
||||
|
||||
const downloadProgress = makeProgress(
|
||||
|
|
@ -49,13 +57,14 @@ if (!reuseFolder) {
|
|||
},
|
||||
})
|
||||
.then((res) => res.arrayBuffer())
|
||||
.then((data) => Bun.write(join(workFolder, `${apk}.zip`), data)),
|
||||
.then((data) => Bun.write(join(channel.workFolder, `${apk}.zip`), data)),
|
||||
downloadProgress,
|
||||
apk,
|
||||
);
|
||||
}),
|
||||
);
|
||||
if (downloadProgress.anyFailed()) throw new Error(`Failed to download all APKs!\n${downloadProgress.prettyErrors()}`);
|
||||
if (downloadProgress.anyFailed())
|
||||
throw new Error(`Failed to download all APKs!\n${downloadProgress.prettyErrors()}`);
|
||||
|
||||
log("\nUnzipping APKs...");
|
||||
|
||||
|
|
@ -66,9 +75,9 @@ if (!reuseFolder) {
|
|||
await Promise.allSettled(
|
||||
Object.entries(apkAssets).map(([apk, assets]) =>
|
||||
wrapPromise(
|
||||
Bun.$`unzip -o ${join(workFolder, `${apk}.zip`)} ${{
|
||||
Bun.$`unzip -o ${join(channel.workFolder, `${apk}.zip`)} ${{
|
||||
raw: assets.map((file) => Bun.$.escape(file)).join(" "),
|
||||
}} -d ${join(apksFolder, apk)} 2>/dev/null`
|
||||
}} -d ${channel.apksFolder} 2>/dev/null`
|
||||
.quiet()
|
||||
.nothrow()
|
||||
.then(handleShellErr),
|
||||
|
|
@ -79,11 +88,57 @@ if (!reuseFolder) {
|
|||
);
|
||||
if (unzipProgress.anyFailed()) throw new Error(`Failed to unzip all APKs!\n${unzipProgress.prettyErrors()}`);
|
||||
|
||||
await Bun.write(join(workFolder, "ver"), version);
|
||||
await Bun.write(join(channel.workFolder, "ver"), channel.version);
|
||||
} else log("Reusing folder!");
|
||||
}
|
||||
|
||||
// Run tasks
|
||||
async function resetRepos() {
|
||||
for (const ch of channels) await rm(join("../data", `.update-ok-${ch}`), { force: true }).catch(() => {});
|
||||
for (const cwd of ["../data", "../canvas"] as const) {
|
||||
await Bun.$`git reset --hard`.cwd(cwd).nothrow().quiet().then(handleShellErr);
|
||||
await Bun.$`git restore --staged .`
|
||||
.cwd(cwd)
|
||||
.nothrow()
|
||||
.quiet()
|
||||
.catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
await runTasks();
|
||||
async function processChannel(channelName: Channel, targetVersion: string) {
|
||||
const channel = setChannel(channelName, targetVersion);
|
||||
log(`\n${"═".repeat(30)}\n${channel.channel}: ${channel.version}\n${"═".repeat(30)}`);
|
||||
|
||||
log("✅ Done");
|
||||
try {
|
||||
await downloadApks(channel);
|
||||
await runTasks(channel);
|
||||
await Bun.write(join("../data", `.update-ok-${channel.channel}`), channel.version);
|
||||
} finally {
|
||||
await rm(channel.workFolder, { force: true, recursive: true }).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
if (!isMock) {
|
||||
await resetRepos();
|
||||
|
||||
for (const channelName of channels) {
|
||||
const targetVersion = latest[channelName].toString();
|
||||
const local = await Bun.file(join("../data", channelName, "version.txt"))
|
||||
.text()
|
||||
.catch(() => "");
|
||||
if (!force && local === targetVersion) {
|
||||
log(`${channelName}: up to date (${targetVersion})`);
|
||||
continue;
|
||||
}
|
||||
await processChannel(channelName, targetVersion);
|
||||
}
|
||||
} else {
|
||||
for (const channelName of channels) {
|
||||
const version =
|
||||
(await Bun.file(join("../data", channelName, "version.txt"))
|
||||
.text()
|
||||
.catch(() => "0")) || "0";
|
||||
await runTasks(setChannel(channelName, version));
|
||||
}
|
||||
}
|
||||
|
||||
log("\n✅ Done");
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
import mock from "../mock";
|
||||
import type { Differs } from "../types";
|
||||
import type { ChannelContext } from "./channel";
|
||||
import { commit } from "./git";
|
||||
import { log, makeProgress, wrapPromise } from "./progress";
|
||||
import { apksFolder, codePath, cuteVersion, isMock, isQuiet, oprevFiles, prevFiles, version } from "./shared";
|
||||
import { isMock, isQuiet, oprevFiles, prevFiles } from "./shared";
|
||||
import codeTask from "./tasks/code";
|
||||
import colorsTask from "./tasks/colors";
|
||||
import decompile from "./tasks/decompile";
|
||||
import diffs from "./tasks/diffs";
|
||||
import { formatError, handleShellErr, join } from "./utils";
|
||||
import { formatError, join } from "./utils";
|
||||
|
||||
export async function runTasks() {
|
||||
log("\nRunning tasks...");
|
||||
export async function runTasks(channel: ChannelContext) {
|
||||
log(`\nRunning tasks for ${channel.cuteVersion}...`);
|
||||
const progress = makeProgress(
|
||||
{
|
||||
preinit: "Preinit",
|
||||
preinit_discard: "Discarding changes",
|
||||
preinit_save: "Caching original files",
|
||||
decompile: "Decompilation",
|
||||
decompile_downloading: "Downloading decompiler",
|
||||
|
|
@ -46,34 +46,10 @@ export async function runTasks() {
|
|||
try {
|
||||
progress.start("preinit");
|
||||
|
||||
progress.start("preinit_discard");
|
||||
await Bun.$`git clean -f -- .update-skipped`
|
||||
.cwd("../data")
|
||||
.nothrow()
|
||||
.quiet()
|
||||
.catch(() => {});
|
||||
await Bun.$`git reset --hard`.cwd("../data").nothrow().quiet().then(handleShellErr);
|
||||
await Bun.write("../data/version.txt", version);
|
||||
// canvas branch may not exist in test
|
||||
await Bun.$`git reset --hard`
|
||||
.cwd("../canvas")
|
||||
.nothrow()
|
||||
.quiet()
|
||||
.then(() => {});
|
||||
await Bun.write("../canvas/version.txt", version).catch(() => {});
|
||||
|
||||
await Bun.$`git restore --staged .`.cwd("../data").nothrow().quiet().then(handleShellErr);
|
||||
await Bun.$`git restore --staged .`
|
||||
.cwd("../canvas")
|
||||
.nothrow()
|
||||
.quiet()
|
||||
.then(() => {});
|
||||
|
||||
progress.update("preinit_discard", true);
|
||||
|
||||
progress.start("preinit_save");
|
||||
prevFiles.clear();
|
||||
for (const oprev of oprevFiles) {
|
||||
for (const base of ["../data", "../canvas"] as const) {
|
||||
for (const base of [channel.dataDir, channel.canvasDir] as const) {
|
||||
const file = Bun.file(join(base, oprev));
|
||||
if (await file.exists()) {
|
||||
prevFiles.set(oprev, await file.arrayBuffer());
|
||||
|
|
@ -85,12 +61,12 @@ export async function runTasks() {
|
|||
progress.update("preinit", true);
|
||||
} catch (e) {
|
||||
progress.update("preinit", false);
|
||||
throw new Error(`Failed to discard changes!\n${e}`);
|
||||
throw new Error(`Failed to cache original files!\n${e}`);
|
||||
}
|
||||
|
||||
try {
|
||||
gzipDone = (await wrapPromise(
|
||||
decompile(progress, join(apksFolder, "base", "assets", "index.android.bundle")),
|
||||
decompile(channel, progress, join(channel.apksFolder, "base", "assets", "index.android.bundle")),
|
||||
progress,
|
||||
"decompile",
|
||||
)) as Promise<void> | undefined;
|
||||
|
|
@ -98,30 +74,20 @@ export async function runTasks() {
|
|||
const detail = (formatError(e) || String(e)).split("\n")[0];
|
||||
console.warn(
|
||||
`\nDecompile failed or timed out (${detail}).` +
|
||||
`\nSkipping update for this release so the next run can retry.\n`,
|
||||
`\nSkipping ${channel.channel} for this release so the next run can retry.\n`,
|
||||
);
|
||||
// restore version.txt so the next check sees the release as still pending and re-runs
|
||||
await Bun.$`git checkout -- version.txt`
|
||||
.cwd("../data")
|
||||
.nothrow()
|
||||
.quiet()
|
||||
.catch(() => {});
|
||||
await Bun.$`git checkout -- version.txt`
|
||||
.cwd("../canvas")
|
||||
.nothrow()
|
||||
.quiet()
|
||||
.catch(() => {});
|
||||
// mark the run as skipped so the workflow won't publish/tag a release that never decompiled
|
||||
await Bun.write("../data/.update-skipped", version).catch(() => {});
|
||||
return;
|
||||
}
|
||||
|
||||
const code = (await Bun.file(codePath).text()).replace(/\r/g, "").split("\n");
|
||||
await Bun.write(join(channel.dataDir, "version.txt"), channel.version);
|
||||
await Bun.write(join(channel.canvasDir, "version.txt"), channel.version).catch(() => {});
|
||||
|
||||
await wrapPromise(codeTask(progress, code), progress, "code");
|
||||
const code = (await Bun.file(channel.codePath).text()).replace(/\r/g, "").split("\n");
|
||||
|
||||
await wrapPromise(codeTask(channel, progress, code), progress, "code");
|
||||
if (progress.someFailed("code")) throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code")}`);
|
||||
// colors is non-critical, run in background
|
||||
wrapPromise(colorsTask(code), progress, "colors").catch((e) => {
|
||||
wrapPromise(colorsTask(channel, code), progress, "colors").catch((e) => {
|
||||
progress.update("colors", false, String(e));
|
||||
console.warn("Colors task failed (non-critical):", e);
|
||||
});
|
||||
|
|
@ -147,7 +113,7 @@ export async function runTasks() {
|
|||
throw new Error(`Failed at the decompile gzip task!\n${progress.prettyErrors("decompile_gzip")}`);
|
||||
|
||||
try {
|
||||
const result = await diffs(progress);
|
||||
const result = await diffs(channel, progress);
|
||||
if (result) {
|
||||
differs = result;
|
||||
progress.update("diff", true);
|
||||
|
|
@ -171,9 +137,9 @@ export async function runTasks() {
|
|||
progress.update("webhook", null);
|
||||
}
|
||||
|
||||
await commit(["version.txt"], `chore: bump app version to ${cuteVersion}`);
|
||||
await commit(["version.txt"], `chore: bump app version to ${channel.cuteVersion}`, channel.dataDir);
|
||||
// canvas version bump (if canvas checkout exists)
|
||||
try {
|
||||
await commit(["version.txt"], `chore: bump app version to ${cuteVersion}`, "../canvas");
|
||||
await commit(["version.txt"], `chore: bump app version to ${channel.cuteVersion}`, channel.canvasDir);
|
||||
} catch {}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1,12 @@
|
|||
import { join } from "node:path";
|
||||
|
||||
export const trackerUserAgent = "Themelings <https://github.com/nexpid/Themelings/>";
|
||||
export const maxDiffChanges = 20;
|
||||
|
||||
export const workFolder = "tmp";
|
||||
export const apksFolder = join(workFolder, "apks");
|
||||
export const codePath = join(workFolder, "code.js");
|
||||
export const modulesPath = join(workFolder, "modules");
|
||||
export const modulePathsDest = join("..", "data", "module-paths.json");
|
||||
export const apkSplits = ["base", "config.hdpi", "config.xxhdpi"] as const;
|
||||
|
||||
export const oprevFiles = ["source.jsonl", "raw.json", "semantic.json", "semantic_simple.json", "icons.json"] as const;
|
||||
export const prevFiles = new Map<(typeof oprevFiles)[number], ArrayBuffer>();
|
||||
|
||||
export const version = await Bun.file("../data/version.txt").text();
|
||||
export const cuteVersion = `${["stable", "beta", "alpha"][Number(version[3])] ?? "version"} ${version.slice(0, 3)}.${Number(version.slice(-2))}`;
|
||||
|
||||
export const force = process.argv.includes("--force");
|
||||
export const isMock = process.argv.includes("--mock");
|
||||
export const diffAnyway = process.argv.includes("--diff-anyway");
|
||||
export const commitAnyway = process.argv.includes("--commit-anyway");
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
import { mkdir, readdir, rm, stat } from "node:fs/promises";
|
||||
import { join as pathJoin } from "node:path";
|
||||
import type { ChannelContext } from "../channel";
|
||||
import { commit } from "../git";
|
||||
import type { Progress } from "../progress";
|
||||
import { commitAnyway, cuteVersion, modulePathsDest, modulesPath } from "../shared";
|
||||
import { commitAnyway } from "../shared";
|
||||
import { formatBytes, join, sortEntries } from "../utils";
|
||||
|
||||
const MAX_FILE_SIZE = 100 * 1024 * 1024;
|
||||
|
|
@ -21,16 +22,16 @@ function sanitizePath(p: string): string {
|
|||
.replace(/[<>:"|?*]/g, "_");
|
||||
}
|
||||
|
||||
export default async function code(progress: Progress, _code: string[]) {
|
||||
export default async function code(channel: ChannelContext, progress: Progress, _code: string[]) {
|
||||
progress.start("code_getting");
|
||||
|
||||
if (!(await stat(modulesPath).catch(() => false))) {
|
||||
throw new Error(`Modules directory not found at ${modulesPath}`);
|
||||
if (!(await stat(channel.modulesPath).catch(() => false))) {
|
||||
throw new Error(`Modules directory not found at ${channel.modulesPath}`);
|
||||
}
|
||||
|
||||
const modulePaths = new Map<number, string>();
|
||||
try {
|
||||
const arr: { id: number; path: string }[] = await Bun.file(modulePathsDest).json();
|
||||
const arr: { id: number; path: string }[] = await Bun.file(channel.modulePathsDest).json();
|
||||
for (const { id, path } of arr) modulePaths.set(id, path);
|
||||
console.log(`Loaded ${modulePaths.size} module path mappings`);
|
||||
} catch {
|
||||
|
|
@ -54,7 +55,7 @@ export default async function code(progress: Progress, _code: string[]) {
|
|||
return files;
|
||||
}
|
||||
|
||||
const rawFiles = await scanDir(modulesPath, modulesPath);
|
||||
const rawFiles = await scanDir(channel.modulesPath, channel.modulesPath);
|
||||
|
||||
function classifyUnmapped(relativePath: string): string {
|
||||
if (/^module_\d+\.js$/.test(relativePath)) return "__polyfill";
|
||||
|
|
@ -88,7 +89,7 @@ export default async function code(progress: Progress, _code: string[]) {
|
|||
}
|
||||
|
||||
await Bun.write(
|
||||
"../data/source.jsonl",
|
||||
join(channel.dataDir, "source.jsonl"),
|
||||
sortEntries([...files.entries()])
|
||||
.map(([file, size]) => `{ "file": ${JSON.stringify(file)}, "size": ${size} }`)
|
||||
.join("\n"),
|
||||
|
|
@ -99,7 +100,7 @@ export default async function code(progress: Progress, _code: string[]) {
|
|||
if (process.env.NODE_ENV !== "test" && !commitAnyway) {
|
||||
progress.start("code_remaking");
|
||||
|
||||
const filePrefix = "../data/source";
|
||||
const filePrefix = join(channel.dataDir, "source");
|
||||
await rm(filePrefix, { recursive: true, force: true });
|
||||
|
||||
await Promise.all(
|
||||
|
|
@ -107,7 +108,7 @@ export default async function code(progress: Progress, _code: string[]) {
|
|||
if (fileSize > MAX_FILE_SIZE) return;
|
||||
const dest = join(filePrefix, resolveDest(relativePath));
|
||||
await mkdir(pathJoin(dest, ".."), { recursive: true });
|
||||
await Bun.write(dest, Bun.file(join(modulesPath, relativePath)));
|
||||
await Bun.write(dest, Bun.file(join(channel.modulesPath, relativePath)));
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
@ -119,9 +120,9 @@ export default async function code(progress: Progress, _code: string[]) {
|
|||
if (skipped.length) console.warn(`Skipped ${skipped.length} file(s) >100MB (GitHub limit):\n${skipped.join("\n")}`);
|
||||
|
||||
const message = skipped.length
|
||||
? `chore: update source for ${cuteVersion}\n\nSkipped ${skipped.length} file(s) >100MB (GitHub limit): ${skipped.join(", ")}`
|
||||
: `chore: update source for ${cuteVersion}`;
|
||||
await commit(["source.jsonl", "source/*"], message);
|
||||
? `chore: update source for ${channel.cuteVersion}\n\nSkipped ${skipped.length} file(s) >100MB (GitHub limit): ${skipped.join(", ")}`
|
||||
: `chore: update source for ${channel.cuteVersion}`;
|
||||
await commit(["source.jsonl", "source/*"], message, channel.dataDir);
|
||||
progress.update("code_pushing", true);
|
||||
} else {
|
||||
progress.update("code_remaking", null);
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import { runInNewContext } from "node:vm";
|
||||
import Color, { type ColorInstance } from "color";
|
||||
import type { RawColors, SemanticColors } from "../../types";
|
||||
import type { ChannelContext } from "../channel";
|
||||
import { commit } from "../git";
|
||||
import { cuteVersion } from "../shared";
|
||||
import { sortObj } from "../utils";
|
||||
import { join, sortObj } from "../utils";
|
||||
|
||||
function hex(color: ColorInstance) {
|
||||
return (color.alpha() === 1 ? color.hex() : color.hexa()).toLowerCase();
|
||||
|
|
@ -97,17 +97,20 @@ export function convertSimpleSemantic(semantic: SemanticColors) {
|
|||
return simpleSemantic;
|
||||
}
|
||||
|
||||
export default async function colors(code: string[]) {
|
||||
export default async function colors(channel: ChannelContext, code: string[]) {
|
||||
const raw = getInternalRawColors(code);
|
||||
await Bun.write("../canvas/raw.json", JSON.stringify(sortObj(raw), null, 4));
|
||||
await Bun.write(join(channel.canvasDir, "raw.json"), JSON.stringify(sortObj(raw), null, 4));
|
||||
|
||||
const semantic = getInternalSemanticColors(code, raw);
|
||||
await Bun.write("../canvas/semantic.json", JSON.stringify(sortObj(semantic), null, 4));
|
||||
await Bun.write("../canvas/semantic_simple.json", JSON.stringify(sortObj(convertSimpleSemantic(semantic)), null, 4));
|
||||
await Bun.write(join(channel.canvasDir, "semantic.json"), JSON.stringify(sortObj(semantic), null, 4));
|
||||
await Bun.write(
|
||||
join(channel.canvasDir, "semantic_simple.json"),
|
||||
JSON.stringify(sortObj(convertSimpleSemantic(semantic)), null, 4),
|
||||
);
|
||||
|
||||
await commit(
|
||||
["raw.json", "semantic.json", "semantic_simple.json"],
|
||||
`chore: update colors for ${cuteVersion}`,
|
||||
"../canvas",
|
||||
`chore: update colors for ${channel.cuteVersion}`,
|
||||
channel.canvasDir,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import { exists } from "node:fs/promises";
|
||||
import { spawn } from "bun";
|
||||
import type { ChannelContext } from "../channel";
|
||||
import { commit } from "../git";
|
||||
import type { Progress } from "../progress";
|
||||
import { codePath, commitAnyway, cuteVersion, modulePathsDest, modulesPath, workFolder } from "../shared";
|
||||
import { commitAnyway } from "../shared";
|
||||
import { handleShellErr, join } from "../utils";
|
||||
|
||||
const bundleDest = join("..", "data", "index.android.bundle");
|
||||
|
||||
const gzipWorkerURL = new URL("decompile-gzip.ts", import.meta.url).href;
|
||||
|
||||
// generous cap so a hanging decompiler is aborted deterministically instead of wedging the job
|
||||
|
|
@ -26,8 +25,9 @@ async function runDecompiler(cmd: string[], label: string) {
|
|||
if (exitCode !== 0 && exitCode !== 11) throw new Error(`${label} failed (exit ${exitCode})\n${out}`.trim());
|
||||
}
|
||||
|
||||
export default async function decompile(progress: Progress, pathToBundle: string) {
|
||||
const pathToDecompiler = join(workFolder, "decompiler");
|
||||
export default async function decompile(channel: ChannelContext, progress: Progress, pathToBundle: string) {
|
||||
// shared across all channels within a run (built once)
|
||||
const pathToDecompiler = join("tmp", "decompiler");
|
||||
|
||||
progress.start("decompile_downloading");
|
||||
if (!(await exists(pathToDecompiler))) {
|
||||
|
|
@ -47,18 +47,18 @@ export default async function decompile(progress: Progress, pathToBundle: string
|
|||
await Bun.$`cargo build --release -p hbc-decomp-cli`.cwd(pathToDecompiler).quiet().nothrow().then(handleShellErr);
|
||||
}
|
||||
|
||||
if (!(await Bun.file(codePath).exists())) {
|
||||
await runDecompiler([decompilerBin, "decompile", pathToBundle, "--output", codePath], "decompile");
|
||||
if (!(await Bun.file(channel.codePath).exists())) {
|
||||
await runDecompiler([decompilerBin, "decompile", pathToBundle, "--output", channel.codePath], "decompile");
|
||||
}
|
||||
|
||||
if (!(await exists(modulesPath))) {
|
||||
await runDecompiler([decompilerBin, "extract", pathToBundle, "--output", modulesPath], "extract");
|
||||
if (!(await exists(channel.modulesPath))) {
|
||||
await runDecompiler([decompilerBin, "extract", pathToBundle, "--output", channel.modulesPath], "extract");
|
||||
}
|
||||
|
||||
progress.update("decompile_decompiling", true);
|
||||
|
||||
progress.start("decompile_paths");
|
||||
const codeJs = await Bun.file(codePath).text();
|
||||
const codeJs = await Bun.file(channel.codePath).text();
|
||||
const modulePaths: { id: number; path: string }[] = [];
|
||||
const moduleSections = codeJs.split(/\/\/ === Module (\d+): .+ ===\n?/);
|
||||
for (let i = 1; i < moduleSections.length; i += 2) {
|
||||
|
|
@ -67,13 +67,14 @@ export default async function decompile(progress: Progress, pathToBundle: string
|
|||
const m = content.match(/fileFinishedImporting\("([^"]+)"\)/);
|
||||
if (m) modulePaths.push({ id: modId, path: m[1] });
|
||||
}
|
||||
await Bun.write(modulePathsDest, JSON.stringify(modulePaths));
|
||||
await Bun.write(channel.modulePathsDest, JSON.stringify(modulePaths));
|
||||
console.log(`Extracted ${modulePaths.length} module paths`);
|
||||
progress.update("decompile_paths", true);
|
||||
|
||||
if (process.env.NODE_ENV !== "test" && !commitAnyway) {
|
||||
const gzFile = "code.js.gz";
|
||||
|
||||
const bundleDest = join("..", "data", channel.channel, "index.android.bundle");
|
||||
await Bun.write(bundleDest, Bun.file(pathToBundle));
|
||||
|
||||
const gzipper = new Worker(gzipWorkerURL);
|
||||
|
|
@ -83,14 +84,15 @@ export default async function decompile(progress: Progress, pathToBundle: string
|
|||
if (data === true) {
|
||||
await commit(
|
||||
[gzFile, "module-paths.json", "index.android.bundle"],
|
||||
`chore: update decompiled code for ${cuteVersion}`,
|
||||
`chore: update decompiled code for ${channel.cuteVersion}`,
|
||||
channel.dataDir,
|
||||
);
|
||||
progress.update("decompile_gzip", true);
|
||||
}
|
||||
gzipper.terminate();
|
||||
resolveGzip();
|
||||
});
|
||||
gzipper.postMessage({ path: codePath, target: join("../data", gzFile) });
|
||||
gzipper.postMessage({ path: channel.codePath, target: join("../data", channel.channel, gzFile) });
|
||||
return gzipDone;
|
||||
}
|
||||
progress.update("decompile_gzip", null);
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
type RawColors,
|
||||
type SemanticColors,
|
||||
} from "../../types";
|
||||
import type { ChannelContext } from "../channel";
|
||||
import { fetchGitChanged, gitChanged } from "../git";
|
||||
import { type Progress, wrapPromise } from "../progress";
|
||||
import { diffAnyway, prevFiles } from "../shared";
|
||||
|
|
@ -100,11 +101,11 @@ function diffEntriesCode(
|
|||
return changes;
|
||||
}
|
||||
|
||||
async function diffRaw() {
|
||||
async function diffRaw(channel: ChannelContext) {
|
||||
if (!prevFiles.has("raw.json")) throw new Error("Missing prevFile: raw.json");
|
||||
|
||||
const oldRaw = JSON.parse(new TextDecoder().decode(prevFiles.get("raw.json"))) as RawColors;
|
||||
const newRaw: RawColors = await Bun.file(join("../canvas", "raw.json")).json();
|
||||
const newRaw: RawColors = await Bun.file(join(channel.canvasDir, "raw.json")).json();
|
||||
|
||||
return diffEntries(oldRaw, newRaw, {
|
||||
getSource: (_, val) => val,
|
||||
|
|
@ -122,11 +123,11 @@ function getSemanticLabel(semantic: Record<string, [string, unknown]>) {
|
|||
.join(", ");
|
||||
}
|
||||
|
||||
async function diffSemantic() {
|
||||
async function diffSemantic(channel: ChannelContext) {
|
||||
if (!prevFiles.has("semantic.json")) throw new Error("Missing prevFile: semantic.json");
|
||||
|
||||
const oldSemantic = JSON.parse(new TextDecoder().decode(prevFiles.get("semantic.json"))) as SemanticColors;
|
||||
const newSemantic: SemanticColors = await Bun.file(join("../canvas", "semantic.json")).json();
|
||||
const newSemantic: SemanticColors = await Bun.file(join(channel.canvasDir, "semantic.json")).json();
|
||||
|
||||
const changes = diffEntries(oldSemantic, newSemantic, {
|
||||
getSource: (_, val) =>
|
||||
|
|
@ -173,13 +174,16 @@ async function diffSemantic() {
|
|||
return changes;
|
||||
}
|
||||
|
||||
async function diffIcons() {
|
||||
async function diffIcons(channel: ChannelContext) {
|
||||
if (!prevFiles.has("icons.json")) throw new Error("Missing prevFile: icons.json");
|
||||
|
||||
const oldIcons = JSON.parse(new TextDecoder().decode(prevFiles.get("icons.json"))) as Icons;
|
||||
const newIcons: Icons = await Bun.file(join("../canvas", "icons.json")).json();
|
||||
const newIcons: Icons = await Bun.file(join(channel.canvasDir, "icons.json")).json();
|
||||
|
||||
const iconDir = { old: join("../canvas", "oldicons"), new: join("../canvas", "icons") };
|
||||
const iconDir = {
|
||||
old: join(channel.canvasDir, "oldicons"),
|
||||
new: join(channel.canvasDir, "icons"),
|
||||
};
|
||||
|
||||
return diffEntries(oldIcons, newIcons, {
|
||||
getSource: (_, val) => join(iconDir.new, val.file),
|
||||
|
|
@ -198,19 +202,23 @@ function parseSource(text: string) {
|
|||
) as Record<string, { size: number; compSize: number }>;
|
||||
}
|
||||
|
||||
async function diffCode() {
|
||||
async function diffCode(channel: ChannelContext) {
|
||||
if (!prevFiles.has("source.jsonl")) throw new Error("Missing prevFile: source.jsonl");
|
||||
|
||||
const oldCode = parseSource(new TextDecoder().decode(prevFiles.get("source.jsonl")));
|
||||
const newCode = parseSource(await Bun.file(join("../data", "source.jsonl")).text());
|
||||
const newCode = parseSource(await Bun.file(join(channel.dataDir, "source.jsonl")).text());
|
||||
|
||||
return diffEntriesCode(oldCode, newCode);
|
||||
}
|
||||
|
||||
export default async function diffs(progress: Progress) {
|
||||
export default async function diffs(channel: ChannelContext, progress: Progress) {
|
||||
await fetchGitChanged();
|
||||
|
||||
if (!(gitChanged.has("version.txt") || diffAnyway)) {
|
||||
const prefix = `${channel.channel}/`;
|
||||
const changed = (name: string) => gitChanged.has(prefix + name);
|
||||
const hasBaseline = (name: string) => prevFiles.has(name as typeof prevFiles extends Map<infer K, any> ? K : never);
|
||||
|
||||
if (!(changed("version.txt") || diffAnyway)) {
|
||||
progress.update("diff", null);
|
||||
progress.update("diff_raw", null);
|
||||
progress.update("diff_semantic", null);
|
||||
|
|
@ -222,30 +230,30 @@ export default async function diffs(progress: Progress) {
|
|||
const differs: Differs = { raw: new Map(), semantic: new Map(), icons: new Map(), code: new Map() };
|
||||
|
||||
await Promise.all([
|
||||
gitChanged.has("raw.json")
|
||||
changed("raw.json") && hasBaseline("raw.json")
|
||||
? wrapPromise(
|
||||
diffRaw().then((x) => (differs.raw = x)),
|
||||
diffRaw(channel).then((x) => (differs.raw = x)),
|
||||
progress,
|
||||
"diff_raw",
|
||||
)
|
||||
: progress.update("diff_raw", null),
|
||||
gitChanged.has("semantic.json")
|
||||
changed("semantic.json") && hasBaseline("semantic.json")
|
||||
? wrapPromise(
|
||||
diffSemantic().then((x) => (differs.semantic = x)),
|
||||
diffSemantic(channel).then((x) => (differs.semantic = x)),
|
||||
progress,
|
||||
"diff_semantic",
|
||||
)
|
||||
: progress.update("diff_semantic", null),
|
||||
gitChanged.has("icons.json")
|
||||
changed("icons.json") && hasBaseline("icons.json")
|
||||
? wrapPromise(
|
||||
diffIcons().then((x) => (differs.icons = x)),
|
||||
diffIcons(channel).then((x) => (differs.icons = x)),
|
||||
progress,
|
||||
"diff_icons",
|
||||
)
|
||||
: progress.update("diff_icons", null),
|
||||
gitChanged.has("source.jsonl")
|
||||
changed("source.jsonl") && hasBaseline("source.jsonl")
|
||||
? wrapPromise(
|
||||
diffCode().then((x) => (differs.code = x)),
|
||||
diffCode(channel).then((x) => (differs.code = x)),
|
||||
progress,
|
||||
"diff_code",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -2,20 +2,16 @@ import { copyFile, mkdir, readdir, rename } from "node:fs/promises";
|
|||
import { basename } from "node:path";
|
||||
import { runInNewContext } from "node:vm";
|
||||
import type { Icons } from "../../types";
|
||||
import type { ChannelContext } from "../channel";
|
||||
import { commit } from "../git";
|
||||
import type { Progress } from "../progress";
|
||||
import { apkSplits, apksFolder, cuteVersion } from "../shared";
|
||||
import { apkSplits } from "../shared";
|
||||
import { discordPath, join, listRequiredDirs, sortObj } from "../utils";
|
||||
|
||||
const infoObjRegex = /({.+?})/;
|
||||
const scalesArrayRegex = /(\[.+?\])/;
|
||||
|
||||
// canvas branch stores icons
|
||||
const iconsJsonPath = "../canvas/icons.json";
|
||||
const iconsDir = "../canvas/icons";
|
||||
const oldIconsDir = "../canvas/oldicons";
|
||||
|
||||
export async function parseAssets(code: string[]) {
|
||||
export async function parseAssets(channel: ChannelContext, code: string[]) {
|
||||
const retrievedAssets: {
|
||||
httpServerLocation: string;
|
||||
width: number;
|
||||
|
|
@ -58,10 +54,12 @@ export async function parseAssets(code: string[]) {
|
|||
|
||||
const apkPaths = new Map<string, string>();
|
||||
for (const split of apkSplits) {
|
||||
const folder = join(apksFolder, split);
|
||||
const folder = join(channel.apksFolder, split);
|
||||
for (const path of await readdir(folder, { recursive: true })) apkPaths.set(basename(path), join(folder, path));
|
||||
}
|
||||
|
||||
const iconsDir = join(channel.canvasDir, "icons");
|
||||
|
||||
const icons: Icons = {};
|
||||
const toCopy: { from: string; to: string }[] = [];
|
||||
for (const asset of retrievedAssets) {
|
||||
|
|
@ -95,21 +93,23 @@ export async function parseAssets(code: string[]) {
|
|||
return { icons, toCopy };
|
||||
}
|
||||
|
||||
export default async function icons(progress: Progress, code: string[]) {
|
||||
export default async function icons(channel: ChannelContext, progress: Progress, code: string[]) {
|
||||
progress.start("icons_getting");
|
||||
|
||||
const { icons, toCopy } = await parseAssets(code);
|
||||
await Bun.write(iconsJsonPath, JSON.stringify(sortObj(icons), undefined, 4));
|
||||
const { icons, toCopy } = await parseAssets(channel, code);
|
||||
await Bun.write(join(channel.canvasDir, "icons.json"), JSON.stringify(sortObj(icons), undefined, 4));
|
||||
progress.update("icons_getting", true);
|
||||
|
||||
progress.start("icons_copying");
|
||||
|
||||
const iconsDir = join(channel.canvasDir, "icons");
|
||||
const oldIconsDir = join(channel.canvasDir, "oldicons");
|
||||
await rename(iconsDir, oldIconsDir).catch(() => {});
|
||||
const dirs = listRequiredDirs(toCopy.map((x) => x.to));
|
||||
|
||||
await Promise.all(dirs.map((dir) => mkdir(dir, { recursive: true })));
|
||||
await Promise.all(toCopy.map(({ from, to }) => copyFile(from, to)));
|
||||
|
||||
await commit(["icons.json", "icons"], `chore: update icons for ${cuteVersion}`, "../canvas");
|
||||
await commit(["icons.json", "icons"], `chore: update icons for ${channel.cuteVersion}`, channel.canvasDir);
|
||||
progress.update("icons_copying", true);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ import {
|
|||
import { drawSections } from "../../canvas";
|
||||
import { makeSections } from "../../canvas/factory";
|
||||
import { type CodeDiff, type Diff, type Differs, DiffType } from "../../types";
|
||||
import { cuteVersion, maxDiffChanges, version } from "../shared";
|
||||
import { ctx } from "../channel";
|
||||
import { maxDiffChanges } from "../shared";
|
||||
import { formatBytes, sortEntries } from "../utils";
|
||||
|
||||
function fileBase(path: string, other?: string) {
|
||||
|
|
@ -143,7 +144,7 @@ async function sendWebhook(webhook: string, role: string, embeds: WebhookEmbed[]
|
|||
components: [
|
||||
{
|
||||
type: ComponentType.TextDisplay,
|
||||
content: `## ${title}\n-# ${version} (${cuteVersion})`,
|
||||
content: `## ${title}\n-# ${ctx().version} (${ctx().cuteVersion})`,
|
||||
},
|
||||
...body.embed.map((content) => ({
|
||||
type: ComponentType.TextDisplay,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue