diff --git a/.env.TEMPLATE b/.env.TEMPLATE index e07d0e013c5..7aadc5d0b40 100644 --- a/.env.TEMPLATE +++ b/.env.TEMPLATE @@ -2,5 +2,3 @@ bot_token="Bot ................." # these are required code_webhook="https://discord.com/api/webhooks/.../......" -color_webhook="https://discord.com/api/webhooks/.../......" -icons_webhook="https://discord.com/api/webhooks/.../......" \ No newline at end of file diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 4f09a66789b..ced6788bacc 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -29,9 +29,7 @@ jobs: ref: data path: data - - uses: actions/setup-python@v6 - with: - python-version: "3.14.1" + - uses: dtolnay/rust-toolchain@stable - uses: oven-sh/setup-bun@v2 @@ -56,8 +54,6 @@ jobs: if: ${{ steps.check.outputs.version != 'false' }} env: code_webhook: ${{ secrets.CODE_WEBHOOK }} - color_webhook: ${{ secrets.COLOR_WEBHOOK }} - icons_webhook: ${{ secrets.ICONS_WEBHOOK }} bot_token: ${{ secrets.BOT_TOKEN }} working-directory: main run: bun run update:prod ${{ github.event.inputs.quiet == 'true' && '--quiet' || '' }} diff --git a/README.md b/README.md index 4723b520e1b..4a3682707d2 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,11 @@
- - Discord server - -

🎨 Themelings

+

Themelings

-A repo that tracks the [Discord](https://github.com/discord) Android app's code, color and icon changes! +A repo that tracks the [Discord](https://github.com/discord) Android app's code changes! For a closer look at what's scraped, take a look at the [data](https://github.com/nexpid/Themelings/tree/data) branch ## Credits -- [hermes-dec](https://github.com/P1sec/hermes-dec) — tool for decompiling hermes React Native bytecode \ No newline at end of file +- [hermes-decomp](https://github.com/SymbioticSec/hermes-decomp) — tool for decompiling hermes React Native bytecode diff --git a/local.env.TEMPLATE b/local.env.TEMPLATE index 630594017ff..ce47e914e48 100644 --- a/local.env.TEMPLATE +++ b/local.env.TEMPLATE @@ -4,5 +4,3 @@ NODE_ENV="test" bot_token="Bot ................." # all values below are optional, they are overrides for the base .env file code_webhook="https://discord.com/api/webhooks/.../......" -color_webhook="https://discord.com/api/webhooks/.../......" -icons_webhook="https://discord.com/api/webhooks/.../......" \ No newline at end of file diff --git a/package.json b/package.json index 73301f77001..a5bca90f287 100644 --- a/package.json +++ b/package.json @@ -7,21 +7,14 @@ "update:commit": "bun run --env-file local.env src/update/index.ts --commit-anyway", "update:mock": "bun run --env-file local.env src/update/index.ts --mock", "update:prod": "bun run src/update/index.ts", - "canvas:test": "bun run --watch src/canvas/index.test.ts", - "deminify:test": "bun run src/update/deminify.test.ts", - "colors:test": "bun run src/update/tasks/colors.test.ts", - "icons:test": "bun run src/update/tasks/icons.test.ts", "format": "biome check --fix ." }, "dependencies": { - "@napi-rs/canvas": "^0.1.97", - "color": "^5.0.3", "discord-api-types": "^0.38.46" }, "devDependencies": { "@biomejs/biome": "^2.4.11", - "@types/bun": "^1.3.12", - "@types/color": "^4.2.1" + "@types/bun": "^1.3.12" }, "peerDependencies": { "typescript": "^6.0.2" diff --git a/src/canvas/factory.ts b/src/canvas/factory.ts deleted file mode 100644 index 112b56b3677..00000000000 --- a/src/canvas/factory.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { join } from "node:path"; -import { type Image, loadImage } from "@napi-rs/canvas"; -import { type Diff, DiffType } from "../types"; - -export type Item = - | { - type: "color"; - colors: string[][]; - } - | { - type: "image"; - image: Image; - } - | { - type: "blob"; - blob: string; - }; - -export interface Row { - title?: string; - subtitle?: string; - item: Item; -} - -export interface Section { - header: string; - columns: Row[][]; -} - -function placeholderFile(file: string) { - return file.endsWith(".lottie") ? join(import.meta.dir, "placeholders/lottie.png") : file; -} - -const numColumns = 5; -const maxRows = 3; - -async function makeSection(diffs: Map, isFile: boolean, type: DiffType, header: string) { - const filtered = [...diffs.entries()].filter(([, diff]) => diff.type === type); - const section: Section = { - header, - columns: Array.from({ length: numColumns }, () => []), - }; - - let i = 0; - for (const [name, diff] of filtered) { - const columnInd = i++ % numColumns; - - const column = section.columns[columnInd], - nextColumn = section.columns[i % numColumns]; - if (!column) throw new Error(`Couldn't find column of index ${columnInd}`); - - if (!nextColumn || nextColumn.length >= maxRows) { - column.push({ - item: { - type: "blob", - blob: `+${(filtered.length - i + 1).toLocaleString("en-US")} ${header.toLowerCase()}`, - }, - }); - break; - } - - const changed = diff.type === DiffType.Changed; - const item: Item = isFile - ? { type: "image", image: await loadImage(placeholderFile(diff.source)) } - : { type: "color", colors: [changed && diff.oldSource.split(","), diff.source.split(",")].filter((x) => !!x) }; - - column.push( - ...[ - changed && - isFile && { - subtitle: name, - item: { - type: "image", - image: await loadImage(placeholderFile(diff.oldSource)), - } as Item, - }, - { - title: name, - subtitle: diff.type === DiffType.Renamed ? diff.oldName : undefined, - item, - }, - ].filter((x) => !!x), - ); - } - - section.columns = section.columns.filter((x) => x.length); - return section; -} - -export async function makeSections(diffs: Map, isFile = false) { - return [ - await makeSection(diffs, isFile, DiffType.Added, "Added"), - await makeSection(diffs, isFile, DiffType.Changed, "Changed"), - await makeSection(diffs, isFile, DiffType.Renamed, "Renamed"), - await makeSection(diffs, isFile, DiffType.Removed, "Removed"), - ].filter((x) => x.columns[0]); -} diff --git a/src/canvas/fonts/ggmono/normal-400.ttf b/src/canvas/fonts/ggmono/normal-400.ttf deleted file mode 100644 index 3a2aa0cb035..00000000000 Binary files a/src/canvas/fonts/ggmono/normal-400.ttf and /dev/null differ diff --git a/src/canvas/fonts/ggsans/normal-400.ttf b/src/canvas/fonts/ggsans/normal-400.ttf deleted file mode 100644 index 45eefcd3c69..00000000000 Binary files a/src/canvas/fonts/ggsans/normal-400.ttf and /dev/null differ diff --git a/src/canvas/fonts/ggsans/semibold-600.ttf b/src/canvas/fonts/ggsans/semibold-600.ttf deleted file mode 100644 index 28e88c38c8e..00000000000 Binary files a/src/canvas/fonts/ggsans/semibold-600.ttf and /dev/null differ diff --git a/src/canvas/index.test.ts b/src/canvas/index.test.ts deleted file mode 100644 index 5e531d247e4..00000000000 --- a/src/canvas/index.test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import mock from "../mock"; -import { drawSections } from "."; -import { makeSections } from "./factory"; - -await Bun.write( - "temp/files.png", - new Uint8Array(drawSections(await makeSections(mock.icons, true)).toBuffer("image/png")), -); -await Bun.write( - "temp/colors.png", - new Uint8Array(drawSections(await makeSections(mock.semantic)).toBuffer("image/png")), -); diff --git a/src/canvas/index.ts b/src/canvas/index.ts deleted file mode 100644 index 16fa988499d..00000000000 --- a/src/canvas/index.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { readdir } from "node:fs/promises"; -import { join } from "node:path"; -import { createCanvas, GlobalFonts } from "@napi-rs/canvas"; -import { file } from "bun"; -import type { Row, Section } from "./factory"; -import { measureText, transformSvg, wrapText } from "./utils"; - -async function bulkRegister(family: string, path: string) { - for (const item of await readdir(path)) GlobalFonts.registerFromPath(join(path, item), family); -} -await bulkRegister("GG Sans", join(import.meta.dir, "fonts/ggsans")); -await bulkRegister("GG Mono", join(import.meta.dir, "fonts/ggmono")); - -function calcColumnWidth(column: { textWidth: number; itemWidth: number }[]) { - return Math.max(0, ...column.map(({ textWidth, itemWidth }) => Math.max(0, textWidth, itemWidth))); -} - -function calcColumnHeight(column: Row[]) { - const texts = column - .map((row) => Number(!!row.title) + Number(!!row.subtitle) + Number(row.item.type === "blob")) - .reduce((a, b) => a + b, 0); - const items = texts + column.length - 1; - return texts * textSizes.title + column.length * itemSize + items * layout.card.gapY; -} - -function calcSectionHeight(columns: Row[][]) { - return Math.max(0, ...columns.map((column) => calcColumnHeight(column))); -} - -const layout = { - background: { - padding: 24, - gap: 18, - }, - section: { - gap: 8, - }, - card: { - radius: 24, - padding: 12, - gapX: 16, - gapY: 8, - }, - blob: { - radius: 24 - 12, - }, - color: { - radius: 24, - }, -}; - -const textSizes = { - header: 24, - title: 18, - blob: 24, -}; - -const colors = { - background: "#1c1d23", // BACKGROUND_BASE_LOWEST - section: "#26272f", // CARD_BACKGROUND_DEFAULT - blob: "#6c6f7c1f", // BACKGROUND_MOD_MUTED - textNormal: "#c7c8ce", // TEXT_NORMAL - textMuted: "#818491", // TEXT_MUTED - watermark: "#2f3035", // TEXT_OVERLAY_DARK -}; - -const itemSize = 128; -const blobSize = itemSize + layout.card.gapY + textSizes.title; - -const watermark = await transformSvg(await file(join(import.meta.dir, "placeholders/watermark.svg")).text(), { - color: colors.watermark, - height: layout.background.padding, -}); - -const mctx = createCanvas(1, 1).getContext("2d"); -export function drawSections(sections: Section[]) { - const headerFont = `600 ${textSizes.header}px GG Sans`; - const titleFont = `400 ${textSizes.title}px GG Mono`; - - const sects = sections.map(({ header, columns }) => ({ - headerWidth: measureText(mctx, headerFont, header).width, - header, - columns: columns.map((column) => - column.map(({ title, subtitle, item }) => ({ - title, - subtitle, - textWidth: Math.max( - 0, - title ? measureText(mctx, titleFont, title).width : 0, - subtitle ? measureText(mctx, titleFont, subtitle).width : 0, - ), - item, - itemWidth: - item.type === "blob" - ? blobSize - : item.type === "image" - ? (item.image.width / item.image.height) * itemSize - : itemSize, - })), - ), - })); - const columnWidths = Array.from({ length: Math.max(0, ...sects.map(({ columns }) => columns.length)) }, (_, i) => - Math.max(0, ...sects.map(({ columns }) => (columns[i] ? calcColumnWidth(columns[i]) : 0))), - ); - const sectionWidth = - columnWidths.reduce((a, b) => a + b, 0) + layout.card.gapX * (columnWidths.length - 1) + layout.card.padding * 2; - - const width = Math.max(0, ...sects.map(({ headerWidth }) => headerWidth), sectionWidth); - const height = - sects - .map( - ({ columns }) => - textSizes.header + - layout.section.gap + - layout.card.padding * 2 + - calcSectionHeight(columns) + - layout.background.gap, - ) - .reduce((a, b) => a + b, 0) - layout.background.gap; - - const canvas = createCanvas(layout.background.padding * 2 + width, layout.background.padding * 2 + height); - const ctx = canvas.getContext("2d"); - ctx.textBaseline = "top"; - - ctx.fillStyle = colors.background; - ctx.fillRect(0, 0, canvas.width, canvas.height); - - let x = layout.background.padding, - y = layout.background.padding; - for (const sect of sects) { - ctx.fillStyle = colors.textNormal; - ctx.font = headerFont; - ctx.fillText(sect.header, x, y); - y += textSizes.header + layout.section.gap; - - const sectionHeight = calcSectionHeight(sect.columns); - ctx.fillStyle = colors.section; - ctx.beginPath(); - ctx.roundRect(x, y, sectionWidth, sectionHeight + layout.card.padding * 2, layout.card.radius); - ctx.fill(); - y += layout.card.padding; - x += layout.card.padding; - - let wx = x; - for (let i = 0; i < sect.columns.length; i++) { - let wy = y; - - const column = sect.columns[i]; - const columnWidth = columnWidths[i]; - if (!columnWidth) throw new Error(`Failed to get columnWidth for ${i}, ${columnWidth}`); - - const centerX = wx + columnWidth / 2; - for (const row of column) { - ctx.font = titleFont; - if (row.subtitle) { - ctx.fillStyle = colors.textMuted; - ctx.fillText(row.subtitle, centerX - row.textWidth / 2, wy); - wy += textSizes.title + layout.card.gapY; - } - if (row.title) { - ctx.fillStyle = colors.textNormal; - ctx.fillText(row.title, centerX - row.textWidth / 2, wy); - wy += textSizes.title + layout.card.gapY; - } - - const midX = centerX - row.itemWidth / 2; - if (row.item.type === "blob") { - const blobHeight = sectionHeight - wy + y; - - ctx.fillStyle = colors.blob; - ctx.beginPath(); - ctx.roundRect(wx, wy, columnWidth, blobHeight, layout.blob.radius); - ctx.fill(); - - ctx.font = `600 ${textSizes.blob}px GG Sans`; - ctx.fillStyle = colors.textNormal; - ctx.textAlign = "center"; - ctx.textBaseline = "middle"; - - const wrapped = wrapText(ctx, row.item.blob, columnWidth); - const height = textSizes.blob / 2; - for (let i = 0; i < wrapped.length; i++) - ctx.fillText( - wrapped[i], - wx + columnWidth / 2, - wy + blobHeight / 2 - height * (wrapped.length - 1) + textSizes.blob * i, - ); - - ctx.textAlign = "left"; - ctx.textBaseline = "top"; - } else if (row.item.type === "image") { - ctx.drawImage(row.item.image, midX, wy, row.itemWidth, itemSize); - wy += itemSize + layout.card.gapY; - } else if (row.item.type === "color") { - for (let i = 0; i < row.item.colors.length; i++) { - const colors = row.item.colors[i]; - for (let j = 0; j < colors.length; j++) { - const color = colors[j]; - ctx.save(); - ctx.beginPath(); - ctx.rect( - Math.floor(midX + (itemSize / colors.length) * j), - Math.floor(wy + (itemSize / row.item.colors.length) * i), - Math.ceil(itemSize / colors.length), - Math.ceil(itemSize / row.item.colors.length), - ); - ctx.clip(); - - ctx.beginPath(); - ctx.fillStyle = color; - ctx.roundRect(midX, wy, itemSize, itemSize, layout.color.radius); - ctx.fill(); - ctx.restore(); - } - } - wy += itemSize + layout.card.gapY; - } - } - wx += columnWidth + layout.card.gapX; - } - y += sectionHeight; - - x -= layout.card.padding; - y += layout.card.padding; - y += layout.background.gap; - } - - // watermark - ctx.drawImage(watermark, canvas.width - watermark.width - 1, 1); - - return canvas; -} diff --git a/src/canvas/mock/FirstMockIcon.png b/src/canvas/mock/FirstMockIcon.png deleted file mode 100644 index 580037001a0..00000000000 Binary files a/src/canvas/mock/FirstMockIcon.png and /dev/null differ diff --git a/src/canvas/mock/FourthMockIcon.png b/src/canvas/mock/FourthMockIcon.png deleted file mode 100644 index 4bb486b21c8..00000000000 Binary files a/src/canvas/mock/FourthMockIcon.png and /dev/null differ diff --git a/src/canvas/mock/LottieMockIcon.lottie b/src/canvas/mock/LottieMockIcon.lottie deleted file mode 100644 index a2a4fc1216a..00000000000 Binary files a/src/canvas/mock/LottieMockIcon.lottie and /dev/null differ diff --git a/src/canvas/mock/SecondMockIcon.png b/src/canvas/mock/SecondMockIcon.png deleted file mode 100644 index 0d7eab30520..00000000000 Binary files a/src/canvas/mock/SecondMockIcon.png and /dev/null differ diff --git a/src/canvas/mock/SecondMockIconNew.png b/src/canvas/mock/SecondMockIconNew.png deleted file mode 100644 index f83d0103c6b..00000000000 Binary files a/src/canvas/mock/SecondMockIconNew.png and /dev/null differ diff --git a/src/canvas/mock/SvgMockIcon.svg b/src/canvas/mock/SvgMockIcon.svg deleted file mode 100644 index b3a9647b356..00000000000 --- a/src/canvas/mock/SvgMockIcon.svg +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/src/canvas/mock/ThirdMockIcon.png b/src/canvas/mock/ThirdMockIcon.png deleted file mode 100644 index 65718d5454b..00000000000 Binary files a/src/canvas/mock/ThirdMockIcon.png and /dev/null differ diff --git a/src/canvas/placeholders/lottie.png b/src/canvas/placeholders/lottie.png deleted file mode 100644 index 1a69318a9b1..00000000000 Binary files a/src/canvas/placeholders/lottie.png and /dev/null differ diff --git a/src/canvas/placeholders/watermark.svg b/src/canvas/placeholders/watermark.svg deleted file mode 100644 index 7d336c52399..00000000000 --- a/src/canvas/placeholders/watermark.svg +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/src/canvas/utils.ts b/src/canvas/utils.ts deleted file mode 100644 index 73d93bbdde0..00000000000 --- a/src/canvas/utils.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { type CanvasRenderingContext2D, loadImage } from "@napi-rs/canvas"; - -export function measureText(ctx: CanvasRenderingContext2D, font: string, text: string) { - ctx.font = font; - return ctx.measureText(text); -} - -export function wrapText(ctx: CanvasRenderingContext2D, text: string, maxWidth: number) { - // https://stackoverflow.com/a/16599668 - const words = text.split(" "); - let lines: string[] = [], - line = words[0]; - - for (let i = 1; i < words.length; i++) { - const word = words[i]; - const newText = `${line} ${word}`; - - const width = ctx.measureText(newText).width; - if (width < maxWidth) line = newText; - else { - lines.push(line); - line = word; - } - } - - lines.push(line); - return lines; -} - -export async function transformSvg(svg: string, config: { color?: string; height?: number; width?: number }) { - const text = config.color ? svg.replace(/fill="#[a-z0-9]{3,6}"/gi, `fill="${config.color}"`) : svg; - - const image = await loadImage(Buffer.from(text)); - if (config.width && config.height) { - image.width = config.width; - image.height = config.height; - } else if (config.width) { - image.height = (image.height / image.width) * config.width; - image.width = config.width; - } else if (config.height) { - image.width = (image.width / image.height) * config.height; - image.height = config.height; - } - return image; -} diff --git a/src/mock.ts b/src/mock.ts index a7f1a07a7b6..c25b1d25c1b 100644 --- a/src/mock.ts +++ b/src/mock.ts @@ -1,125 +1,6 @@ -import Color from "color"; -import { type CodeDiff, type Diff, DiffType } from "./types"; +import { type CodeDiff, DiffType } from "./types"; -const randomColor = () => - `#${Math.floor(Math.random() * 0xffffff) - .toString(16) - .padStart(6, "0")}`; -const randomColors = () => { - const base = Color(randomColor()); - return [ - base.lighten(0.6).hex(), - base.desaturate(0.3).darken(0.5).hex(), - base.darken(0.6).hex(), - base.darken(0.8).hex(), - ].join(","); -}; - -const semNum = 20; - -// mock data for "bun run update:mock" and "bun run canvas:test" export default { - semantic: new Map([ - ...Array.from({ length: semNum }, (_, i) => [ - `MOCK_SEMANTIC_SAMPLE_${i + 1}`, - { - type: DiffType.Added, - source: randomColors(), - label: "☀️ #aaaaaa, 🌙 #bbbbbb", - } as Diff, - ]), - ...Array.from({ length: semNum }, (_, i) => [ - `MOCK_SEMANTIC_SAMPLE_${i + 1 + semNum}`, - { - type: DiffType.Changed, - source: randomColors(), - label: "☀️ #aaaaaa, 🌙 #bbbbbb", - oldSource: randomColors(), - oldLabel: "☀️ #888888, 🌙 #999999", - } as Diff, - ]), - ...Array.from({ length: semNum }, (_, i) => [ - `MOCK_SEMANTIC_SAMPLE_${i + 1 + semNum * 2}`, - { - type: DiffType.Renamed, - oldName: `mocked_samentic_sample_${i + 1 + semNum * 2}`, - source: randomColors(), - } as Diff, - ]), - ...Array.from({ length: semNum }, (_, i) => [ - `MOCK__SEMANTIC__REMOVED__SAMPLE__${i + 1 + semNum * 3}`, - { - type: DiffType.Removed, - source: randomColors(), - label: "☀️ #aaaaaa, 🌙 #bbbbbb", - } as Diff, - ]), - ] as [string, Diff][]), - semanticBig: new Map( - Array.from({ length: 35 }, (_, i) => [ - `SOME_EXAMPLE_SEMANTIC_${i}`, - { - type: DiffType.Changed, - source: randomColors(), - label: "☀️ #aaaaaa, 🌙 #bbbbbb", - oldSource: randomColors(), - oldLabel: "☀️ #888888, 🌙 #999999", - } as Diff, - ]), - ), - raw: new Map(), - icons: new Map([ - [ - "FirstMockIcon", - { - type: DiffType.Added, - source: "src/canvas/mock/FirstMockIcon.png", - label: "c6f6357f", - } as Diff, - ], - [ - "SvgMockIcon", - { - type: DiffType.Added, - source: "src/canvas/mock/SvgMockIcon.svg", - label: "5d534560", - } as Diff, - ], - [ - "LottieMockIcon", - { - type: DiffType.Added, - source: "src/canvas/mock/LottieMockIcon.lottie", - label: "340efc52", - } as Diff, - ], - [ - "SecondMockIcon", - { - type: DiffType.Changed, - source: "src/canvas/mock/SecondMockIconNew.png", - label: "76ffeecd", - oldSource: "src/canvas/mock/SecondMockIcon.png", - oldLabel: "6e235744", - } as Diff, - ], - [ - "ThirdMockIcon", - { - type: DiffType.Renamed, - oldName: "thirdmkocicon", - source: "src/canvas/mock/ThirdMockIcon.png", - } as Diff, - ], - [ - "FourthMockIcon", - { - type: DiffType.Removed, - source: "src/canvas/mock/FourthMockIcon.png", - label: "cf4e1340", - } as Diff, - ], - ]), code: new Map([ [ "app/foomodule.tsx", diff --git a/src/types.ts b/src/types.ts index 2994c6feaf3..34640cd5a1e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,25 +5,6 @@ export enum DiffType { Removed, } -export type Diff = - | { - type: DiffType.Added | DiffType.Removed; - source: string; - label?: string; - } - | { - type: DiffType.Changed; - source: string; - label?: string; - oldSource: string; - oldLabel?: string; - } - | { - type: DiffType.Renamed; - oldName: string; - source: string; - }; - export type CodeDiff = | { type: DiffType.Added | DiffType.Removed; @@ -35,22 +16,6 @@ export type CodeDiff = size: number; }; -export type Differs = Record<"semantic" | "raw" | "icons", Map | undefined> & { +export type Differs = { code: Map | undefined; }; - -export type RawColors = Record; - -export type Icons = Record< - string, - { - file: string; - hash: string; - scales: number[]; - width: number | null; - height: number | null; - } ->; - -export type Semantic = Record; -export type SemanticColors = Record; diff --git a/src/update/deminify.test.ts b/src/update/deminify.test.ts deleted file mode 100644 index e0e691789b3..00000000000 --- a/src/update/deminify.test.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { deminify } from "./deminify"; - -const SNIPPET = `r7 = function(a0, a1, a2, a3, a4, a5, a6) { // Environment: r3 - r4 = a1; - r2 = a5; - r5 = a6; - var _closure1_slot0 = r4; - r0 = a2; - var _closure1_slot1 = r0; - var _closure1_slot2 = r5; - r0 = global; - r7 = r0.Object; - r6 = r7.defineProperty; - r1 = {}; - r0 = true; - r1['value'] = r0; - r0 = '__esModule'; - r0 = r6.bind(r7)(r2, r0, r1); - r0 = 0; - r6 = r5[r0]; - r1 = a3; - r0 = undefined; - r1 = r1.bind(r0)(r6); - r1 = 1; - r1 = r5[r1]; - r1 = r4.bind(r0)(r1); - r1 = r1.jsx; - var _closure1_slot3 = r1; - r1 = {}; - r6 = function(a0) { // Original name: openLazy, environment: r3 - _fun32591: for(var _fun32591_ip = 0; ; ) switch(_fun32591_ip) { -case 0: - r3 = a0; - r2 = r3.importer; - r4 = r3.hideActionSheet; - r1 = undefined; - if(!(r4 === r1)) { _fun32591_ip = 25; continue _fun32591 } -case 23: - r4 = true; -case 25: - var _closure2_slot0 = r4; - r3 = r3.isDismissable; - if(!(r3 === r1)) { _fun32591_ip = 41; continue _fun32591 } -case 39: - r3 = false; -case 41: - var _closure2_slot1 = r3; - r2 = r2.bind(r1)(); - r1 = r2.then; - r0 = function(a0) { // Environment: r0 - _fun32592: for(var _fun32592_ip = 0; ; ) switch(_fun32592_ip) { -case 0: - r0 = _closure2_slot0; - if(!r0) { _fun32592_ip = 45; continue _fun32592 } -case 10: - r2 = _closure1_slot1; - r1 = _closure1_slot2; - r0 = 2; - r1 = r1[r0]; - r0 = undefined; - r1 = r2.bind(r0)(r1); - r0 = r1.hideActionSheet; - r0 = r0.bind(r1)(); -case 45: - r2 = _closure1_slot1; - r1 = _closure1_slot2; - r0 = 3; - r1 = r1[r0]; - r0 = undefined; - r3 = r2.bind(r0)(r1); - r2 = r3.dispatch; - r1 = {}; - r5 = 'ALERT_OPEN'; - r1['type'] = r5; - r5 = a0; - r1['alert'] = r5; - r4 = _closure2_slot1; - r1['isDismissable'] = r4; - r1 = r2.bind(r3)(r1); - return r0; - } - }; - r0 = r1.bind(r2)(r0); - return r0; - } - }; - r1['openLazy'] = r6; - r6 = function() { // Original name: close, environment: r3 - r2 = _closure1_slot1; - r1 = _closure1_slot2; - r0 = 3; - r1 = r1[r0]; - r0 = undefined; - r3 = r2.bind(r0)(r1); - r2 = r3.dispatch; - r1 = {}; - r4 = 'ALERT_CLOSE'; - r1['type'] = r4; - r1 = r2.bind(r3)(r1); - return r0; - }; - r1['close'] = r6; - r6 = function(a0) { // Original name: show, environment: r3 - _fun32594: for(var _fun32594_ip = 0; ; ) switch(_fun32594_ip) { -case 0: - r7 = a0; - r3 = this; - r5 = r7.hideActionSheet; - r4 = r7.isDismissable; - r1 = r7.confirmText; - r0 = undefined; - if(!(r1 === r0)) { _fun32594_ip = 90; continue _fun32594 } -case 32: - r10 = _closure1_slot0; - r11 = _closure1_slot2; - r2 = 4; - r8 = r11[r2]; - r8 = r10.bind(r0)(r8); - r9 = r8.intl; - r8 = r9.string; - r2 = r11[r2]; - r2 = r10.bind(r0)(r2); - r2 = r2.t; - r2 = r2.BddRzc; - r1 = r8.bind(r9)(r2); -case 90: - var _closure2_slot0 = r1; - r2 = {'hideActionSheet': 0, 'isDismissable': 0, 'confirmText': 0}; - r13 = null; - r14 = r2; - r1 = silentSetPrototypeOf(r14, r13); - r14 = {}; - r13 = r7; - r12 = r2; - r1 = copyDataProperties(r14, r13, r12); - var _closure2_slot1 = r1; - r2 = r3.openLazy; - r1 = {}; - r6 = function() { // Original name: importer, environment: r6 - r2 = _closure1_slot0; - r0 = _closure1_slot2; - r1 = 6; - r1 = r0[r1]; - r3 = undefined; - r2 = r2.bind(r3)(r1); - r1 = 5; - r1 = r0[r1]; - r0 = r0.paths; - r2 = r2.bind(r3)(r1, r0); - r1 = r2.then; - r0 = function(a0) { // Environment: r0 - r1 = a0; - r1 = r1.default; - var _closure4_slot0 = r1; - r0 = function(a0) { // Environment: r0 - r3 = _closure1_slot3; - r2 = _closure4_slot0; - r1 = {}; - r5 = a0; - r6 = r1; - r0 = copyDataProperties(r6, r5); - r5 = _closure2_slot1; - r6 = r1; - r4 = copyDataProperties(r6, r5); - r4 = _closure2_slot0; - r0 = 'confirmText'; - r1[r0] = r4; - r0 = undefined; - r0 = r3.bind(r0)(r2, r1); - return r0; - }; - return r0; - }; - r0 = r1.bind(r2)(r0); - return r0; - }; - r1['importer'] = r6; - r1['hideActionSheet'] = r5; - r1['isDismissable'] = r4; - r1 = r2.bind(r3)(r1); - return r0; - } - }; - r1['show'] = r6; - r3 = function(a0) { // Original name: confirm, environment: r3 - r1 = this; - var _closure2_slot0 = r1; - r1 = a0; - var _closure2_slot1 = r1; - r1 = global; - r2 = r1.Promise; - r1 = r2.prototype; - r1 = Object.create(r1, {constructor: {value: r2}}); - r3 = function(a0) { // Environment: r0 - r0 = a0; - var _closure3_slot0 = r0; - r3 = _closure2_slot0; - r2 = r3.show; - r1 = {}; - r0 = function() { // Original name: onConfirm, environment: r5 - r2 = _closure3_slot0; - r0 = undefined; - r1 = true; - r1 = r2.bind(r0)(r1); - return r0; - }; - r1['onConfirm'] = r0; - r9 = _closure1_slot0; - r10 = _closure1_slot2; - r6 = 4; - r7 = r10[r6]; - r0 = undefined; - r7 = r9.bind(r0)(r7); - r8 = r7.intl; - r7 = r8.string; - r6 = r10[r6]; - r6 = r9.bind(r0)(r6); - r6 = r6.t; - r6 = r6.ETE/oK; - r6 = r7.bind(r8)(r6); - r1['cancelText'] = r6; - r5 = function() { // Original name: onCancel, environment: r5 - r2 = _closure3_slot0; - r0 = undefined; - r1 = false; - r1 = r2.bind(r0)(r1); - return r0; - }; - r1['onCancel'] = r5; - r11 = _closure2_slot1; - r12 = r1; - r4 = copyDataProperties(r12, r11); - r1 = r2.bind(r3)(r1); - return r0; - }; - r4 = r1; - r0 = new r4[r2](r3, r2); - r0 = r0 instanceof Object ? r0 : r1; - return r0; - }; - r1['confirm'] = r3; - r3 = 7; - r3 = r5[r3]; - r5 = r4.bind(r0)(r3); - r4 = r5.fileFinishedImporting; - r3 = 'actions/native/AlertActionCreators.tsx'; - r3 = r4.bind(r5)(r3); - r2['default'] = r1; - return r0; -};`; - -await Bun.write("temp/deminified.js", deminify(SNIPPET, "actions/native/AlertActionCreators.js").res); diff --git a/src/update/deminify.ts b/src/update/deminify.ts deleted file mode 100644 index 4379439ad7b..00000000000 --- a/src/update/deminify.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { writeFileSync } from "node:fs"; -import { extname } from "node:path"; -import { log } from "./progress"; - -const variableMatch = /\br(\d{1,2})\b/g; -const argumentMatch = /\ba(\d{1,2})\b/g; - -const funMatch = /\b(_fun)(\d+)(_ip)?(?: = (\d+))?\b/g; -const caseMatch = /^case (\d+):/g; - -const functNameMatch = /(function\*?) ?(\(.*?\) {) \/\/ Original name: +\??(.+), environment: \w+$/g; -const functOpenMatch = /function\*? ?\((.*?)\) {/; -const functCloseMatch = /^\s+};/; - -const stringMatch = /(["'`])(?:\\.|[^\\])*?\1/g; -// biome-ignore lint/suspicious/noControlCharactersInRegex: this is needed -const nullstringMatch = /(["'`])\x00(\d+)\x00\1/g; - -const nativeModuleNames = [ - "global", - "require", - "metroImportDefault", - "metroImportAll", - "moduleObject", - "exports", - "dependencyMap", -]; - -export function deminify(code: string, path: string) { - const cleaned = code - .split(" = ") - .slice(1) - .join(" = ") - .slice(0, -1) - .replace(/ \/\/ Environment: \w+$/gm, "") - .split("\n"); - - const lines: string[] = []; - - let depth = 0; - const localArgs = new Map(); - const funMap = new Map(); - const caseMap = new Map(); - caseMap.set("0", 0); - - const varReplacer = (_: string, id: string) => `var${Number(id) + 1}`; - const argReplacer = (_: string, id: string) => { - const n = Number(id), - depth = localArgs.get(n); - if (!depth) log(`WARNING! arg${n} does not have any depth (${path})`); - - if (depth === 1) return nativeModuleNames[n] ?? `native${n + 1}`; - return `arg${n + 1}`; - }; - const funReplacer = (_: string, prefix: string, id: string, suffix: string, caseId?: string) => { - const num = funMap.get(id) ?? String(funMap.size + 1).padStart(4, "0"); - funMap.set(id, num); - - const text = `${prefix}${num}${suffix ?? ""}`; - if (!caseId) return text; - - const caseNum = caseMap.get(caseId) ?? caseMap.size + 1; - caseMap.set(caseId, caseNum); - return `${text} = ${caseNum}`; - }; - const caseReplacer = (_: string, id: string) => { - const num = caseMap.get(id) ?? caseMap.size + 1; - caseMap.set(id, num); - return `case ${num}:`; - }; - const functNameReplacer = (_: string, prefix: string, stuff: string, name: string) => `${prefix} ${name}${stuff}`; - - let j = 1; - for (const raw of cleaned) { - const nullStrings = new Map(); - const line = raw.replace(stringMatch, (match, quot: string) => { - const index = nullStrings.size + 1; - nullStrings.set(String(index), match); - return `${quot}\x00${index}\x00${quot}`; - }); - - // check for start of function - if (functOpenMatch.test(line)) { - const openedArgs = line.match(functOpenMatch)?.[1]?.split(", ")?.length || 0; - - depth++; - for (let i = 0; i < openedArgs; i++) { - localArgs.set(i, depth); - } - } - - // check for end of function - if (functCloseMatch.test(line)) { - for (const arg of localArgs.keys()) { - const n = localArgs.get(arg); - if (n === depth) localArgs.set(arg, n - 1); - } - depth--; - } - - // sanity check - const y = j++; - if (depth < 0) { - writeFileSync("temp/depthcrashed.js", cleaned.join("\n")); - throw new Error(`parsing failed, depth went below 0 (LINE: ${y}, FILE: ${path})`); - } - - const parsed = line - .replace(variableMatch, varReplacer) - .replace(argumentMatch, argReplacer) - .replace(funMatch, funReplacer) - .replace(caseMatch, caseReplacer) - .replace(functNameMatch, functNameReplacer) - .replace(nullstringMatch, (_, __, id: string) => { - const nullString = nullStrings.get(id); - if (nullString) return nullString; - writeFileSync("temp/nullcrashed.js", cleaned.join("\n")); - throw new Error(`parsing failed, nullstring ${id} doesn't exist (LINE: ${y}, FILE: ${path})`); - }); - lines.push(parsed); - } - - const final = lines.join("\n"); - return { - final, - res: `// ${path}\n${extname(path).startsWith(".ts") ? "export default" : "module.exports ="} (${final})();`, - }; -} diff --git a/src/update/git.ts b/src/update/git.ts index bb592e2ba6f..f3c4a083a9d 100644 --- a/src/update/git.ts +++ b/src/update/git.ts @@ -5,7 +5,7 @@ export const gitChanged = new Set(); export async function fetchGitChanged() { for (const changed of ( - await Bun.$`git status -z -- ':!oldicons' ':!icons' ':!source'`.cwd("../data").quiet().then(handleShellErr) + await Bun.$`git status -z -- ':!source'`.cwd("../data").quiet().then(handleShellErr) ) .text() .split("\x00") @@ -27,16 +27,13 @@ export async function commit(files: string[], message: string) { if (process.env.NODE_ENV === "test" && !commitAnyway) { await fetchGitChanged(); } else { - // unstage all files await Bun.$`git restore --staged .`.cwd("../data").nothrow().quiet().then(handleShellErr); await fetchGitChanged(); - // stage files await Bun.$`git add ${{ raw: files.map((x) => Bun.$.escape(x)).join(" ") }}` .cwd("../data") .nothrow() .quiet() .then(handleShellErr); - // commiiiiiit await Bun.$`git commit -m ${message}` .cwd("../data") .nothrow() diff --git a/src/update/runner.ts b/src/update/runner.ts index 9c010b3af58..13f229740a9 100644 --- a/src/update/runner.ts +++ b/src/update/runner.ts @@ -5,11 +5,8 @@ import { commit } from "./git"; import { log, makeProgress, wrapPromise } from "./progress"; import { apksFolder, codePath, cuteVersion, isMock, isQuiet, oprevFiles, prevFiles, version } from "./shared"; import codeTask from "./tasks/code"; -import colors from "./tasks/colors"; import decompile from "./tasks/decompile"; import diffs from "./tasks/diffs"; -import icons from "./tasks/icons"; -import { webhook } from "./tasks/webhook"; import { formatError, handleShellErr, join } from "./utils"; export async function runTasks() { @@ -27,34 +24,21 @@ export async function runTasks() { code_getting: "Finding file imports", code_remaking: "Recreating code", code_pushing: "Committing source", - colors: "Parsing colors", - icons: "Icon parser", - icons_getting: "Writing icons.json", - icons_copying: "Copying images", diff: "Diffs", - diff_raw: "Diffing raw colors", - diff_semantic: "Diffing semantic colors", - diff_icons: "Diffing icons", diff_code: "Diffing code", - webhook: "Sending webhook messages", }, true, ); let differs: Differs | undefined; if (!isMock) { - // discard changes try { progress.start("preinit"); - // reset to remote branch (without pulling latest commit) progress.start("preinit_discard"); await Bun.$`git reset --hard`.cwd("../data").nothrow().quiet().then(handleShellErr); - if (await exists(join("../data", "oldicons"))) - await rm(join("../data", "oldicons"), { force: true, recursive: true }); - await Bun.write("../data/version.txt", version); // lel + await Bun.write("../data/version.txt", version); - // unstage all files await Bun.$`git restore --staged .`.cwd("../data").nothrow().quiet().then(handleShellErr); progress.update("preinit_discard", true); @@ -83,13 +67,9 @@ export async function runTasks() { const code = (await Bun.file(codePath).text()).replace(/\r/g, "").split("\n"); - await Promise.allSettled([ - wrapPromise(codeTask(progress, code), progress, "code"), - wrapPromise(colors(code), progress, "colors"), - wrapPromise(icons(progress, code), progress, "icons"), - ]); - if (progress.someFailed("code", "colors", "icons")) - throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code", "colors", "icons")}`); + await wrapPromise(codeTask(progress, code), progress, "code"); + if (progress.someFailed("code")) + throw new Error(`Failed at parser tasks!\n${progress.prettyErrors("code")}`); while (!progress.isFinished("decompile_gzip")) { await Bun.sleep(1000); @@ -111,6 +91,7 @@ export async function runTasks() { if (differs && !isQuiet) { try { + const { webhook } = await import("./tasks/webhook"); await wrapPromise(webhook(differs), progress, "webhook"); } catch (e: any) { throw new Error(`Failed to send webhook messages!\n${formatError(e)}`); @@ -120,5 +101,4 @@ export async function runTasks() { } await commit(["version.txt"], `chore: bump app version to ${cuteVersion}`); - await rm("../data/oldicons", { force: true, recursive: true }); } diff --git a/src/update/shared.ts b/src/update/shared.ts index 3d09c45e6ef..6d4c3028c1e 100644 --- a/src/update/shared.ts +++ b/src/update/shared.ts @@ -6,9 +6,10 @@ export const maxDiffChanges = 20; export const workFolder = "tmp"; export const apksFolder = join(workFolder, "apks"); export const codePath = join(workFolder, "code.js"); +export const modulesPath = join(workFolder, "modules"); export const apkSplits = ["base", "config.hdpi", "config.xxhdpi"] as const; -export const oprevFiles = ["semantic.json", "raw.json", "icons.json", "source.jsonl"] as const; +export const oprevFiles = ["source.jsonl"] as const; export const prevFiles = new Map<(typeof oprevFiles)[number], ArrayBuffer>(); export const version = await Bun.file("../data/version.txt").text(); diff --git a/src/update/tasks/code.ts b/src/update/tasks/code.ts index 533d0b53b2b..fbfc0f9e5e4 100644 --- a/src/update/tasks/code.ts +++ b/src/update/tasks/code.ts @@ -1,70 +1,40 @@ -import { mkdir, rm } from "node:fs/promises"; -import { deminify } from "../deminify"; +import { mkdir, rm, readdir, stat } from "node:fs/promises"; +import { join as pathJoin } from "node:path"; import { commit } from "../git"; import type { Progress } from "../progress"; -import { commitAnyway, cuteVersion } from "../shared"; -import { discordPath, join, listRequiredDirs, sortEntries } from "../utils"; +import { commitAnyway, cuteVersion, modulesPath } from "../shared"; +import { join, sortEntries } from "../utils"; -// scary code matching below, be warned - -const moduleStartIndentation = " ".repeat(4); -const pathCheckRegex = / = '(.*?)';$/; -const lineImportRegex = / = r\d{1,2}\.fileFinishedImporting;$/; - -export default async function code(progress: Progress, code: string[]) { +export default async function code(progress: Progress, _code: string[]) { progress.start("code_getting"); - const files = new Map(); - let moduleStart: number | null = null; - - for (let i = 0; i < code.length; i++) { - const line = code[i]; - - // find the start of the file import based on the indentation - if ( - new RegExp(`^${moduleStartIndentation}r\\d{1,2} = function\\(`).test(line) && - line.includes(" { // Environment:") - ) { - moduleStart = i; - } - - // easiest way to check - const path = code[i + 1]?.match(pathCheckRegex)?.[1]; - if (line.match(lineImportRegex) && path) { - const start = moduleStart; - if (!start) throw `moduleStart was null for ${start}~${i}; null ~ ${code[i]}`; - moduleStart = null; - - // find the end of the file import based on the indentation - const moduleEndThing = `${moduleStartIndentation}};`; - let moduleEnd: number | null = null; - for (let j = i; j < code.length; j++) { - if (code[j] === moduleEndThing) { - moduleEnd = i = j; - break; - } - } - - if (!moduleEnd) throw `moduleEnd was null for ${start}~${i}; ${code[start]} ~ ${code[i]}`; - - const realPath = discordPath(path); - files.set( - realPath, - deminify( - code - .slice(start, moduleEnd + 1) - .map((line) => line.replace(new RegExp(`^${moduleStartIndentation}`), "")) - .join("\n"), - realPath, - ), - ); - } + if (!(await stat(modulesPath).catch(() => false))) { + throw new Error(`Modules directory not found at ${modulesPath}`); } + async function scanDir(dir: string, basePath: string): Promise> { + const files = new Map(); + const entries = await readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + if (entry.isDirectory()) { + const subFiles = await scanDir(fullPath, basePath); + for (const [name, size] of subFiles) files.set(name, size); + } else if (entry.isFile()) { + const relativePath = fullPath.replace(basePath, "").replace(/^\//, ""); + const stats = await stat(fullPath); + files.set(relativePath, stats.size); + } + } + return files; + } + + const files = await scanDir(modulesPath, modulesPath); + await Bun.write( "../data/source.jsonl", sortEntries([...files.entries()]) - .map(([file, text]) => `{ "file": ${JSON.stringify(file)}, "size": ${text.final.length} }`) + .map(([file, size]) => `{ "file": ${JSON.stringify(file)}, "size": ${size} }`) .join("\n"), ); @@ -76,15 +46,17 @@ export default async function code(progress: Progress, code: string[]) { const filePrefix = "../data/source"; await rm(filePrefix, { recursive: true, force: true }); - const dirs = listRequiredDirs([...files.keys()]); - - await Promise.all(dirs.map((dir) => mkdir(join(filePrefix, dir), { recursive: true }))); - await Promise.all(files.entries().map(([file, text]) => Bun.write(join(filePrefix, file), text.res))); + await Promise.all( + [...files.keys()].map(async (file) => { + const dest = join(filePrefix, file); + await mkdir(pathJoin(dest, ".."), { recursive: true }); + await Bun.write(dest, Bun.file(join(modulesPath, file))); + }), + ); progress.update("code_remaking", true); progress.start("code_pushing"); - // hope the files get written in this time idk :P blehhh await Bun.sleep(1500); await commit(["source.jsonl", "source/*"], `chore: update source for ${cuteVersion}`); diff --git a/src/update/tasks/colors.test.ts b/src/update/tasks/colors.test.ts deleted file mode 100644 index 2046d656a75..00000000000 --- a/src/update/tasks/colors.test.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { sortObj } from "../utils"; -import { convertSimpleSemantic, getInternalRawColors, getInternalSemanticColors } from "./colors"; - -const codeFile = Bun.file("tmp/code.js"); -if (!(await codeFile.exists())) throw "Missing decompiled code.js"; - -const code = (await codeFile.text()).replace(/\r/g, "").split("\n"); - -const raw = getInternalRawColors(code); -await Bun.write("temp/raw.json", JSON.stringify(sortObj(raw), undefined, 4)); - -const semantic = getInternalSemanticColors(code, raw); -await Bun.write("temp/semantic.json", JSON.stringify(sortObj(semantic), undefined, 4)); -await Bun.write("temp/semantic_simple.json", JSON.stringify(sortObj(convertSimpleSemantic(semantic)), null, 4)); diff --git a/src/update/tasks/colors.ts b/src/update/tasks/colors.ts deleted file mode 100644 index 310b3f11760..00000000000 --- a/src/update/tasks/colors.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { runInNewContext } from "node:vm"; -import Color, { type ColorInstance } from "color"; -import type { RawColors, SemanticColors } from "../../types"; -import { commit } from "../git"; -import { cuteVersion } from "../shared"; -import { sortObj } from "../utils"; - -function hex(color: ColorInstance) { - return (color.alpha() === 1 ? color.hex() : color.hexa()).toLowerCase(); -} - -// FIXME fixes an issue introduced in hermes-dec commit 1c112f85e3fb49442c57a0ed31b83bb3825db639, -// where code such as "rX[rY] = z" is instead parsed as "rX[Y] = z" -function fixHermesDecIndex(code: string) { - return code.replace(/r(\d+)\[(\d+)\] =/g, "r$1[r$2] ="); -} - -// more scary code matching below, be warned - -export function evalMetroModule(code: string[], importName: string, imports: object = {}): T { - const hookLine = code.findIndex((x) => x.includes(importName)); - if (hookLine === -1) throw new Error(`Cannot find hookLine for ${importName}!`); - - // usually fileFinishedImporting is called 5 lines before the function ends, so we pray - const endLine = hookLine + 5; - - let startLine: number | null = null; - for (let i = hookLine; i > 0; i--) - if (code[i].includes("// Environment:")) { - startLine = i; - break; - } - - if (startLine === null) throw new Error(`Failed to find startLine for ${importName}!`); - - const snippet = fixHermesDecIndex(code.slice(startLine, endLine).join("\n")); - const fn = runInNewContext(snippet, { - global: { - Object: Object.assign(Object, { - freeze(obj: any) { - return obj; - }, - }), - }, - }); - - const exports: any = {}; - fn( - null, // global - () => ({ - fileFinishedImporting() {}, - ...imports, - }), // require - null, // metroImportDefault - null, // metroImportAll - null, // moduleObject - exports, // exports - [], // dependencyMap - ); - - return exports; -} - -export function getInternalRawColors(code: string[]) { - const module = evalMetroModule<{ - _private: { - RawColors: RawColors; - }; - }>(code, "/tokens/colors/generated/raw-color-definitions.tsx'", {}); - - const raw: RawColors = {}; - for (const [key, color] of Object.entries(module._private.RawColors)) { - raw[key] = hex(Color(color)); - } - - return raw; -} - -export function getInternalSemanticColors(code: string[], raw: Record) { - const module = evalMetroModule<{ - _private: { - SemanticColors: Record>; - }; - }>(code, "/tokens/colors/generated/native/generated-definitions.tsx'", { - _private: { - Themes: new Proxy( - {}, - { - get(_, key) { - return key.toString().toLowerCase(); - }, - }, - ), - }, - }); - - const semantic: SemanticColors = {}; - for (const [key, colors] of Object.entries(module._private.SemanticColors)) { - const color: SemanticColors[string] = {}; - for (const [theme, val] of Object.entries(colors)) { - if (typeof val !== "object" || !("raw" in val)) continue; - - color[theme] = [hex(Color(raw[val.raw]).alpha(val.opacity)), { raw: val.raw, opacity: val.opacity }]; - } - - semantic[key] = color; - } - - return semantic; -} - -export function convertSimpleSemantic(semantic: SemanticColors) { - const simpleSemantic: Record> = {}; - for (const [key, colors] of Object.entries(semantic)) { - const color: Record = {}; - for (const [theme, [clr]] of Object.entries(colors)) color[theme] = clr; - - simpleSemantic[key] = color; - } - - return simpleSemantic; -} - -export default async function colors(code: string[]) { - const raw = getInternalRawColors(code); - await Bun.write("../data/raw.json", JSON.stringify(sortObj(raw), null, 4)); - - const semantic = getInternalSemanticColors(code, raw); - await Bun.write("../data/semantic.json", JSON.stringify(sortObj(semantic), null, 4)); - await Bun.write("../data/semantic_simple.json", JSON.stringify(sortObj(convertSimpleSemantic(semantic)), null, 4)); - - await commit(["raw.json", "semantic.json", "semantic_simple.json"], `chore: update colors for ${cuteVersion}`); -} diff --git a/src/update/tasks/decompile.ts b/src/update/tasks/decompile.ts index ebfd17429f3..53565268691 100644 --- a/src/update/tasks/decompile.ts +++ b/src/update/tasks/decompile.ts @@ -1,37 +1,52 @@ import { exists } from "node:fs/promises"; import { commit } from "../git"; import type { Progress } from "../progress"; -import { codePath, commitAnyway, cuteVersion, workFolder } from "../shared"; +import { codePath, commitAnyway, cuteVersion, modulesPath, workFolder } from "../shared"; import { handleShellErr, join } from "../utils"; const gzipWorkerURL = new URL("decompile-gzip.ts", import.meta.url).href; -const decompilerPy = "src/hermes_dec/decompilation/hbc_decompiler.py"; export default async function decompile(progress: Progress, pathToBundle: string) { const pathToDecompiler = join(workFolder, "decompiler"); progress.start("decompile_downloading"); if (!(await exists(pathToDecompiler))) { - await Bun.$`git clone https://github.com/P1sec/hermes-dec.git --depth=1 ${pathToDecompiler}` + await Bun.$`git clone https://github.com/SymbioticSec/hermes-decomp.git --depth=1 ${pathToDecompiler}` .quiet() .nothrow() .then(handleShellErr); progress.update("decompile_downloading", true); } else progress.update("decompile_downloading", null); - const { exitCode: hasPython } = await Bun.$`python --version`.nothrow().quiet(); - if (hasPython !== 0) throw new Error("Cannot use Python! Are you sure it's installed?"); + const { exitCode: hasCargo } = await Bun.$`cargo --version`.nothrow().quiet(); + if (hasCargo !== 0) throw new Error("Cannot use Cargo! Are you sure Rust is installed?"); progress.start("decompile_decompiling"); - if (!(await Bun.file(codePath).exists())) { - await Bun.$`python ${join(pathToDecompiler, decompilerPy)} ${pathToBundle} ${codePath}` + const decompilerBin = join(pathToDecompiler, "target/release/hermes-decomp"); + if (!(await Bun.file(decompilerBin).exists())) { + await Bun.$`cargo build --release -p hermes-decomp` + .cwd(pathToDecompiler) .quiet() .nothrow() .then(handleShellErr); - progress.update("decompile_decompiling", true); - } else progress.update("decompile_decompiling", null); + } + + if (!(await Bun.file(codePath).exists())) { + await Bun.$`${decompilerBin} decompile ${pathToBundle} --output ${codePath}` + .quiet() + .nothrow() + .then(handleShellErr); + } + + if (!(await exists(modulesPath))) { + await Bun.$`${decompilerBin} extract ${pathToBundle} --output ${modulesPath}` + .quiet() + .nothrow() + .then(handleShellErr); + } + + progress.update("decompile_decompiling", true); - // background step if (process.env.NODE_ENV !== "test" && !commitAnyway) { const gzFile = "code.js.gz"; diff --git a/src/update/tasks/diffs.ts b/src/update/tasks/diffs.ts index 14a0ac27dca..39a6cbbea83 100644 --- a/src/update/tasks/diffs.ts +++ b/src/update/tasks/diffs.ts @@ -1,199 +1,9 @@ -import { - type CodeDiff, - type Diff, - type Differs, - DiffType, - type Icons, - type RawColors, - type Semantic, - type SemanticColors, -} from "../../types"; +import { type CodeDiff, type Differs, DiffType } from "../../types"; import { fetchGitChanged, gitChanged } from "../git"; import { type Progress, wrapPromise } from "../progress"; import { diffAnyway, prevFiles } from "../shared"; import { join, parseJsonl } from "../utils"; -async function diffRaw() { - if (!prevFiles.has("raw.json")) { - throw new Error("Missing prevFile: raw.json"); - } - - const oldRaw = JSON.parse(new TextDecoder().decode(prevFiles.get("raw.json"))) as RawColors; - const newRaw: RawColors = await Bun.file(join("../data", "raw.json")).json(); - - const renamed = new Set(); - const changes = new Map(); - for (const [name, source] of Object.entries(newRaw)) - if (!oldRaw[name]) { - const oldName = Object.entries(oldRaw).find(([key, val]) => !newRaw[key] && val === source)?.[0]; - - if (!oldName) - changes.set(name, { - type: DiffType.Added, - source, - }); - else { - renamed.add(oldName); - changes.set(name, { - type: DiffType.Renamed, - oldName, - source, - }); - } - } else if (oldRaw[name].toLowerCase() !== source.toLowerCase()) - changes.set(name, { - type: DiffType.Changed, - source, - oldSource: oldRaw[name], - }); - for (const [name, source] of Object.entries(oldRaw)) - if (!(newRaw[name] || renamed.has(name))) - changes.set(name, { - type: DiffType.Removed, - source, - }); - - return changes; -} - -function getSemanticLabel(semantic: Semantic) { - const light = semantic.light, - dark = semantic.darker ?? semantic.dark; - if (dark && light) return `☀️ ${light[0]}, 🌙 ${dark[0]}`; - return Object.entries(semantic) - .map(([name, [color]]) => `${name} ${color}`) - .join(", "); -} - -async function diffSemantic() { - if (!prevFiles.has("semantic.json")) { - throw new Error("Missing prevFile: semantic.json"); - } - - const oldSemantic = JSON.parse(new TextDecoder().decode(prevFiles.get("semantic.json"))) as SemanticColors; - const newSemantic: SemanticColors = await Bun.file(join("../data", "semantic.json")).json(); - - const renamed = new Set(); - const changes = new Map(); - for (const [name, semantic] of Object.entries(newSemantic)) { - const source = Object.values(semantic) - .map(([color]) => color) - .join(","); - if (!oldSemantic[name]) { - const themes = Object.keys(semantic).length; - const oldName = Object.entries(oldSemantic).find( - ([key, val]) => - !newSemantic[key] && - Object.keys(val).length === themes && - Object.entries(val).every(([skey, sval]) => sval[0] === semantic[skey][0]), - )?.[0]; - - if (!oldName) - changes.set(name, { - type: DiffType.Added, - source, - label: getSemanticLabel(semantic), - }); - else { - renamed.add(oldName); - changes.set(name, { - type: DiffType.Renamed, - oldName, - source, - }); - } - } else - for (const [theme, [color]] of Object.entries(semantic)) - if (!oldSemantic[name][theme]) - changes.set(`${name}.${theme}`, { - type: DiffType.Added, - source: color, - }); - else if (oldSemantic[name][theme][0].toLowerCase() !== color.toLowerCase()) - changes.set(`${name}.${theme}`, { - type: DiffType.Changed, - source: color, - oldSource: oldSemantic[name][theme][0], - }); - } - for (const [name, semantic] of Object.entries(oldSemantic)) { - if (renamed.has(name)) continue; - - const source = Object.values(semantic) - .map(([color]) => color) - .join(","); - if (!newSemantic[name]) - changes.set(name, { - type: DiffType.Removed, - source, - label: getSemanticLabel(semantic), - }); - else - for (const [theme, [source]] of Object.entries(semantic)) - if (!newSemantic[name][theme]) - changes.set(`${name}.${theme}`, { - type: DiffType.Removed, - source, - }); - } - - return changes; -} - -async function diffIcons() { - if (!prevFiles.has("icons.json")) { - throw new Error("Missing prevFile: icons.json"); - } - - const oldIcons = JSON.parse(new TextDecoder().decode(prevFiles.get("icons.json"))) as Icons; - const newIcons: Icons = await Bun.file(join("../data", "icons.json")).json(); - - const iconDir = { - old: join("../data", "oldicons"), - new: join("../data", "icons"), - }; - - const renamed = new Set(); - const changes = new Map(); - for (const [name, icon] of Object.entries(newIcons)) { - const label = icon.hash.slice(0, 8); - if (!oldIcons[name]) { - const oldName = Object.entries(oldIcons).find(([key, val]) => !newIcons[key] && val.hash === icon.hash)?.[0]; - - if (!oldName) - changes.set(name, { - type: DiffType.Added, - source: join(iconDir.new, icon.file), - label, - }); - else { - renamed.add(oldName); - changes.set(name, { - type: DiffType.Renamed, - oldName, - source: join(iconDir.new, icon.file), - }); - } - } else if (oldIcons[name].hash !== icon.hash) - changes.set(name, { - type: DiffType.Changed, - source: join(iconDir.new, icon.file), - label, - oldSource: join(iconDir.old, oldIcons[name].file), - oldLabel: oldIcons[name].hash.slice(0, 8), - }); - } - for (const [name, icon] of Object.entries(oldIcons)) - if (!(newIcons[name] || renamed.has(name))) - changes.set(name, { - type: DiffType.Removed, - source: join(iconDir.old, icon.file), - label: icon.hash.slice(0, 8), - }); - - return changes; -} - function parseSource(text: string) { const source = parseJsonl(text) as { file: string; size: number }[]; return Object.fromEntries( @@ -245,9 +55,6 @@ export default async function diffs(progress: Progress) { if (!(gitChanged.has("version.txt") || diffAnyway)) { progress.update("diff", null); - progress.update("diff_raw", null); - progress.update("diff_semantic", null); - progress.update("diff_icons", null); progress.update("diff_code", null); return; } @@ -255,27 +62,6 @@ export default async function diffs(progress: Progress) { const differs = {} as Differs; await Promise.all([ - gitChanged.has("raw.json") - ? wrapPromise( - diffRaw().then((x) => (differs.raw = x)), - progress, - "diff_raw", - ) - : progress.update("diff_raw", null), - gitChanged.has("semantic.json") - ? wrapPromise( - diffSemantic().then((x) => (differs.semantic = x)), - progress, - "diff_semantic", - ) - : progress.update("diff_semantic", null), - gitChanged.has("icons.json") - ? wrapPromise( - diffIcons().then((x) => (differs.icons = x)), - progress, - "diff_icons", - ) - : progress.update("diff_icons", null), gitChanged.has("source.jsonl") ? wrapPromise( diffCode().then((x) => (differs.code = x)), diff --git a/src/update/tasks/icons.test.ts b/src/update/tasks/icons.test.ts deleted file mode 100644 index d48262aed2a..00000000000 --- a/src/update/tasks/icons.test.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { sortObj } from "../utils"; -import { parseAssets } from "./icons"; - -const codeFile = Bun.file("tmp/code.js"); -if (!(await codeFile.exists())) throw "Missing decompiled code.js"; - -const code = (await codeFile.text()).replace(/\r/g, "").split("\n"); - -const { icons } = await parseAssets(code); -await Bun.write("temp/icons.json", JSON.stringify(sortObj(icons), undefined, 4)); diff --git a/src/update/tasks/icons.ts b/src/update/tasks/icons.ts deleted file mode 100644 index 6d62b22ca90..00000000000 --- a/src/update/tasks/icons.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { copyFile, mkdir, readdir, rename } from "node:fs/promises"; -import { basename } from "node:path"; -import { runInNewContext } from "node:vm"; -import type { Icons } from "../../types"; -import { commit } from "../git"; -import type { Progress } from "../progress"; -import { apkSplits, apksFolder, cuteVersion } from "../shared"; -import { discordPath, join, listRequiredDirs, sortObj } from "../utils"; - -const infoObjRegex = /({.+?})/; -const scalesArrayRegex = /(\[.+?\])/; - -// a bit less scary code matching below, be warned -export async function parseAssets(code: string[]) { - const retrievedAssets: { - httpServerLocation: string; - width: number; - height: number; - scales: number[]; - hash: string; - name: string; - type: "png" | "svg" | "lottie"; - }[] = []; - - for (let i = 0; i < code.length; i++) { - // next line has asset info, after that are scales - const line = code[i], - infoLine = code[i + 1], - scalesLine = code[i + 2]; - - if (line.includes(".registerAsset") && infoLine?.includes("'httpServerLocation'") && scalesLine?.includes("[")) { - const infoText = infoLine.match(infoObjRegex)?.[1]; - if (!infoText) throw new Error(`Failed to find infoText for ${infoLine} (line ${i + 1})`); - - const scalesText = scalesLine.match(scalesArrayRegex)?.[1]; - if (!scalesText) throw new Error(`Failed to find scalesText for ${scalesLine} (line ${i + 2})`); - - const info = runInNewContext(`(${infoText})`); - if (typeof info !== "object") continue; - - const scales = runInNewContext(`(${scalesText})`); - if (!Array.isArray(scales)) continue; - - info.scales = scales; - - if ( - ["httpServerLocation", "hash", "name", "type"].every((x) => x in info) && - ["svg", "png", "lottie"].includes(info.type) && - !info.httpServerLocation.includes("node_modules/.pnpm") - ) { - retrievedAssets.push(info); - } - } - } - - // get files of all apks by hierarchy - const apkPaths = new Map(); - for (const split of apkSplits) { - const folder = join(apksFolder, split); - for (const path of await readdir(folder, { recursive: true })) apkPaths.set(basename(path), join(folder, path)); - } - - const icons: Icons = {}; - const toCopy: { from: string; to: string }[] = []; - for (const asset of retrievedAssets) { - const baseName = [...asset.httpServerLocation.split("/").slice(2), asset.name] - .join("_") - .toLowerCase() - .replace(/\W+/g, ""); - const path = `${baseName}.${asset.type}`; - - const apkPath = apkPaths.get(path); - if (apkPath) { - const realPath = join( - discordPath(asset.httpServerLocation.split("/").slice(2).join("/")), - `${asset.name}.${asset.type}`, - ); - - icons[asset.name] = { - file: realPath, - hash: asset.hash, - scales: asset.scales, - width: asset.width ?? null, - height: asset.height ?? null, - }; - toCopy.push({ - from: apkPath, - to: join("../data/icons", realPath), - }); - } - } - - return { icons, toCopy }; -} - -export default async function icons(progress: Progress, code: string[]) { - progress.start("icons_getting"); - - const { icons, toCopy } = await parseAssets(code); - await Bun.write("../data/icons.json", JSON.stringify(sortObj(icons), undefined, 4)); - progress.update("icons_getting", true); - - progress.start("icons_copying"); - - // don't delete the old icons folder so we can display the old images - await rename("../data/icons", "../data/oldicons"); - const dirs = listRequiredDirs(toCopy.map((x) => x.to)); - - await Promise.all(dirs.map((dir) => mkdir(dir, { recursive: true }))); - await Promise.all(toCopy.map(({ from, to }) => copyFile(from, to))); - - await commit(["icons.json", "icons"], `chore: update icons for ${cuteVersion}`); - progress.update("icons_copying", true); -} diff --git a/src/update/tasks/webhook.ts b/src/update/tasks/webhook.ts index 5761e44cde8..60dad69efcf 100644 --- a/src/update/tasks/webhook.ts +++ b/src/update/tasks/webhook.ts @@ -1,18 +1,7 @@ import { basename } from "node:path"; -import type { Canvas } from "@napi-rs/canvas"; -import { - type APIMessageTopLevelComponent, - ComponentType, - MessageFlags, - type RESTPostAPIWebhookWithTokenJSONBody, - RouteBases, - Routes, -} from "discord-api-types/v10"; -import { drawSections } from "../../canvas"; -import { makeSections } from "../../canvas/factory"; -import { type CodeDiff, type Diff, type Differs, DiffType } from "../../types"; +import { type CodeDiff, type Differs, DiffType } from "../../types"; import { cuteVersion, maxDiffChanges, version } from "../shared"; -import { assert, formatBytes, sortEntries } from "../utils"; +import { formatBytes, sortEntries } from "../utils"; function fileBase(path: string, other?: string) { if (!other) return `... ${basename(path)}`; @@ -48,42 +37,27 @@ interface FormattedDiff { capped: boolean; } -function formatDiff(diffs: Map): FormattedDiff { +function formatDiff(diffs: Map): FormattedDiff { const entries = [...diffs.entries()]; const sections = { Added: sortEntries(entries) .map( ([name, diff]) => - diff.type === DiffType.Added && - ("size" in diff - ? `+ ${fileBase(name)} (${formatBytes(diff.size)})` - : `+ ${name}: ${diff.label || diff.source}`), - ) - .filter((x) => typeof x === "string"), - Changed: sortEntries(entries) - .map( - ([name, diff]) => - diff.type === DiffType.Changed && - `- ${name}: ${diff.oldLabel || diff.oldSource}\n+ ${name}: ${diff.label || diff.source}`, + diff.type === DiffType.Added && `+ ${fileBase(name)} (${formatBytes(diff.size)})`, ) .filter((x) => typeof x === "string"), Renamed: sortEntries(entries) .map( ([name, diff]) => diff.type === DiffType.Renamed && - ("size" in diff - ? `- ${fileBase(diff.oldName, name)}\n+ ${fileBase(name, diff.oldName)}` - : `- ${diff.oldName}\n+ ${name}`), + `- ${fileBase(diff.oldName, name)}\n+ ${fileBase(name, diff.oldName)}`, ) .filter((x) => typeof x === "string"), Removed: sortEntries(entries) .map( ([name, diff]) => - diff.type === DiffType.Removed && - ("size" in diff - ? `- ${fileBase(name)} (${formatBytes(diff.size)})` - : `- ${name}: ${diff.label || diff.source}`), + diff.type === DiffType.Removed && `- ${fileBase(name)} (${formatBytes(diff.size)})`, ) .filter((x) => typeof x === "string"), }; @@ -98,174 +72,26 @@ function formatDiff(diffs: Map): FormattedDiff { }; } -interface WebhookEmbed { - title: string; - body: FormattedDiff; - image?: Canvas; - footer: string; - key: string; -} - -async function sendWebhook(webhook: string, role: string, embeds: WebhookEmbed[], color?: number) { - const body = new FormData(); - const images = new Map(), - diffs = new Map(); - - let fi = 0; - for (let i = 0; i < embeds.length; i++) { - const embed = embeds[i]; - if (embed.image) { - const key = fi++; - const filename = `${embed.key}.png`; - images.set(i, filename); - - body.append(`files[${key}]`, new Blob([embed.image.toBuffer("image/png")], { type: "image/png" }), filename); - } - if (embed.body.capped) { - const key = fi++; - const filename = `${embed.key}-changes.diff`; - diffs.set(i, filename); - - body.append(`files[${key}]`, new Blob([embed.body.file], { type: "text/plain" }), filename); - } - } - - body.append( - "payload_json", - JSON.stringify({ - flags: MessageFlags.IsComponentsV2, - components: [ - { - type: ComponentType.TextDisplay, - content: `<@&${role}>`, - }, - ...embeds.map( - ({ title, body, image, footer }, i) => - ({ - type: ComponentType.Container, - accent_color: color, - components: [ - { - type: ComponentType.TextDisplay, - content: `## ${title}\n-# ${version} (${cuteVersion})`, - }, - ...body.embed.map((content) => ({ - type: ComponentType.TextDisplay, - content, - })), - body.capped && { - type: ComponentType.File, - file: { - url: `attachment://${diffs.get(i)}`, - }, - }, - body.capped && { - type: ComponentType.Separator, - }, - image && { - type: ComponentType.MediaGallery, - items: [ - { - media: { - url: `attachment://${images.get(i)}`, - }, - }, - ], - }, - { - type: ComponentType.TextDisplay, - content: footer, - }, - ].filter((x) => !!x), - }) as APIMessageTopLevelComponent, - ), - ], - allowed_mentions: process.env.NODE_ENV === "test" ? { parse: [] } : { roles: [role] }, - } as RESTPostAPIWebhookWithTokenJSONBody), - ); - - const url = new URL(webhook); - url.searchParams.set("wait", "true"); - url.searchParams.set("with_components", "true"); - - const res = await fetch(url, { - method: "POST", - body, - }); - - const data = await res.text(); - if (!res.ok) - throw new Error(`Failed to send webhook message with embeds ${embeds.map((x) => x.title).join(", ")}: ${data}`); - - const discordToken = process.env.bot_token; - if (!discordToken) return; - - // this can fail quietly - try { - const message = JSON.parse(data); - - const route = `${RouteBases.api}/${Routes.channelMessageCrosspost(message.channel_id, message.id)}`; - await fetch(route, { - method: "POST", - headers: { - Authorization: discordToken, - }, - }); - } catch {} -} - export async function webhook(diffs: Differs) { - if (diffs.raw?.size || diffs.semantic?.size) - await sendWebhook( - assert(process.env.color_webhook, "Missing color webhook env"), - "1227327297795657850", - [ - diffs.raw?.size && { - title: "Raw colors", - body: formatDiff(diffs.raw), - image: drawSections(await makeSections(diffs.raw)), - footer: makeFooter(diffs.raw.size, "raw color"), - key: "raw-colors", - }, - diffs.semantic?.size && { - title: "Semantic colors", - body: formatDiff(diffs.semantic), - image: drawSections(await makeSections(diffs.semantic)), - footer: makeFooter(diffs.semantic.size, "semantic color"), - key: "semantic-colors", - }, - ].filter((x) => !!x), - 0xd8ef9e, - ); - - if (diffs.icons?.size) - await sendWebhook( - assert(process.env.icons_webhook, "Missing icons webhook env"), - "1227327765079003217", - [ - { - title: "Icons", - body: formatDiff(diffs.icons), - image: drawSections(await makeSections(diffs.icons, true)), - footer: makeFooter(diffs.icons.size, "icon"), - key: "icons", - }, - ], - 0xf4b8f7, - ); - - if (diffs.code?.size) - await sendWebhook( - assert(process.env.code_webhook, "Missing code webhook env"), - "1233861867059941387", - [ + if (diffs.code?.size) { + const body = JSON.stringify({ + content: `Code changes for **${version} (${cuteVersion})**`, + embeds: [ { title: "Code", - body: formatDiff(diffs.code), - footer: makeFooter(diffs.code.size, "code"), - key: "code", + description: formatDiff(diffs.code).embed.join("\n"), + footer: { text: makeFooter(diffs.code.size, "code") }, }, ], - 0x86faf3, - ); + }); + + const webhookUrl = process.env.code_webhook; + if (webhookUrl) { + await fetch(webhookUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body, + }); + } + } }