import { mkdir, readdir, rm, stat } from "node:fs/promises"; import { join as pathJoin } from "node:path"; import { commit } from "../git"; import type { Progress } from "../progress"; import { commitAnyway, cuteVersion, modulePathsDest, modulesPath } from "../shared"; import { formatBytes, join, sortEntries } from "../utils"; const MAX_FILE_SIZE = 100 * 1024 * 1024; function extractModuleId(filename: string): number | undefined { const m1 = filename.match(/^module_(\d+)\.js$/); if (m1) return Number.parseInt(m1[1], 10); const m2 = filename.match(/^(\d+)_/); if (m2) return Number.parseInt(m2[1], 10); } function sanitizePath(p: string): string { return p .replace(/\\/g, "/") .replace(/^\.\.\//, "") .replace(/[<>:"|?*]/g, "_"); } export default async function code(progress: Progress, _code: string[]) { progress.start("code_getting"); if (!(await stat(modulesPath).catch(() => false))) { throw new Error(`Modules directory not found at ${modulesPath}`); } const modulePaths = new Map(); try { const arr: { id: number; path: string }[] = await Bun.file(modulePathsDest).json(); for (const { id, path } of arr) modulePaths.set(id, path); console.log(`Loaded ${modulePaths.size} module path mappings`); } catch { console.log("No module-paths.json found, using extract names"); } async function scanDir(dir: string, basePath: string): Promise> { const files = new Map(); const entries = await readdir(dir, { withFileTypes: true }); for (const entry of entries) { const fullPath = join(dir, entry.name); if (entry.isDirectory()) { const subFiles = await scanDir(fullPath, basePath); for (const [name, size] of subFiles) files.set(name, size); } else if (entry.isFile()) { const relativePath = fullPath.replace(basePath, "").replace(/^\//, ""); const stats = await stat(fullPath); files.set(relativePath, stats.size); } } return files; } const rawFiles = await scanDir(modulesPath, modulesPath); function classifyUnmapped(relativePath: string): string { if (/^module_\d+\.js$/.test(relativePath)) return "__polyfill"; const rest = relativePath.replace(/^\d+_/, ""); if (rest.startsWith("_")) return "__helpers"; return "__vendor"; } function resolveDest(relativePath: string): string { const id = extractModuleId(relativePath); if (id !== undefined) { const sourcePath = modulePaths.get(id); if (sourcePath) return sanitizePath(sourcePath); const bucket = Math.floor(id / 1000); const bucketStr = `${String(bucket).padStart(2, "0")}xxx`; const cat = classifyUnmapped(relativePath); return `${cat}/${bucketStr}/${relativePath}`; } return `__unmapped/${relativePath}`; } const files = new Map(); const skipped: string[] = []; for (const [relativePath, fileSize] of rawFiles) { if (fileSize > MAX_FILE_SIZE) { skipped.push(`${join("source", resolveDest(relativePath))} (${formatBytes(fileSize)})`); continue; } const dest = resolveDest(relativePath); if (!files.has(dest)) files.set(dest, fileSize); } await Bun.write( "../data/source.jsonl", sortEntries([...files.entries()]) .map(([file, size]) => `{ "file": ${JSON.stringify(file)}, "size": ${size} }`) .join("\n"), ); progress.update("code_getting", true); if (process.env.NODE_ENV !== "test" && !commitAnyway) { progress.start("code_remaking"); const filePrefix = "../data/source"; await rm(filePrefix, { recursive: true, force: true }); await Promise.all( [...rawFiles.entries()].map(async ([relativePath, fileSize]) => { if (fileSize > MAX_FILE_SIZE) return; const dest = join(filePrefix, resolveDest(relativePath)); await mkdir(pathJoin(dest, ".."), { recursive: true }); await Bun.write(dest, Bun.file(join(modulesPath, relativePath))); }), ); progress.update("code_remaking", true); progress.start("code_pushing"); await Bun.sleep(1500); if (skipped.length) console.warn(`Skipped ${skipped.length} file(s) >100MB (GitHub limit):\n${skipped.join("\n")}`); const message = skipped.length ? `chore: update source for ${cuteVersion}\n\nSkipped ${skipped.length} file(s) >100MB (GitHub limit): ${skipped.join(", ")}` : `chore: update source for ${cuteVersion}`; await commit(["source.jsonl", "source/*"], message); progress.update("code_pushing", true); } else { progress.update("code_remaking", null); progress.update("code_pushing", null); } }