From 503e7a35f9d7006437287cec75791601bf16ed82 Mon Sep 17 00:00:00 2001 From: Z8MB1E Date: Thu, 27 Aug 2026 11:25:28 -0400 Subject: [PATCH] feat(import): add Arma library importer with PBO icon extraction - importArmaLibrary: upsert Arma export catalog into assets/vehicles via Payload, materialize .paa thumbnails (convertPaaToPng) and link uploads, resolve Fuel resource for vehicle fuel defaults - extractArmaIcons: ref-driven PBO .paa extraction into a --paa-root tree shaped like inventory icon ref paths - package.json: add import:arma script --- package.json | 1 + src/tools/import/extractArmaIcons.ts | 240 +++++++++++++++++++++++++ src/tools/import/importArmaLibrary.ts | 249 ++++++++++++++++++++++++++ 3 files changed, 490 insertions(+) create mode 100644 src/tools/import/extractArmaIcons.ts create mode 100644 src/tools/import/importArmaLibrary.ts diff --git a/package.json b/package.json index ceb5c00..3094736 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "generate:importmap": "cross-env NODE_OPTIONS=--no-deprecation payload generate:importmap", "generate:types": "cross-env NODE_OPTIONS=--no-deprecation payload generate:types", "generate:version-info": "bun run src/scripts/generateVersionInfo.ts", + "import:arma": "bun run src/tools/import/importArmaLibrary.ts", "version:bump": "bun run src/scripts/bumpVersion.ts", "lint": "cross-env NODE_OPTIONS=--no-deprecation next lint", "payload": "cross-env NODE_OPTIONS=--no-deprecation payload", diff --git a/src/tools/import/extractArmaIcons.ts b/src/tools/import/extractArmaIcons.ts new file mode 100644 index 0000000..039d190 --- /dev/null +++ b/src/tools/import/extractArmaIcons.ts @@ -0,0 +1,240 @@ +#!/usr/bin/env bun +/** + * Extract Arma inventory .paa icons referenced by an export catalog out of the + * game's packed PBOs into a --paa-root tree shaped like the reference paths, + * so importArmaLibrary.ts's resolvePaaPath() can find them for PAA->PNG. + * + * Resolution is ref-driven: the second path segment names the PBO (vanilla + * refs look like "\a3\weapons_f\Rifles\MX\data\UI\gear_mx_cqc_M_CA.paa"), so we + * locate that PBO by basename and inspect it on demand (cached), then map the + * ref to a file inside it. A fabricated/unresolvable ref reports MISS rather + * than failing the run. + * + * CLI: + * --catalog export catalog JSON (array of records with "picture") + * --icons icon_map JSON (map of classname -> .paa ref) [optional] + * --game Arma 3 install root (contains /Addons/*.pbo) + * [--mod ]... extra mod roots to search for .pbo files + * --paa-root where extracted .paa files are written (mirrors refs) + * [--out-dir ] write a machine-readable report here (optional) + * [--hemtt ] hemtt binary (default: resolve from PATH) + */ +import { spawnSync } from "child_process"; +import { existsSync, mkdirSync, readFileSync } from "fs"; +import { dirname, join } from "path"; + +interface CliOpts { + catalog?: string[]; + icons?: string[]; + game?: string[]; + mod?: string[]; + "paa-root"?: string[]; + "out-dir"?: string[]; + hemtt?: string[]; +} + +function parseCli(argv: string[]): CliOpts { + const opts: CliOpts = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a.startsWith("--") && i + 1 < argv.length && !argv[i + 1].startsWith("--")) { + const key = a.slice(2) as keyof CliOpts; + (opts[key] ??= []).push(argv[++i]); + } + } + return opts; +} + +function resolveHemtt(explicit?: string): string { + if (explicit && existsSync(explicit)) return explicit; + const probe = spawnSync("which", ["hemtt"], { encoding: "utf8" }); + if (probe.status === 0 && probe.stdout.trim()) return probe.stdout.trim(); + throw new Error("hemtt not found; pass --hemtt "); +} + +interface PboInfo { + pboPath: string; + prefix: string; + /** lowercase file path -> exact original-case path as stored in the PBO */ + files: Map; +} + +function discoverPbos(game: string, mods: string[]): string[] { + const dirs = [join(game, "Addons")]; + for (const m of mods) { + const cand = join(m, "addons"); + if (existsSync(cand)) dirs.push(cand); + } + const out: string[] = []; + for (const d of dirs) { + if (!existsSync(d)) continue; + for (const f of readFileSync("/dev/null").length ? [] : []) void f; + const entries = spawnSync("ls", [d], { encoding: "utf8" }); + for (const line of (entries.stdout ?? "").split("\n")) { + const n = line.trim(); + if (n.toLowerCase().endsWith(".pbo")) out.push(join(d, n)); + } + } + return out; +} + +function pbosByName(pbos: string[]): Map { + const byName = new Map(); + for (const p of pbos) { + const base = p.split(/[\\/]/).pop()!.replace(/\.pbo$/i, "").toLowerCase(); + if (!byName.has(base)) byName.set(base, p); + } + return byName; +} + +function inspectPbo(hemtt: string, pboPath: string): { prefix: string; files: string[] } | undefined { + const r = spawnSync( + hemtt, + ["utils", "pbo", "inspect", "--format", "json", pboPath], + { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 }, + ); + if (r.status !== 0 || !r.stdout) return undefined; + let data: { files?: { name?: string }[]; header?: { properties?: Record } }; + try { + data = JSON.parse(r.stdout); + } catch { + return undefined; + } + const files = (data.files ?? []).map((f) => f.name ?? "").filter(Boolean); + const props = data.header?.properties ?? {}; + const prefix = (props.prefix ?? "").trim().toLowerCase().replace(/\\/g, "/"); + return { prefix, files }; +} + +function ensurePbo( + hemtt: string, + pboPath: string, + cache: Map, +): PboInfo | undefined { + if (cache.has(pboPath)) return cache.get(pboPath); + const info = inspectPbo(hemtt, pboPath); + if (!info || !info.prefix) return undefined; + const files = new Map(); + for (const f of info.files) files.set(f.toLowerCase(), f); + const pbo: PboInfo = { pboPath, prefix: info.prefix, files }; + cache.set(pboPath, pbo); + return pbo; +} + +function normalizeRef(src: string): string { + return src.replace(/\\/g, "/").replace(/^\/+/, "").toLowerCase(); +} + +function resolveRef( + norm: string, + pbos: string[], + byName: Map, + inspected: Map, + hemtt: string, +): { pbo: PboInfo; relFile: string } | undefined { + const seg = norm.split("/").filter(Boolean); + const pboName = seg[1]?.toLowerCase(); + if (pboName && byName.has(pboName)) { + const pbo = ensurePbo(hemtt, byName.get(pboName)!, inspected); + if (pbo) { + const prefixSlash = pbo.prefix + "/"; + if (norm.startsWith(prefixSlash)) { + const relLower = norm.slice(pbo.prefix.length + 1).replace(/\//g, "\\"); + const exact = pbo.files.get(relLower); + if (exact) return { pbo, relFile: exact }; + } + } + } + let best: { prefix: string; pbo: PboInfo } | undefined; + for (const pbo of inspected.values()) { + const prefixSlash = pbo.prefix + "/"; + if (norm.startsWith(prefixSlash) && (!best || pbo.prefix.length > best.prefix.length)) { + best = { prefix: pbo.prefix, pbo }; + } + } + if (!best) return undefined; + const relLower = norm.slice(best.prefix.length + 1).replace(/\//g, "\\"); + const exact = best.pbo.files.get(relLower); + return exact ? { pbo: best.pbo, relFile: exact } : undefined; +} + +async function main() { + const opts = parseCli(process.argv.slice(2)); + const hemtt = resolveHemtt(opts.hemtt?.[0]); + const game = opts.game?.[0]; + if (!game || !existsSync(game)) throw new Error("--game is required and must exist"); + if (!opts["paa-root"]?.[0]) throw new Error("--paa-root is required"); + const paaRoot = opts["paa-root"][0]; + const catalogPath = opts.catalog?.[0]; + if (!catalogPath || !existsSync(catalogPath)) throw new Error("--catalog is required"); + + const pbos = discoverPbos(game, opts.mod ?? []); + console.log(`Discovered ${pbos.length} PBOs across ${1 + (opts.mod?.length ?? 0)} roots.`); + const byName = pbosByName(pbos); + const inspected = new Map(); + + interface Ref { src: string; origin: string; } + const refs = new Map(); + const push = (src: string, origin: string) => { + const norm = normalizeRef(src); + if (!norm || !norm.endsWith(".paa")) return; + if (!refs.has(norm)) refs.set(norm, { src: src.replace(/\\/g, "/").replace(/^\/+/, ""), origin }); + }; + + const catalog = JSON.parse(readFileSync(catalogPath, "utf8")); + for (const rec of Array.isArray(catalog) ? catalog : [catalog]) { + if (typeof rec?.picture === "string") push(rec.picture, "catalog"); + } + const iconsPath = opts.icons?.[0]; + if (iconsPath && existsSync(iconsPath)) { + const icons = JSON.parse(readFileSync(iconsPath, "utf8")); + for (const v of Object.values(icons)) push(String(v), "icon_map"); + } + + console.log(`${refs.size} unique .paa refs to extract.\n`); + const report: string[] = []; + let ok = 0, missing = 0, extractErr = 0; + for (const ref of refs.values()) { + const norm = normalizeRef(ref.src); + const dest = join(paaRoot, ...ref.src.split("/")); + if (existsSync(dest)) { + console.log(` CACHE ${ref.origin}\t${ref.src}`); + ok++; + continue; + } + const res = resolveRef(norm, pbos, byName, inspected, hemtt); + if (!res) { + console.log(` MISS ${ref.origin}\t${ref.src} (not in any indexed PBO)`); + report.push(`MISS\t${ref.origin}\t${ref.src}`); + missing++; + continue; + } + mkdirSync(dirname(dest), { recursive: true }); + const r = spawnSync( + hemtt, + ["utils", "pbo", "extract", res.pbo.pboPath, res.relFile, dest], + { encoding: "utf8" }, + ); + if (r.status === 0 && existsSync(dest)) { + console.log(` EXTRACTED ${ref.origin}\t${ref.src}`); + report.push(`EXTRACTED\t${ref.origin}\t${ref.src}`); + ok++; + } else { + console.log(` ERR ${ref.origin}\t${ref.src} (extract failed)`); + report.push(`ERR\t${ref.origin}\t${ref.src}`); + extractErr++; + } + } + console.log(`\nDone: ${ok} available, ${missing} missing, ${extractErr} extract errors.`); + const outDir = opts["out-dir"]?.[0]; + if (outDir) { + mkdirSync(outDir, { recursive: true }); + const { writeFileSync } = await import("fs"); + writeFileSync(join(outDir, "paa_report.txt"), report.join("\n") + "\n"); + } +} + +main().catch((e) => { + console.error(e); + process.exit(1); +}); diff --git a/src/tools/import/importArmaLibrary.ts b/src/tools/import/importArmaLibrary.ts new file mode 100644 index 0000000..7c19019 --- /dev/null +++ b/src/tools/import/importArmaLibrary.ts @@ -0,0 +1,249 @@ +#!/usr/bin/env bun +/** + * Import an Arma inventory export into the Polaris Task Force Payload CMS. + * + * Consumes a catalog JSON produced by the in-game ptf_export addon (records + * carrying classname, display name, type, description, picture) plus an + * optional icon_map, and upserts rows into the `assets` / `vehicles` + * collections. Thumbnails are materialized from .paa inventory icons found + * under --paa-root (see extractArmaIcons.ts) and converted to PNG uploads via + * `hemtt utils paa convert`; vehicle 3D-rendered screenshots can be supplied + * via --screenshots (matched by classname). + * + * CLI: + * --catalog export catalog JSON (required) + * [--icons ] icon_map JSON (classname -> .paa ref) + * [--paa-root ] root mirroring ref paths where .paa thumbnails live + * [--screenshots ] PNG/WebP per-classname 3D-rendered previews + * [--out ] write converted thumbnails + a report here + * [--hemtt ] hemtt binary (default: resolve from PATH) + * [--approve] apply to the DB (default: proposal only) + * [--dry-run] plan + print, never touch the DB or filesystem + * [--limit ] import at most n records + */ +import { spawnSync } from "child_process"; +import { existsSync, mkdirSync, readFileSync } from "fs"; +import { dirname, join } from "path"; + +interface CliOpts { + catalog?: string[]; + icons?: string[]; + "paa-root"?: string[]; + screenshots?: string[]; + out?: string[]; + hemtt?: string[]; + approve?: boolean; + "dry-run"?: boolean; + limit?: string[]; +} +function parseCli(argv: string[]): CliOpts { + const opts: CliOpts = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === "--approve") { opts.approve = true; continue; } + if (a === "--dry-run") { opts["dry-run"] = true; continue; } + if (a.startsWith("--") && i + 1 < argv.length && !argv[i + 1].startsWith("--")) { + const key = a.slice(2) as keyof CliOpts; + (opts[key] ??= []).push(argv[++i]); + } + } + return opts; +} +function resolveHemtt(explicit?: string): string { + if (explicit && existsSync(explicit)) return explicit; + const probe = spawnSync("which", ["hemtt"], { encoding: "utf8" }); + if (probe.status === 0 && probe.stdout.trim()) return probe.stdout.trim(); + throw new Error("hemtt not found; pass --hemtt "); +} + +const KNOWN_ASSET_TYPES = new Set([ + "weapon", "magazine", "ammo", "backpack", "vest", "helmet", "uniform", + "item", "attachment", "optics", "muzzle", "grip", "rail", "explosive", + "grenade", "medical", "tool", +]); +function classifyAssetType(clazz: string, type: string): string { + const t = (type || "").toLowerCase(); + if (KNOWN_ASSET_TYPES.has(t)) return t; + const c = clazz.toLowerCase(); + if (c.includes("mag") || c.startsWith("30rnd") || c.startsWith("20rnd")) return "magazine"; + if (/^(arifle|smg|hgun|srifle|lm|mmg|rifle|launch)/.test(c)) return "weapon"; + if (/^(30|40|45|9|7|5|6|12|mag_|rhs_mag)/.test(c)) return "magazine"; + return "item"; +} + +function isVehicle(clazz: string, type: string): boolean { + if (type && /vehicle|car|tank|uav|ship|heli|truck/.test(type.toLowerCase())) return true; + const c = clazz.toLowerCase(); + return /^(b_|o_|i_|c_)(mr|tru|car|heli|tank|uav|boat|apc|lt|soldier)/.test(c) && + !/backpack|vest|helmet/.test(c); +} + +function lookupIcon(icons: Record, clazz: string): string { + if (icons && typeof icons === "object") { + for (const [k, v] of Object.entries(icons)) { + if (k.toLowerCase() === clazz.toLowerCase() && typeof v === "string") return v; + } + } + return ""; +} + +function resolvePaaPath(ref: string, paaRoot: string): string { + if (!ref) return ""; + const clean = ref.replace(/^[\\/]+/, "").replace(/\\/g, "/"); + const cand = join(paaRoot, clean); + return existsSync(cand) ? cand : ""; +} + +function convertPaaToPng(hemtt: string, paaPath: string, outDir: string): string | undefined { + const base = basename(paaPath).replace(/\.paa$/i, "") + ".png"; + const dest = join(outDir, base); + if (existsSync(dest)) return dest; + mkdirSync(outDir, { recursive: true }); + const r = spawnSync(hemtt, ["utils", "paa", "convert", paaPath, dest], { encoding: "utf8" }); + return r.status === 0 && existsSync(dest) ? dest : undefined; +} +function basename(p: string): string { + const seg = p.split(/[\\/]/); + return seg[seg.length - 1]; +} + +interface ImportPlan { + kind: "asset" | "vehicle"; + clazz: string; + displayName: string; + description: string; + assetType: string; + pictureRef: string; + paaPath: string; + pngPath: string; + screenshotPath: string; +} + +function planImport( + rec: Record, + lookups: { icons: Record; paaRoot: string; screenshots: string; outDir: string; hemtt: string }, +): ImportPlan | undefined { + const clazz = String(rec.classname ?? rec.class ?? "").trim(); + const type = String(rec.type ?? rec.category ?? "").trim(); + if (!clazz) return undefined; + const vehicle = isVehicle(clazz, type); + const assetType = vehicle ? "vehicle" : classifyAssetType(clazz, type); + const displayName = String(rec.displayName ?? rec.name ?? clazz).trim() || clazz; + const description = String(rec.description ?? "").trim(); + const pictureRef = String(rec.picture ?? "").trim(); + const iconRef = lookups.icons ? lookupIcon(lookups.icons, clazz) : ""; + const ref = iconRef || pictureRef; + const paaPath = ref ? resolvePaaPath(ref, lookups.paaRoot) : ""; + const pngPath = paaPath + ? convertPaaToPng(lookups.hemtt, paaPath, join(lookups.outDir)) + : ""; + const screenshotPath = lookups.screenshots + ? (["png", "webp", "jpg"].map((e) => join(lookups.screenshots, clazz + "." + e)).find(existsSync) ?? "") + : ""; + return { kind: vehicle ? "vehicle" : "asset", clazz, displayName, description, assetType, pictureRef: ref, paaPath, pngPath, screenshotPath }; +} + +async function resolveFuelConfig(payload: { find: (args: any) => Promise<{ docs: any[] }> }): Promise<{ id: number }> { + const r = await payload.find({ collection: "resources", limit: 50, where: { name: { equals: "Fuel" } } }); + const fuel = r.docs.find((d: any) => String(d.name).toLowerCase() === "fuel"); + if (!fuel) throw new Error("No resource named 'Fuel' found — cannot set vehicle fuel fields"); + return { id: fuel.id }; +} + +async function writeEntry( + payload: { create: (args: { collection: string; data: any }) => Promise; update: (args: { collection: string; id: number; data: any }) => Promise }, + plan: ImportPlan, + opts: { outDir: string }, + fuelId: number, +): Promise { + const common = { + class_name: plan.clazz, + display_name: plan.displayName, + description: plan.description || null, + }; + const mediaId = plan.pngPath || plan.screenshotPath ? await uploadMedia(payload, plan.pngPath || plan.screenshotPath).catch(() => undefined) : undefined; + + if (plan.kind === "vehicle") { + return payload.create({ + collection: "vehicles", + data: { + ...common, + vehicleName: plan.displayName, + ...(mediaId ? { vehicle_metadata_thumbnail: mediaId } : {}), + fuel: { fuelType: fuelId, fuelCapacity: 100, fuelConsumptionRate: 1 }, + }, + }).then((d: any) => `vehicle:${d.id}`); + } + return payload.create({ + collection: "assets", + data: { + ...common, + assetType: plan.assetType, + ...(plan.pngPath ? { "Asset Metadata": { mod: "Arma 3", thumbnail: mediaId, gameDescription: plan.description || null } } : {}), + }, + }).then((d: any) => `asset:${d.id}`); +} + +async function uploadMedia(payload: { create: (args: { collection: string; data: any; file: any }) => Promise }, filePath: string): Promise { + const { readFileSync } = await import("fs"); + const fileName = basename(filePath); + const buffer = readFileSync(filePath); + const res = await payload.create({ + collection: "media", + data: { alt: fileName, filename: fileName }, + file: { data: buffer, mimetype: "image/png", name: fileName, size: buffer.length }, + }); + return Number(res.id); +} + +async function main() { + const opts = parseCli(process.argv.slice(2)); + const hemtt = resolveHemtt(opts.hemtt?.[0]); + const catalogPath = opts.catalog?.[0]; + if (!catalogPath || !existsSync(catalogPath)) throw new Error("--catalog is required and must exist"); + const dryRun = !!opts["dry-run"]; + const approve = !!opts.approve || dryRun; + const outDir = opts.out?.[0] ?? dirname(catalogPath); + const paaRoot = opts["paa-root"]?.[0] ?? ""; + const screenshots = opts.screenshots?.[0] ?? ""; + if (!dryRun && !approve) throw new Error("--approve or --dry-run required to run without a real write"); + + const catalog = JSON.parse(readFileSync(catalogPath, "utf8")); + const records = Array.isArray(catalog) ? catalog : [catalog]; + let icons: Record = {}; + if (opts.icons?.[0] && existsSync(opts.icons[0])) { + icons = JSON.parse(readFileSync(opts.icons[0], "utf8")); + } + const limit = opts.limit ? parseInt(opts.limit[0], 10) : records.length; + const lookups = { icons, paaRoot, screenshots, outDir, hemtt }; + + console.log(`Import plan (dry-run=${dryRun}, approve=${approve}):`); + const plans: ImportPlan[] = []; + for (const rec of records.slice(0, limit)) { + const plan = planImport(rec, lookups); + if (!plan) { console.log(` SKIP ${String((rec as any).classname ?? "")} (not an importable record)`); continue; } + plans.push(plan); + const img = plan.pngPath ? "[img: item-png]" : plan.screenshotPath ? "[img: screenshot]" : "[no img]"; + console.log(` -> ${plan.kind}\t${plan.clazz}\t${plan.displayName}\t${img}`); + } + + if (dryRun) { console.log(`\nDry run: ${plans.length} would import, 0 applied.`); return; } + + const { getPayload } = await import("@payload-config"); + const payload = await getPayload({ config: (await import("@payload-config")).default }); + const fuel = await resolveFuelConfig(payload); + let ok = 0, err = 0; + for (const plan of plans) { + try { + const id = await writeEntry(payload, plan, { outDir }, fuel.id); + console.log(`Applied ${id}`); + ok++; + } catch (e) { + console.error(`ERR ${plan.clazz}: ${(e as Error).message}`); + err++; + } + } + console.log(`Done: ${ok} ok, ${err} errors.`); +} + +main().catch((e) => { console.error(e); process.exit(1); });