import { getPayload, Payload } from "payload"; import config from "@/payload.config"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { acknowledgeCommand, recoverExpiredCommands } from "@/lib/arma-bridge/commands"; import { replayOperationEvent } from "@/lib/operations/reconcile"; import { backfillOperationEvents, isOperationLedgerEnabled } from "@/lib/operations/rollout"; let payload: Payload; const RUN = `ops-rollout-${Date.now().toString(36)}`; const syncEventIds: number[] = []; const opEventIds: number[] = []; const opEffectIds: number[] = []; const commandIds: number[] = []; const userIds: number[] = []; const roleIds: number[] = []; let serverId: number; let adminUserId: number; let plainUserId: number; let ledgerUserId: number; async function makeUser(label: string, roleDocIds?: number[]): Promise { const user = await payload.create({ collection: "users", data: { username: `${RUN}-${label}`, discordUsername: `${RUN}-${label}-discord`, displayName: `${RUN} ${label}`, steamId: `${RUN}-steam-${label}`, password: "Test1234", ...(roleDocIds && roleDocIds.length > 0 ? { roleDocs: roleDocIds } : {}), }, overrideAccess: true, depth: 0, }); userIds.push(user.id); return user.id; } async function makeCommand( label: string, status: "queued" | "delivered" | "succeeded" | "failed", deliveredAt?: string, ): Promise { const command = await payload.create({ collection: "arma-commands", data: { commandId: `${RUN}-cmd-${label}`, server: serverId, type: "test.command", payload: { action: "test" }, status, ...(deliveredAt ? { deliveredAt } : {}), }, overrideAccess: true, depth: 0, }); commandIds.push(command.id); return command.id; } async function opEventFor(rawId: number): Promise<{ id: number; status: string } | null> { const found = await payload.find({ collection: "operation-events", where: { sourceEvent: { equals: rawId } }, limit: 1, depth: 0, overrideAccess: true, }); return (found.docs[0] as { id: number; status: string } | undefined) ?? null; } beforeAll(async () => { const payloadConfig = await config; payload = await getPayload({ config: payloadConfig }); const server = await payload.create({ collection: "game-servers", data: { serverId: `${RUN}-srv`, name: `${RUN} Server`, status: "online" }, overrideAccess: true, depth: 0, }); serverId = server.id; const ledgerRole = await payload.create({ collection: "roles", data: { name: `${RUN} Ledger Op`, slug: `${RUN}-ledger-op`, permissions: ["operation-events:update"], }, overrideAccess: true, depth: 0, }); roleIds.push(ledgerRole.id); const superRole = await payload.create({ collection: "roles", data: { name: `${RUN} Rollout Admin`, slug: `${RUN}-rollout-admin`, isSuperuser: true }, overrideAccess: true, depth: 0, }); roleIds.push(superRole.id); plainUserId = await makeUser("plain"); ledgerUserId = await makeUser("ledger", [ledgerRole.id]); adminUserId = await makeUser("admin", [superRole.id]); }); afterAll(async () => { if (!payload) return; process.env.OPERATION_LEDGER_ENABLED = "true"; for (const id of opEffectIds) { await payload.delete({ collection: "operation-effects", id, overrideAccess: true }).catch(() => {}); } for (const id of opEventIds) { await payload.delete({ collection: "operation-events", id, overrideAccess: true }).catch(() => {}); } for (const id of syncEventIds) { await payload.delete({ collection: "arma-sync-events", id, overrideAccess: true }).catch(() => {}); } for (const id of commandIds) { await payload.delete({ collection: "arma-commands", id, overrideAccess: true }).catch(() => {}); } for (const id of userIds) { await payload.delete({ collection: "users", id, overrideAccess: true }).catch(() => {}); } for (const id of roleIds) { await payload.delete({ collection: "roles", id, overrideAccess: true }).catch(() => {}); } }); describe("rollout: staged activation", () => { it("defaults the ledger flag to enabled", () => { expect(isOperationLedgerEnabled()).toBe(true); }); it("stores raw evidence but no derived ledger while disabled, then backfills", async () => { const previous = process.env.OPERATION_LEDGER_ENABLED; process.env.OPERATION_LEDGER_ENABLED = "false"; try { const raw = await payload.create({ collection: "arma-sync-events", data: { eventId: `${RUN}-evt-disabled`, server: serverId, type: "operation.start", occurredAt: new Date().toISOString(), payload: { operationId: `${RUN}-op-disabled`, version: 1 }, }, overrideAccess: true, depth: 0, }); syncEventIds.push(raw.id); expect(await opEventFor(raw.id)).toBeNull(); const admin = await payload.findByID({ collection: "users", id: adminUserId, overrideAccess: true, depth: 0 }); expect(admin.id).toBe(adminUserId); const backfill = await backfillOperationEvents(payload, { id: adminUserId }); expect(backfill.forbidden).toBe(false); expect(backfill.processed).toBeGreaterThanOrEqual(1); const event = await opEventFor(raw.id); expect(event).not.toBeNull(); expect(event?.status).toBe("validated"); if (event) opEventIds.push(event.id); const effects = await payload.find({ collection: "operation-effects", where: { operationEvent: { equals: event?.id ?? -1 } }, limit: 10, depth: 0, overrideAccess: true, }); expect(effects.docs).toHaveLength(1); opEffectIds.push(effects.docs[0].id); const again = await backfillOperationEvents(payload, { id: adminUserId }); expect(again.processed).toBe(0); } finally { if (previous === undefined) delete process.env.OPERATION_LEDGER_ENABLED; else process.env.OPERATION_LEDGER_ENABLED = previous; } }); }); describe("rollout: command lease and ack", () => { it("re-queues only commands whose lease expired", async () => { const staleId = await makeCommand( "stale", "delivered", new Date(Date.now() - 10 * 60_000).toISOString(), ); const freshId = await makeCommand("fresh", "delivered", new Date().toISOString()); const recovered = await recoverExpiredCommands(payload, serverId); expect(recovered).toBeGreaterThanOrEqual(1); const stale = await payload.findByID({ collection: "arma-commands", id: staleId, overrideAccess: true, depth: 0 }); expect((stale as unknown as { status: string }).status).toBe("queued"); expect((stale as unknown as { deliveredAt: unknown }).deliveredAt ?? null).toBeNull(); const fresh = await payload.findByID({ collection: "arma-commands", id: freshId, overrideAccess: true, depth: 0 }); expect((fresh as unknown as { status: string }).status).toBe("delivered"); }); it("accepts a first ack, treats duplicates as no-ops, and rejects acking undelivered commands", async () => { const deliveredId = await makeCommand("acked", "delivered", new Date().toISOString()); const queuedId = await makeCommand("never-delivered", "queued"); const first = await acknowledgeCommand(payload, serverId, { id: `${RUN}-cmd-acked`, success: true, result: { done: true }, }); expect(first.ok).toBe(true); expect(first.duplicate).toBeUndefined(); const afterFirst = await payload.findByID({ collection: "arma-commands", id: deliveredId, overrideAccess: true, depth: 0, }); const completedAt = (afterFirst as unknown as { completedAt: string }).completedAt; const duplicate = await acknowledgeCommand(payload, serverId, { id: `${RUN}-cmd-acked`, success: false, error: "late duplicate", }); expect(duplicate.ok).toBe(true); expect(duplicate.duplicate).toBe(true); const afterDuplicate = await payload.findByID({ collection: "arma-commands", id: deliveredId, overrideAccess: true, depth: 0, }); expect((afterDuplicate as unknown as { status: string }).status).toBe("succeeded"); expect((afterDuplicate as unknown as { completedAt: string }).completedAt).toBe(completedAt); const undelivered = await acknowledgeCommand(payload, serverId, { id: `${RUN}-cmd-never-delivered`, success: true, }); expect(undelivered.ok).toBe(false); expect(undelivered.error).toBe("not-delivered"); const missing = await acknowledgeCommand(payload, serverId, { id: `${RUN}-cmd-does-not-exist`, success: true, }); expect(missing.ok).toBe(false); expect(missing.error).toBe("not-found"); void queuedId; }); }); describe("rollout: reconciliation authorization and replay", () => { it("forbids replay without the operation-events:update permission", async () => { const raw = await payload.create({ collection: "arma-sync-events", data: { eventId: `${RUN}-evt-dead`, server: serverId, type: "operation.unsupported.thing", occurredAt: new Date().toISOString(), payload: { anything: true }, }, overrideAccess: true, depth: 0, }); syncEventIds.push(raw.id); const event = await opEventFor(raw.id); expect(event?.status).toBe("dead-letter"); if (event) opEventIds.push(event.id); const forbidden = await replayOperationEvent(payload, { id: plainUserId }, event?.id ?? -1); expect(forbidden.ok).toBe(false); expect(forbidden.error).toBe("forbidden"); }); it("replays a dead-letter row in place, keeps invalid evidence invalid, and creates no effects", async () => { const raw = await payload.create({ collection: "arma-sync-events", data: { eventId: `${RUN}-evt-dead-2`, server: serverId, type: "operation.still-unsupported", occurredAt: new Date().toISOString(), payload: { anything: true }, }, overrideAccess: true, depth: 0, }); syncEventIds.push(raw.id); const event = await opEventFor(raw.id); expect(event?.status).toBe("dead-letter"); if (event) opEventIds.push(event.id); const result = await replayOperationEvent(payload, { id: ledgerUserId }, event?.id ?? -1); expect(result.ok).toBe(true); expect(result.status).toBe("dead-letter"); expect(result.effectsCreated).toBe(0); const refreshed = await opEventFor(raw.id); expect((refreshed as unknown as { reconciliationNotes: string | null } | null)?.reconciliationNotes ?? "").toContain( "Replayed by user", ); }); it("rejects replay of an already-validated row", async () => { const raw = await payload.create({ collection: "arma-sync-events", data: { eventId: `${RUN}-evt-valid`, server: serverId, type: "operation.start", occurredAt: new Date().toISOString(), payload: { operationId: `${RUN}-op-valid`, version: 1 }, }, overrideAccess: true, depth: 0, }); syncEventIds.push(raw.id); const event = await opEventFor(raw.id); expect(event?.status).toBe("validated"); if (event) opEventIds.push(event.id); const result = await replayOperationEvent(payload, { id: ledgerUserId }, event?.id ?? -1); expect(result.ok).toBe(false); expect(result.error).toBe("already-validated"); }); });