polaris-task-force/tests/int/operation-rollout.int.spec.ts

import { getPayload, Payload } from "payload";
import config from "@/payload.config";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { acknowledgeCommand, recoverExpiredCommands } from "@/lib/arma-bridge/commands";
import { replayOperationEvent } from "@/lib/operations/reconcile";
import { backfillOperationEvents, isOperationLedgerEnabled } from "@/lib/operations/rollout";

let payload: Payload;
const RUN = `ops-rollout-${Date.now().toString(36)}`;

const syncEventIds: number[] = [];
const opEventIds: number[] = [];
const opEffectIds: number[] = [];
const commandIds: number[] = [];
const userIds: number[] = [];
const roleIds: number[] = [];
let serverId: number;
let adminUserId: number;
let plainUserId: number;
let ledgerUserId: number;

async function makeUser(label: string, roleDocIds?: number[]): Promise<number> {
  const user = await payload.create({
    collection: "users",
    data: {
      username: `${RUN}-${label}`,
      discordUsername: `${RUN}-${label}-discord`,
      displayName: `${RUN} ${label}`,
      steamId: `${RUN}-steam-${label}`,
      password: "Test1234",
      ...(roleDocIds && roleDocIds.length > 0 ? { roleDocs: roleDocIds } : {}),
    },
    overrideAccess: true,
    depth: 0,
  });
  userIds.push(user.id);
  return user.id;
}

async function makeCommand(
  label: string,
  status: "queued" | "delivered" | "succeeded" | "failed",
  deliveredAt?: string,
): Promise<number> {
  const command = await payload.create({
    collection: "arma-commands",
    data: {
      commandId: `${RUN}-cmd-${label}`,
      server: serverId,
      type: "test.command",
      payload: { action: "test" },
      status,
      ...(deliveredAt ? { deliveredAt } : {}),
    },
    overrideAccess: true,
    depth: 0,
  });
  commandIds.push(command.id);
  return command.id;
}

async function opEventFor(rawId: number): Promise<{ id: number; status: string } | null> {
  const found = await payload.find({
    collection: "operation-events",
    where: { sourceEvent: { equals: rawId } },
    limit: 1,
    depth: 0,
    overrideAccess: true,
  });
  return (found.docs[0] as { id: number; status: string } | undefined) ?? null;
}

beforeAll(async () => {
  const payloadConfig = await config;
  payload = await getPayload({ config: payloadConfig });

  const server = await payload.create({
    collection: "game-servers",
    data: { serverId: `${RUN}-srv`, name: `${RUN} Server`, status: "online" },
    overrideAccess: true,
    depth: 0,
  });
  serverId = server.id;

  const ledgerRole = await payload.create({
    collection: "roles",
    data: {
      name: `${RUN} Ledger Op`,
      slug: `${RUN}-ledger-op`,
      permissions: ["operation-events:update"],
    },
    overrideAccess: true,
    depth: 0,
  });
  roleIds.push(ledgerRole.id);

  const superRole = await payload.create({
    collection: "roles",
    data: { name: `${RUN} Rollout Admin`, slug: `${RUN}-rollout-admin`, isSuperuser: true },
    overrideAccess: true,
    depth: 0,
  });
  roleIds.push(superRole.id);

  plainUserId = await makeUser("plain");
  ledgerUserId = await makeUser("ledger", [ledgerRole.id]);
  adminUserId = await makeUser("admin", [superRole.id]);
});

afterAll(async () => {
  if (!payload) return;
  process.env.OPERATION_LEDGER_ENABLED = "true";
  for (const id of opEffectIds) {
    await payload.delete({ collection: "operation-effects", id, overrideAccess: true }).catch(() => {});
  }
  for (const id of opEventIds) {
    await payload.delete({ collection: "operation-events", id, overrideAccess: true }).catch(() => {});
  }
  for (const id of syncEventIds) {
    await payload.delete({ collection: "arma-sync-events", id, overrideAccess: true }).catch(() => {});
  }
  for (const id of commandIds) {
    await payload.delete({ collection: "arma-commands", id, overrideAccess: true }).catch(() => {});
  }
  for (const id of userIds) {
    await payload.delete({ collection: "users", id, overrideAccess: true }).catch(() => {});
  }
  for (const id of roleIds) {
    await payload.delete({ collection: "roles", id, overrideAccess: true }).catch(() => {});
  }
});

describe("rollout: staged activation", () => {
  it("defaults the ledger flag to enabled", () => {
    expect(isOperationLedgerEnabled()).toBe(true);
  });

  it("stores raw evidence but no derived ledger while disabled, then backfills", async () => {
    const previous = process.env.OPERATION_LEDGER_ENABLED;
    process.env.OPERATION_LEDGER_ENABLED = "false";
    try {
      const raw = await payload.create({
        collection: "arma-sync-events",
        data: {
          eventId: `${RUN}-evt-disabled`,
          server: serverId,
          type: "operation.start",
          occurredAt: new Date().toISOString(),
          payload: { operationId: `${RUN}-op-disabled`, version: 1 },
        },
        overrideAccess: true,
        depth: 0,
      });
      syncEventIds.push(raw.id);

      expect(await opEventFor(raw.id)).toBeNull();

      const admin = await payload.findByID({ collection: "users", id: adminUserId, overrideAccess: true, depth: 0 });
      expect(admin.id).toBe(adminUserId);
      const backfill = await backfillOperationEvents(payload, { id: adminUserId });
      expect(backfill.forbidden).toBe(false);
      expect(backfill.processed).toBeGreaterThanOrEqual(1);

      const event = await opEventFor(raw.id);
      expect(event).not.toBeNull();
      expect(event?.status).toBe("validated");
      if (event) opEventIds.push(event.id);

      const effects = await payload.find({
        collection: "operation-effects",
        where: { operationEvent: { equals: event?.id ?? -1 } },
        limit: 10,
        depth: 0,
        overrideAccess: true,
      });
      expect(effects.docs).toHaveLength(1);
      opEffectIds.push(effects.docs[0].id);

      const again = await backfillOperationEvents(payload, { id: adminUserId });
      expect(again.processed).toBe(0);
    } finally {
      if (previous === undefined) delete process.env.OPERATION_LEDGER_ENABLED;
      else process.env.OPERATION_LEDGER_ENABLED = previous;
    }
  });
});

describe("rollout: command lease and ack", () => {
  it("re-queues only commands whose lease expired", async () => {
    const staleId = await makeCommand(
      "stale",
      "delivered",
      new Date(Date.now() - 10 * 60_000).toISOString(),
    );
    const freshId = await makeCommand("fresh", "delivered", new Date().toISOString());

    const recovered = await recoverExpiredCommands(payload, serverId);
    expect(recovered).toBeGreaterThanOrEqual(1);

    const stale = await payload.findByID({ collection: "arma-commands", id: staleId, overrideAccess: true, depth: 0 });
    expect((stale as unknown as { status: string }).status).toBe("queued");
    expect((stale as unknown as { deliveredAt: unknown }).deliveredAt ?? null).toBeNull();

    const fresh = await payload.findByID({ collection: "arma-commands", id: freshId, overrideAccess: true, depth: 0 });
    expect((fresh as unknown as { status: string }).status).toBe("delivered");
  });

  it("accepts a first ack, treats duplicates as no-ops, and rejects acking undelivered commands", async () => {
    const deliveredId = await makeCommand("acked", "delivered", new Date().toISOString());
    const queuedId = await makeCommand("never-delivered", "queued");

    const first = await acknowledgeCommand(payload, serverId, {
      id: `${RUN}-cmd-acked`,
      success: true,
      result: { done: true },
    });
    expect(first.ok).toBe(true);
    expect(first.duplicate).toBeUndefined();

    const afterFirst = await payload.findByID({
      collection: "arma-commands",
      id: deliveredId,
      overrideAccess: true,
      depth: 0,
    });
    const completedAt = (afterFirst as unknown as { completedAt: string }).completedAt;

    const duplicate = await acknowledgeCommand(payload, serverId, {
      id: `${RUN}-cmd-acked`,
      success: false,
      error: "late duplicate",
    });
    expect(duplicate.ok).toBe(true);
    expect(duplicate.duplicate).toBe(true);

    const afterDuplicate = await payload.findByID({
      collection: "arma-commands",
      id: deliveredId,
      overrideAccess: true,
      depth: 0,
    });
    expect((afterDuplicate as unknown as { status: string }).status).toBe("succeeded");
    expect((afterDuplicate as unknown as { completedAt: string }).completedAt).toBe(completedAt);

    const undelivered = await acknowledgeCommand(payload, serverId, {
      id: `${RUN}-cmd-never-delivered`,
      success: true,
    });
    expect(undelivered.ok).toBe(false);
    expect(undelivered.error).toBe("not-delivered");

    const missing = await acknowledgeCommand(payload, serverId, {
      id: `${RUN}-cmd-does-not-exist`,
      success: true,
    });
    expect(missing.ok).toBe(false);
    expect(missing.error).toBe("not-found");

    void queuedId;
  });
});

describe("rollout: reconciliation authorization and replay", () => {
  it("forbids replay without the operation-events:update permission", async () => {
    const raw = await payload.create({
      collection: "arma-sync-events",
      data: {
        eventId: `${RUN}-evt-dead`,
        server: serverId,
        type: "operation.unsupported.thing",
        occurredAt: new Date().toISOString(),
        payload: { anything: true },
      },
      overrideAccess: true,
      depth: 0,
    });
    syncEventIds.push(raw.id);
    const event = await opEventFor(raw.id);
    expect(event?.status).toBe("dead-letter");
    if (event) opEventIds.push(event.id);

    const forbidden = await replayOperationEvent(payload, { id: plainUserId }, event?.id ?? -1);
    expect(forbidden.ok).toBe(false);
    expect(forbidden.error).toBe("forbidden");
  });

  it("replays a dead-letter row in place, keeps invalid evidence invalid, and creates no effects", async () => {
    const raw = await payload.create({
      collection: "arma-sync-events",
      data: {
        eventId: `${RUN}-evt-dead-2`,
        server: serverId,
        type: "operation.still-unsupported",
        occurredAt: new Date().toISOString(),
        payload: { anything: true },
      },
      overrideAccess: true,
      depth: 0,
    });
    syncEventIds.push(raw.id);
    const event = await opEventFor(raw.id);
    expect(event?.status).toBe("dead-letter");
    if (event) opEventIds.push(event.id);

    const result = await replayOperationEvent(payload, { id: ledgerUserId }, event?.id ?? -1);
    expect(result.ok).toBe(true);
    expect(result.status).toBe("dead-letter");
    expect(result.effectsCreated).toBe(0);

    const refreshed = await opEventFor(raw.id);
    expect((refreshed as unknown as { reconciliationNotes: string | null } | null)?.reconciliationNotes ?? "").toContain(
      "Replayed by user",
    );
  });

  it("rejects replay of an already-validated row", async () => {
    const raw = await payload.create({
      collection: "arma-sync-events",
      data: {
        eventId: `${RUN}-evt-valid`,
        server: serverId,
        type: "operation.start",
        occurredAt: new Date().toISOString(),
        payload: { operationId: `${RUN}-op-valid`, version: 1 },
      },
      overrideAccess: true,
      depth: 0,
    });
    syncEventIds.push(raw.id);
    const event = await opEventFor(raw.id);
    expect(event?.status).toBe("validated");
    if (event) opEventIds.push(event.id);

    const result = await replayOperationEvent(payload, { id: ledgerUserId }, event?.id ?? -1);
    expect(result.ok).toBe(false);
    expect(result.error).toBe("already-validated");
  });
});