Env kill switch with raw-evidence retention, idempotent backfill, in-place dead-letter replay with reconciliation notes, command lease recovery on pull, ack terminal-state guards, and an operations runbook. Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
104 lines
2.7 KiB
TypeScript
104 lines
2.7 KiB
TypeScript
import type { Payload } from "payload";
|
|
import { asJson } from "./json";
|
|
|
|
type PayloadType = Awaited<ReturnType<typeof import("payload").getPayload>>;
|
|
|
|
/** A delivered command older than this is treated as lost and re-queued. */
|
|
export const COMMAND_LEASE_TIMEOUT_MS = 5 * 60_000;
|
|
|
|
export interface AckInput {
|
|
id: string;
|
|
success: boolean;
|
|
result?: unknown;
|
|
error?: string;
|
|
}
|
|
|
|
export interface AckOutcome {
|
|
ok: boolean;
|
|
duplicate?: boolean;
|
|
error?: "not-found" | "not-delivered";
|
|
}
|
|
|
|
/**
|
|
* Re-queue delivered commands whose lease expired without an ack. Returns the
|
|
* number of commands recovered.
|
|
*/
|
|
export async function recoverExpiredCommands(
|
|
payload: PayloadType,
|
|
serverId: number,
|
|
): Promise<number> {
|
|
const cutoff = new Date(Date.now() - COMMAND_LEASE_TIMEOUT_MS).toISOString();
|
|
const expired = await payload.find({
|
|
collection: "arma-commands",
|
|
where: {
|
|
and: [
|
|
{ server: { equals: serverId } },
|
|
{ status: { equals: "delivered" } },
|
|
{ deliveredAt: { less_than: cutoff } },
|
|
],
|
|
},
|
|
limit: 100,
|
|
depth: 0,
|
|
overrideAccess: true,
|
|
});
|
|
|
|
for (const command of expired.docs) {
|
|
await payload.update({
|
|
collection: "arma-commands",
|
|
id: command.id,
|
|
data: { status: "queued", deliveredAt: null },
|
|
overrideAccess: true,
|
|
depth: 0,
|
|
});
|
|
}
|
|
if (expired.docs.length > 0) {
|
|
payload.logger.warn(
|
|
`[ArmaBridge] Recovered ${expired.docs.length} expired command lease(s) for server ${serverId}`,
|
|
);
|
|
}
|
|
return expired.docs.length;
|
|
}
|
|
|
|
/**
|
|
* Apply a command acknowledgement with terminal-state and delivery guards.
|
|
* Duplicate acks are accepted as no-ops (the first result wins); acking a
|
|
* queued (never delivered) command is rejected.
|
|
*/
|
|
export async function acknowledgeCommand(
|
|
payload: PayloadType,
|
|
serverId: number,
|
|
input: AckInput,
|
|
): Promise<AckOutcome> {
|
|
const found = await payload.find({
|
|
collection: "arma-commands",
|
|
where: {
|
|
and: [{ commandId: { equals: input.id } }, { server: { equals: serverId } }],
|
|
},
|
|
limit: 1,
|
|
depth: 0,
|
|
overrideAccess: true,
|
|
});
|
|
const command = found.docs[0] as { id: number; status: string } | undefined;
|
|
if (!command) return { ok: false, error: "not-found" };
|
|
|
|
if (command.status === "succeeded" || command.status === "failed") {
|
|
return { ok: true, duplicate: true };
|
|
}
|
|
if (command.status !== "delivered") {
|
|
return { ok: false, error: "not-delivered" };
|
|
}
|
|
|
|
await payload.update({
|
|
collection: "arma-commands",
|
|
id: command.id,
|
|
data: {
|
|
status: input.success ? "succeeded" : "failed",
|
|
result: asJson(input.result, null),
|
|
error: input.error,
|
|
completedAt: new Date().toISOString(),
|
|
},
|
|
overrideAccess: true,
|
|
depth: 0,
|
|
});
|
|
return { ok: true };
|
|
}
|