From 287a0f7db86b1fe010a7ed607bb0b674a805d2d9 Mon Sep 17 00:00:00 2001 From: Z8MB1E Date: Wed, 26 Aug 2026 18:05:57 -0400 Subject: [PATCH] feat(server): add stale presence monitoring --- AGENTS.md | 3 +- DEPLOYMENT.md | 5 +- README.md | 3 +- src/collections/game/GameEventLogs.ts | 1 + src/lib/arma-bridge/presence.ts | 83 ++++++++++++ ...26_060000_add_game_servers_event_target.ts | 12 ++ src/payload-generated-schema.ts | 5 +- src/payload-types.ts | 1 + src/payload.config.ts | 4 + src/scripts/lib/binFileLogger.ts | 2 +- src/scripts/serverTick.ts | 46 +++++++ tests/int/server-presence.int.spec.ts | 118 ++++++++++++++++++ 12 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 src/lib/arma-bridge/presence.ts create mode 100644 src/migrations/20260826_060000_add_game_servers_event_target.ts create mode 100644 src/scripts/serverTick.ts create mode 100644 tests/int/server-presence.int.spec.ts diff --git a/AGENTS.md b/AGENTS.md index a2d9711..8207e61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -147,8 +147,9 @@ Shipping simulation (`src/collections/logistics/Shipments.ts`, `src/scripts/`, ` - **Shipment fields**: `origin`/`destination` → `game-structures`, `transportVehicle` → `game-vehicles`, `cargo[]` (relationship to resources/assets/vehicles + amount), `distance`, `fuelCost`/`fuelConsumed`, `status` (`pending`/`dispatched`/`in_transit`/`arrived`/`completed`/`cancelled`/`failed`/`stranded`), `autoReturn` checkbox, `failureReason`. - **Game tick**: `bun run payload game-tick` — a `bin` registered on `payload.config.ts`, NOT an npm script. It processes active shipments + fuel consumption, then `process.exit(0)`. -- **Bin file logging**: all four bins (`game-tick`, `market-tick`, `mission-tick`, `generate-mission`) tee their logs to daily files `logs//.log` (UTC) via `createBinLogger` in `src/scripts/lib/binFileLogger.ts` — in addition to the console. Writes are `appendFileSync` (bin scripts `process.exit` immediately, so async streams would truncate). A fatal error in a bin is caught, logged to the file, and exits code 1. Directory override: `BIN_LOG_DIR` env (defaults to `/logs` — inside Docker point it at a persistent volume). `logs/` is gitignored. +- **Bin file logging**: all five bins (`game-tick`, `market-tick`, `mission-tick`, `server-tick`, `generate-mission`) tee their logs to daily files `logs//.log` (UTC) via `createBinLogger` in `src/scripts/lib/binFileLogger.ts` — in addition to the console. Writes are `appendFileSync` (bin scripts `process.exit` immediately, so async streams would truncate). A fatal error in a bin is caught, logged to the file, and exits code 1. Directory override: `BIN_LOG_DIR` env (defaults to `/logs` — inside Docker point it at a persistent volume). `logs/` is gitignored. - **Mission auto-completion**: `bun run payload mission-tick` — bin registered like the others (needs external cron). Sweeps missions whose status is `Scheduled`/`Active` and whose scheduled date (`classification.startDateTime`) has fully passed — from the start of the following **server-local** day — and sets them to `Completed`, emitting a `mission:auto-complete` event per mission (logic: `src/lib/intelligence/missionLifecycle.ts`, test: `tests/int/mission-lifecycle.int.spec.ts`). Draft statuses (Concept/Planning/Ready) and terminal statuses (Completed/Cancelled) are never touched; idempotent. Notifies clients via the same SSE path as the other ticks. +- **Server presence**: `bun run payload server-tick` — bin registered like the others (needs external cron, every minute). Flips `game-servers` docs that claim `status: "online"` but whose last heartbeat (`lastSeenAt`) is older than 180s to `offline`, including online docs with no `lastSeenAt` at all; emits a `server:offline` event per flipped server (logic: `src/lib/arma-bridge/presence.ts`, test: `tests/int/server-presence.int.spec.ts`). Idempotent. Notifies clients via the same SSE path as the other ticks. - **Arrival handling** (`src/scripts/processShipmentTick.ts`): destination storage rules are re-checked on arrival; rejected cargo bounces back to origin, shipment goes `failed`, `ShipmentFail` event logged. - **GameRules tuning**: `proximityThreshold` and `gameTickIntervalMinutes` live on the global `game-rules` doc. - **UI**: `src/app/(frontend)/logistics/shipments/` (list + `[id]` detail with `ShipmentActions` controls), `src/app/(frontend)/logistics/game-vehicles/` (deployed vehicle views). diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 96cdc21..37f6d3f 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -115,7 +115,7 @@ The bot ships inside the same app image and starts **only when `DISCORD_TOKEN` i --- -## 6. Scheduled jobs (game-tick, market-tick) +## 6. Scheduled jobs (game-tick, market-tick, server-tick) The `game-tick` and `market-tick` Payload bins are registered on `payload.config.ts` and expected to run on a regular cadence (cron). In Coolify: @@ -127,6 +127,7 @@ Coolify has a **Scheduled Tasks** feature per service (` → Scheduled |---|---|---| | Process shipments | `*/5 * * * *` (every 5 min, tune to match `gameTickIntervalMinutes` in your Game Rules global) | `node node_modules/payload/bin.js game-tick` | | Refresh market | `*/10 * * * *` (every 10 min — adjust as needed) | `node node_modules/payload/bin.js market-tick` | +| Server presence | `* * * * *` (every minute) | `node node_modules/payload/bin.js server-tick` | The commands run inside the app container with the app's env (incl. `DATABASE_URI`, `APP_URL`, `GAME_TICK_NOTIFY_SECRET`), so the bins talk to the same DB and post the notify callback correctly. @@ -139,6 +140,8 @@ If you prefer existing infra: docker exec polaris-task-force-app- node node_modules/payload/bin.js game-tick */10 * * * * ssh -i /path/to/key root@coolify-host \ docker exec polaris-task-force-app- node node_modules/payload/bin.js market-tick +* * * * * ssh -i /path/to/key root@coolify-host \ + docker exec polaris-task-force-app- node node_modules/payload/bin.js server-tick ``` The container name comes from Coolify's per-service container name (visible on the service dashboard). diff --git a/README.md b/README.md index 187e141..d4155a7 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,12 @@ Dev credentials: `dev` / `Test123` (if seed data has been applied). | `bun run payload game-tick` | Process shipment arrivals and fuel consumption | | `bun run payload market-tick` | Expire listings and refresh NPC vendor stock | | `bun run payload mission-tick` | Auto-complete missions whose scheduled day has passed | +| `bun run payload server-tick` | Mark game servers without a recent heartbeat offline | | `bun run deploy` | Bump patch version, build, and deploy | Note: `game-tick` and `market-tick` are Payload bins registered in `payload.config.ts`, not npm scripts. Run via `bun run payload`. -All four bin scripts also log to daily files under `logs//` (e.g. `logs/game-tick/2026-08-24.log`) so tick output can be tracked over time. Override the directory with the `BIN_LOG_DIR` env var. +All five bin scripts also log to daily files under `logs//` (e.g. `logs/game-tick/2026-08-24.log`) so tick output can be tracked over time. Override the directory with the `BIN_LOG_DIR` env var. For an Arma 3 unit deployment we self-host on **Coolify** with one container per environment (dev / stg / prod), a Coolify-managed PostgreSQL service per environment, a persistent volume for `/app/media`, and `docker exec` scheduled jobs for the game tick. The Dockerfile is Bun-based (multi-stage, Next.js standalone runtime + Payload CLI kept at runtime), and `docker-compose.yml` mirrors the same shape for local full-stack dev. diff --git a/src/collections/game/GameEventLogs.ts b/src/collections/game/GameEventLogs.ts index b864c7a..19c7e39 100644 --- a/src/collections/game/GameEventLogs.ts +++ b/src/collections/game/GameEventLogs.ts @@ -24,6 +24,7 @@ const TARGET_COLLECTIONS = [ { label: "Market Listings", value: "market-listings" }, { label: "Market Negotiations", value: "market-negotiations" }, { label: "Tickets", value: "tickets" }, + { label: "Game Servers", value: "game-servers" }, ] as const; export const GameEventLogs: CollectionConfig = { diff --git a/src/lib/arma-bridge/presence.ts b/src/lib/arma-bridge/presence.ts new file mode 100644 index 0000000..72183ec --- /dev/null +++ b/src/lib/arma-bridge/presence.ts @@ -0,0 +1,83 @@ +import type { Where } from "payload"; +import type { GameServer } from "@/payload-types"; +import { emitGameEvent } from "@/utils/event-log/emit"; +import { EventTypes } from "@/utils/event-log/eventTypes"; + +type Payload = Awaited>; + +/** + * A server whose last successful heartbeat is older than this window is + * considered dead and flipped to `offline` by the server-tick sweep. + */ +export const SERVER_OFFLINE_AFTER_MS = 180_000; + +export interface ServerPresenceSummary { + markedOffline: number; + servers: { id: number; serverId: string; name: string }[]; +} + +export interface MarkStaleServersOfflineOptions { + /** Restrict the sweep to specific server ids (keeps tests hermetic). */ + serverIds?: number[]; +} + +/** + * Find `game-servers` docs that claim `status: "online"` but have not + * heartbeated within SERVER_OFFLINE_AFTER_MS and flip them to `offline`. + * + * A doc with no `lastSeenAt` has never heartbeated, so it can never be + * online — those are swept too (self-heals manually set-online docs). + */ +export async function markStaleServersOffline( + payload: Payload, + now: Date = new Date(), + options: MarkStaleServersOfflineOptions = {}, +): Promise { + const cutoff = new Date(now.getTime() - SERVER_OFFLINE_AFTER_MS).toISOString(); + + const conditions: Where[] = [ + { status: { equals: "online" } }, + { + or: [{ lastSeenAt: { less_than: cutoff } }, { lastSeenAt: { exists: false } }], + }, + ]; + if (options.serverIds) conditions.push({ id: { in: options.serverIds } }); + + const stale = await payload.find({ + collection: "game-servers", + where: { and: conditions }, + sort: "lastSeenAt", + limit: 100, + depth: 0, + overrideAccess: true, + }); + + const servers: ServerPresenceSummary["servers"] = []; + for (const server of stale.docs as GameServer[]) { + try { + await payload.update({ + collection: "game-servers", + id: server.id, + data: { status: "offline" }, + overrideAccess: true, + }); + await emitGameEvent(payload, { + type: EventTypes.ServerOffline, + message: `Server "${server.name}" (${server.serverId}) marked offline — no heartbeat for ${Math.round( + SERVER_OFFLINE_AFTER_MS / 1000, + )}s.`, + targetCollection: "game-servers", + targetId: server.id, + data: { lastSeenAt: server.lastSeenAt ?? null }, + }); + servers.push({ id: server.id, serverId: server.serverId, name: server.name }); + } catch (error) { + const message = error instanceof Error ? error.message : "Unknown error"; + payload.logger.error( + `[Server Tick] Failed to mark server ${server.id} (${server.serverId}) offline: ${message}`, + ); + } + } + + return { markedOffline: servers.length, servers }; +} diff --git a/src/migrations/20260826_060000_add_game_servers_event_target.ts b/src/migrations/20260826_060000_add_game_servers_event_target.ts new file mode 100644 index 0000000..1ac09d9 --- /dev/null +++ b/src/migrations/20260826_060000_add_game_servers_event_target.ts @@ -0,0 +1,12 @@ +import { MigrateDownArgs, MigrateUpArgs, sql } from "@payloadcms/db-postgres"; + +export async function up({ db }: MigrateUpArgs): Promise { + await db.execute(sql` + ALTER TYPE "public"."enum_game_event_logs_target_collection" + ADD VALUE IF NOT EXISTS 'game-servers'; + `); +} + +export async function down({ db }: MigrateDownArgs): Promise { + void db; +} diff --git a/src/payload-generated-schema.ts b/src/payload-generated-schema.ts index 3b42435..c24aa7a 100644 --- a/src/payload-generated-schema.ts +++ b/src/payload-generated-schema.ts @@ -33,8 +33,9 @@ export const enum_game_vehicles_status = pgEnum("enum_game_vehicles_status", [ ]); export const enum_game_event_logs_target_collection = pgEnum( "enum_game_event_logs_target_collection", - [ - "game-structures", + [ + "game-servers", + "game-structures", "game-vehicles", "game-npcs", "structures", diff --git a/src/payload-types.ts b/src/payload-types.ts index d5c5b3b..e830dca 100644 --- a/src/payload-types.ts +++ b/src/payload-types.ts @@ -1584,6 +1584,7 @@ export interface GameEventLog { | 'market-listings' | 'market-negotiations' | 'tickets' + | 'game-servers' ) | null; targetId?: number | null; diff --git a/src/payload.config.ts b/src/payload.config.ts index 5934693..8a2eacc 100644 --- a/src/payload.config.ts +++ b/src/payload.config.ts @@ -282,6 +282,10 @@ export default buildConfig({ scriptPath: path.resolve(dirname, "scripts/missionTick.ts"), key: "mission-tick", }, + { + scriptPath: path.resolve(dirname, "scripts/serverTick.ts"), + key: "server-tick", + }, { scriptPath: path.resolve(dirname, "scripts/generateNextMission.ts"), key: "generate-mission", diff --git a/src/scripts/lib/binFileLogger.ts b/src/scripts/lib/binFileLogger.ts index 5272898..47920c0 100644 --- a/src/scripts/lib/binFileLogger.ts +++ b/src/scripts/lib/binFileLogger.ts @@ -3,7 +3,7 @@ import path from "node:path"; /** * File logging for Payload bin scripts (game-tick, market-tick, mission-tick, - * generate-mission). + * server-tick, generate-mission). * * Each log call is appended SYNCHRONOUSLY to a daily file under * `logs//.log` (UTC) and also forwarded to the console diff --git a/src/scripts/serverTick.ts b/src/scripts/serverTick.ts new file mode 100644 index 0000000..27ab711 --- /dev/null +++ b/src/scripts/serverTick.ts @@ -0,0 +1,46 @@ +import type { SanitizedConfig } from "payload"; +import { getPayload } from "payload"; +import { markStaleServersOffline } from "@/lib/arma-bridge/presence"; +import { createBinLogger, errorMessage } from "./lib/binFileLogger"; + +export const script = async (config: SanitizedConfig) => { + const payload = await getPayload({ config }); + const log = createBinLogger("server-tick", payload.logger); + + try { + log.info("[Server Tick] Starting server presence tick..."); + + const nowIso = new Date().toISOString(); + const result = await markStaleServersOffline(payload); + + const notifyUrl = (process.env.APP_URL ?? "http://localhost:3000").replace(/\/$/, ""); + const notifySecret = process.env.GAME_TICK_NOTIFY_SECRET; + if (notifySecret) { + await fetch(`${notifyUrl}/api/game-tick/notify`, { + method: "POST", + headers: { + "content-type": "application/json", + "x-game-tick-secret": notifySecret, + }, + body: JSON.stringify({ processedAt: nowIso, source: "server-tick" }), + }).catch((error: unknown) => { + log.error(`[Server Tick] Failed to notify app clients: ${errorMessage(error)}`); + }); + } else { + log.warn("[Server Tick] GAME_TICK_NOTIFY_SECRET not set; skipping client notification"); + } + + if (result.markedOffline > 0) { + const names = result.servers.map((s) => s.serverId).join(", "); + log.info(`[Server Tick] Complete. Marked ${result.markedOffline} server(s) offline: ${names}`); + } else { + log.info("[Server Tick] Complete. No stale online servers."); + } + process.exit(0); + } catch (error) { + log.error( + `[Server Tick] Fatal: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`, + ); + process.exit(1); + } +}; diff --git a/tests/int/server-presence.int.spec.ts b/tests/int/server-presence.int.spec.ts new file mode 100644 index 0000000..7427883 --- /dev/null +++ b/tests/int/server-presence.int.spec.ts @@ -0,0 +1,118 @@ +import { getPayload, Payload } from "payload"; +import config from "@/payload.config"; + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import type { GameServer } from "@/payload-types"; +import { markStaleServersOffline, SERVER_OFFLINE_AFTER_MS } from "@/lib/arma-bridge/presence"; + +let payload: Payload; + +const RUN = `srv-${Date.now().toString(36)}`; + +describe("Server presence sweep", () => { + let now: Date; + const serverIds: number[] = []; + + beforeAll(async () => { + const payloadConfig = await config; + payload = await getPayload({ config: payloadConfig }); + now = new Date(); + now.setMilliseconds(0); + }); + + afterAll(async () => { + for (const id of serverIds) { + await payload.delete({ collection: "game-servers", id, overrideAccess: true }).catch(() => {}); + } + const events = await payload + .find({ + collection: "game-event-logs", + where: { and: [{ type: { equals: "server:offline" } }, { targetId: { in: serverIds } }] }, + limit: 50, + depth: 0, + overrideAccess: true, + }) + .catch(() => null); + for (const event of events?.docs ?? []) { + await payload + .delete({ collection: "game-event-logs", id: event.id, overrideAccess: true }) + .catch(() => {}); + } + }); + + const makeServer = async ( + label: string, + data: { status?: GameServer["status"]; lastSeenAt?: Date }, + ): Promise => { + const server = (await payload.create({ + collection: "game-servers", + data: { + serverId: `${RUN}-${label}`, + name: `${RUN} ${label}`, + status: data.status ?? "online", + lastSeenAt: data.lastSeenAt ? data.lastSeenAt.toISOString() : undefined, + }, + overrideAccess: true, + depth: 0, + })) as unknown as GameServer; + serverIds.push(server.id); + return server; + }; + + const fetchServer = (id: number): Promise => + payload.findByID({ + collection: "game-servers", + id, + depth: 0, + overrideAccess: true, + }) as unknown as Promise; + + it("flips online servers with no recent heartbeat to offline and leaves the rest alone", async () => { + const fresh = await makeServer("fresh", { lastSeenAt: new Date(now.getTime() - 60_000) }); + const stale = await makeServer("stale", { + lastSeenAt: new Date(now.getTime() - (SERVER_OFFLINE_AFTER_MS + 1_000)), + }); + const neverSeen = await makeServer("never-seen", {}); + const offlineStale = await makeServer("offline-stale", { + status: "offline", + lastSeenAt: new Date(now.getTime() - 3_600_000), + }); + const boundary = await makeServer("boundary", { + lastSeenAt: new Date(now.getTime() - SERVER_OFFLINE_AFTER_MS), + }); + + // Scope to this run's fixtures so the sweep never flips other servers in the + // shared dev database (stale real servers would be flipped legitimately). + const result = await markStaleServersOffline(payload, now, { serverIds }); + + expect(result.markedOffline).toBe(2); + expect(result.servers.map((s) => s.id).sort()).toEqual([stale.id, neverSeen.id].sort()); + expect(result.servers.find((s) => s.id === stale.id)?.serverId).toBe(`${RUN}-stale`); + + expect((await fetchServer(fresh.id)).status).toBe("online"); + expect((await fetchServer(stale.id)).status).toBe("offline"); + expect((await fetchServer(neverSeen.id)).status).toBe("offline"); + expect((await fetchServer(offlineStale.id)).status).toBe("offline"); + // Exactly at the cutoff the server is still within its heartbeat window. + expect((await fetchServer(boundary.id)).status).toBe("online"); + }); + + it("logs a server:offline event per flipped server", async () => { + const events = await payload.find({ + collection: "game-event-logs", + where: { + and: [{ type: { equals: "server:offline" } }, { targetId: { in: serverIds } }], + }, + limit: 10, + depth: 0, + overrideAccess: true, + }); + expect(events.docs).toHaveLength(2); + }); + + it("is idempotent — a second sweep finds nothing left to flip", async () => { + const result = await markStaleServersOffline(payload, now, { serverIds }); + expect(result.markedOffline).toBe(0); + expect(result.servers).toEqual([]); + }); +});