summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/hooks/fleet
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/code-modernization/hooks/fleet')
-rw-r--r--plugins/code-modernization/hooks/fleet/fleet.ts350
1 files changed, 350 insertions, 0 deletions
diff --git a/plugins/code-modernization/hooks/fleet/fleet.ts b/plugins/code-modernization/hooks/fleet/fleet.ts
new file mode 100644
index 00000000..876b9512
--- /dev/null
+++ b/plugins/code-modernization/hooks/fleet/fleet.ts
@@ -0,0 +1,350 @@
+/**
+ * The fleet: every agent loop this session has seen a tool call from, kept
+ * from the events themselves. A workflow's agents carry ids no `$.agent.list()`
+ * names, so the registry is built from `tool.call` and `turn.complete` alone.
+ *
+ * It also watches for one failure repeating across agents: three agents
+ * hitting the same error is a fact about the playbook, not about the agents.
+ */
+
+export type AgentRow = {
+ id: string
+ firstMs: number
+ lastMs: number
+ calls: number
+ errors: number
+ /** `Read`, `Bash`, and so on. */
+ lastTool: string
+ /** A short rendering of the last call's subject: a file's name, a command's head. */
+ lastSubject: string
+ isDone: boolean
+ /** The unit of work it appears to be on: the legacy module its calls touch most. */
+ unit?: string
+}
+
+export type Signature = {
+ /** The normalized failure text the agents share. */
+ text: string
+ agents: Set<string>
+ firstMs: number
+ lastMs: number
+ /** How many agents shared it when it was last called out; 0 while it has not been. */
+ announcedAt: number
+ /** When it was last called out. */
+ announcedMs: number
+ /** What a few of the failing calls were aimed at: the part of the cause the text alone leaves out. */
+ examples: string[]
+}
+
+export type Fleet = {
+ agents: Map<string, AgentRow>
+ signatures: Map<string, Signature>
+ /** Units each agent touched, to name what it is working on. */
+ unitHits: Map<string, Map<string, number>>
+ /** Counted as they happen, so pruning old rows does not shrink the totals. */
+ seen: number
+ ended: number
+ calls: number
+ errors: number
+}
+
+/** An agent with no call for this long reads as finished or stalled, in milliseconds. */
+export const IDLE_MS = 90_000
+
+/** How many agents must share a failure before it counts as shared at all. */
+export const SIGNATURE_THRESHOLD = 3
+
+/** A shared failure is called out again once it has spread this many times further. */
+export const ESCALATE_FACTOR = 5
+
+/** And no sooner than this after the last time, in milliseconds: a fan-out fails all at once. */
+export const ESCALATE_QUIET_MS = 60_000
+
+/**
+ * How many agents must share a failure before it is called out in the
+ * transcript. Three of five agents is a pattern; three of five hundred is not,
+ * so the bar rises with the fleet. The pane lists it from three either way.
+ */
+export const announceThresholdOf = (fleet: Fleet): number =>
+ Math.max(SIGNATURE_THRESHOLD, Math.ceil(fleet.seen * 0.05))
+
+/** Failures further apart than this are not the same incident, in milliseconds. */
+export const SIGNATURE_WINDOW_MS = 15 * 60_000
+
+export const newFleet = (): Fleet => ({
+ agents: new Map(),
+ signatures: new Map(),
+ unitHits: new Map(),
+ seen: 0,
+ ended: 0,
+ calls: 0,
+ errors: 0,
+})
+
+/** A short subject for a call, from the arguments tools commonly carry. */
+export function subjectOf(tool: string, args: Readonly<Record<string, unknown>>): string {
+ const str = (key: string) => (typeof args[key] === 'string' ? (args[key] as string) : undefined)
+ const path = str('file_path') ?? str('path') ?? str('notebook_path')
+
+ if (path !== undefined) {
+ return path.split('/').slice(-1)[0] ?? path
+ }
+
+ const command = str('command')
+
+ if (command !== undefined) {
+ // What runs, not where: a leading `cd …`, `VAR=…;` or `export …` says nothing at a glance.
+ const lead = /^(?:\s*(?:cd\s+(?:"[^"]*"|'[^']*'|\S+)(?:\s+\d?>{1,2}\s*\S+)*|(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|\S*))\s*(?:&&|;)\s*)+/
+
+ return command.replace(/\s+/g, ' ').replace(lead, '').slice(0, 48)
+ }
+
+ return (str('pattern') ?? str('description') ?? str('query') ?? str('prompt') ?? '')
+ .replace(/\s+/g, ' ')
+ .slice(0, 48)
+}
+
+/** Records the start of a call by `agentId`. */
+export function noteCall(
+ fleet: Fleet,
+ agentId: string,
+ tool: string,
+ subject: string,
+ nowMs: number,
+ unit?: string,
+): AgentRow {
+ if (!fleet.agents.has(agentId)) {
+ fleet.seen += 1
+ }
+
+ fleet.calls += 1
+
+ const row = fleet.agents.get(agentId) ?? {
+ id: agentId,
+ firstMs: nowMs,
+ lastMs: nowMs,
+ calls: 0,
+ errors: 0,
+ lastTool: tool,
+ lastSubject: subject,
+ isDone: false,
+ }
+
+ const next: AgentRow = {
+ ...row,
+ lastMs: nowMs,
+ calls: row.calls + 1,
+ lastTool: tool,
+ lastSubject: subject,
+ isDone: false,
+ }
+
+ if (unit !== undefined) {
+ const hits = fleet.unitHits.get(agentId) ?? new Map<string, number>()
+
+ hits.set(unit, (hits.get(unit) ?? 0) + 1)
+ fleet.unitHits.set(agentId, hits)
+
+ const top = [...hits.entries()].sort((a, b) => b[1] - a[1])[0]
+
+ if (top !== undefined) {
+ next.unit = top[0]
+ }
+ }
+
+ fleet.agents.set(agentId, next)
+
+ return next
+}
+
+const ERRORISH = /error|fail|exception|cannot|not found|denied|refused|blocked|unresolved|undefined|no such|does not exist|does not match/i
+
+/** A path named by a "No such file" line, as the command printed it. */
+export function missingPathOf(text: string): string | undefined {
+ return /([^\s:'"]*\/[^\s:'"]+): (?:open: )?No such file or directory/.exec(text)?.[1]
+}
+
+/**
+ * A failure text with what varies between agents taken out: paths, numbers,
+ * hex, quoted values, timestamps. Two agents hitting one cause then match.
+ *
+ * Empty when the text names no cause worth matching on: a bare exit code (a
+ * search that found nothing), or a malformed tool call, which is the model's
+ * slip and not something the agents share.
+ */
+export function normalizeFailure(text: string): string {
+ if (/InputValidationError|could not be parsed as JSON/.test(text)) {
+ return ''
+ }
+
+ const line =
+ text
+ .split('\n')
+ .map(part => part.trim().replace(/^Exit code \d+\s*/i, ''))
+ .find(part => part !== '' && ERRORISH.test(part)) ?? ''
+
+ if (line === '') {
+ return ''
+ }
+
+ // Families: one cause that words itself differently per command or per call.
+ if (/file does not exist|no such file or directory|path does not exist/i.test(line)) {
+ return 'a path that does not exist'
+ }
+
+ const denied = /^Permission to use (\w+) with command\s+(?:cd\s+\S+\s*(?:&&|;)\s*)?([\w./-]+)[\s\S]*has been denied/.exec(line)
+
+ if (denied !== null) {
+ return `Permission to use ${denied[1]} with command ${denied[2]} … was denied`
+ }
+
+ const blocked = /^([\w./-]+) to .+ was blocked by a deny rule/.exec(line)
+
+ if (blocked !== null) {
+ return `${blocked[1]} was blocked by a deny rule`
+ }
+
+ if (/does not match required schema/i.test(line)) {
+ // Which property broke which constraint, once each: `/rules/13/category` and `/rules/27/category` are one fact.
+ const clauses = new Set<string>()
+
+ for (const match of line.matchAll(/((?:\/[\w-]+)+): (must [a-z ]+?)(?=:|,|$)/g)) {
+ clauses.add(`${(match[1] ?? '').replace(/\/\d+(?=\/|$)/g, '/<n>')}: ${(match[2] ?? '').trim()}`)
+ }
+
+ return `Output does not match required schema: ${[...clauses].join('; ')}`.slice(0, 140)
+ }
+
+ if (/\(eval\):\d+: =+\S* not found/.test(line)) {
+ return '(eval):<n>: ==… not found'
+ }
+
+ const key = line
+ .replace(/\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?Z?\b/g, '<time>')
+ .replace(/(?:[A-Za-z]:)?(?:\/[\w.@~+-]+){2,}\/?/g, '<path>')
+ .replace(/\b0x[0-9a-fA-F]+\b/g, '<hex>')
+ .replace(/\b[0-9a-f]{12,}\b/g, '<id>')
+ .replace(/(["'`]).{1,60}?\1/g, '<q>')
+ .replace(/\b\d+(?:\.\d+)?\b/g, '<n>')
+ .replace(/\s+/g, ' ')
+ .trim()
+ .slice(0, 140)
+
+ // What is left must still say something: `<n> <q>` and the like match everything.
+ return key.replace(/<\w+>|[^A-Za-z]+/g, ' ').trim().split(/\s+/).filter(word => word.length > 2).length >= 2 ? key : ''
+}
+
+/**
+ * Records a failed call. Returns the signature when this failure is one to
+ * call out now: the first time enough agents share it, and again each time it
+ * has spread `ESCALATE_FACTOR` times further. `example` is what the failing
+ * call was aimed at (a path, a command's head).
+ */
+export function noteFailure(
+ fleet: Fleet,
+ agentId: string,
+ errorText: string,
+ nowMs: number,
+ example?: string,
+): Signature | null {
+ const row = fleet.agents.get(agentId)
+
+ fleet.errors += 1
+
+ if (row !== undefined) {
+ fleet.agents.set(agentId, { ...row, errors: row.errors + 1 })
+ }
+
+ const key = normalizeFailure(errorText)
+
+ if (key.length < 12) {
+ return null
+ }
+
+ const held = fleet.signatures.get(key)
+
+ const signature: Signature =
+ held !== undefined && nowMs - held.lastMs <= SIGNATURE_WINDOW_MS
+ ? held
+ : { text: key, agents: new Set(), firstMs: nowMs, lastMs: nowMs, announcedAt: 0, announcedMs: 0, examples: [] }
+
+ signature.agents.add(agentId)
+ signature.lastMs = nowMs
+
+ if (example !== undefined && example !== '' && signature.examples.length < 3 && !signature.examples.includes(example)) {
+ signature.examples.push(example)
+ }
+
+ fleet.signatures.set(key, signature)
+
+ const size = signature.agents.size
+
+ const isDue =
+ signature.announcedAt === 0
+ ? size >= announceThresholdOf(fleet)
+ : size >= signature.announcedAt * ESCALATE_FACTOR && nowMs - signature.announcedMs >= ESCALATE_QUIET_MS
+
+ if (isDue) {
+ signature.announcedAt = size
+ signature.announcedMs = nowMs
+
+ return signature
+ }
+
+ return null
+}
+
+/** A shared failure in one line, with what the failing calls were aimed at. */
+export function lineOf(signature: Signature, form: 'full' | 'short' = 'full'): string {
+ const examples = signature.examples.slice(0, form === 'short' ? 1 : 2).join(', ')
+ const lead = form === 'short' ? `${signature.agents.size} agents: ` : `${signature.agents.size} agents hit the same failure: `
+
+ return `${lead}${signature.text}${examples !== '' ? ` (e.g. ${examples})` : ''}`
+}
+
+/** Marks an agent's loop as ended (its `turn.complete`). */
+export function noteDone(fleet: Fleet, agentId: string, nowMs: number): void {
+ const row = fleet.agents.get(agentId)
+
+ if (row !== undefined) {
+ if (!row.isDone) {
+ fleet.ended += 1
+ }
+
+ fleet.agents.set(agentId, { ...row, isDone: true, lastMs: nowMs })
+ }
+}
+
+/** The fleet in numbers, as the pane's header line shows it. */
+export function tallyOf(fleet: Fleet, nowMs: number) {
+ const rows = [...fleet.agents.values()]
+ const active = rows.filter(row => !row.isDone && nowMs - row.lastMs < IDLE_MS)
+
+ return {
+ total: fleet.seen,
+ active: active.length,
+ done: fleet.ended,
+ stalled: rows.filter(row => !row.isDone && nowMs - row.lastMs >= IDLE_MS).length,
+ calls: fleet.calls,
+ errors: fleet.errors,
+ recent: active.sort((a, b) => b.lastMs - a.lastMs),
+ // The widest-spread first: the pane has room for two or three.
+ shared: [...fleet.signatures.values()]
+ .filter(signature => signature.agents.size >= SIGNATURE_THRESHOLD)
+ .sort((a, b) => b.agents.size - a.agents.size),
+ }
+}
+
+/** Forgets agents that ended long ago, keeping the registry bounded on a long run. */
+export function prune(fleet: Fleet, nowMs: number, keepMs = 30 * 60_000, max = 1500): void {
+ if (fleet.agents.size <= max) {
+ return
+ }
+
+ for (const [id, row] of fleet.agents) {
+ if ((row.isDone || nowMs - row.lastMs > keepMs) && fleet.agents.size > max) {
+ fleet.agents.delete(id)
+ fleet.unitHits.delete(id)
+ }
+ }
+}