1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
|
/**
* The fleet: every agent loop this session has seen a tool call from, kept
* from the events themselves. A workflow's agents carry ids no `$.agent.list()`
* names, so the registry is built from `tool.call` and `turn.complete` alone.
*
* It also watches for one failure repeating across agents: three agents
* hitting the same error is a fact about the playbook, not about the agents.
*/
export type AgentRow = {
id: string
firstMs: number
lastMs: number
calls: number
errors: number
/** `Read`, `Bash`, and so on. */
lastTool: string
/** A short rendering of the last call's subject: a file's name, a command's head. */
lastSubject: string
isDone: boolean
/** The unit of work it appears to be on: the legacy module its calls touch most. */
unit?: string
}
export type Signature = {
/** The normalized failure text the agents share. */
text: string
agents: Set<string>
firstMs: number
lastMs: number
/** How many agents shared it when it was last called out; 0 while it has not been. */
announcedAt: number
/** When it was last called out. */
announcedMs: number
/** What a few of the failing calls were aimed at: the part of the cause the text alone leaves out. */
examples: string[]
}
export type Fleet = {
agents: Map<string, AgentRow>
signatures: Map<string, Signature>
/** Units each agent touched, to name what it is working on. */
unitHits: Map<string, Map<string, number>>
/** Counted as they happen, so pruning old rows does not shrink the totals. */
seen: number
ended: number
calls: number
errors: number
}
/** An agent with no call for this long reads as finished or stalled, in milliseconds. */
export const IDLE_MS = 90_000
/** How many agents must share a failure before it counts as shared at all. */
export const SIGNATURE_THRESHOLD = 3
/** A shared failure is called out again once it has spread this many times further. */
export const ESCALATE_FACTOR = 5
/** And no sooner than this after the last time, in milliseconds: a fan-out fails all at once. */
export const ESCALATE_QUIET_MS = 60_000
/**
* How many agents must share a failure before it is called out in the
* transcript. Three of five agents is a pattern; three of five hundred is not,
* so the bar rises with the fleet. The pane lists it from three either way.
*/
export const announceThresholdOf = (fleet: Fleet): number =>
Math.max(SIGNATURE_THRESHOLD, Math.ceil(fleet.seen * 0.05))
/** Failures further apart than this are not the same incident, in milliseconds. */
export const SIGNATURE_WINDOW_MS = 15 * 60_000
export const newFleet = (): Fleet => ({
agents: new Map(),
signatures: new Map(),
unitHits: new Map(),
seen: 0,
ended: 0,
calls: 0,
errors: 0,
})
/** A short subject for a call, from the arguments tools commonly carry. */
export function subjectOf(tool: string, args: Readonly<Record<string, unknown>>): string {
const str = (key: string) => (typeof args[key] === 'string' ? (args[key] as string) : undefined)
const path = str('file_path') ?? str('path') ?? str('notebook_path')
if (path !== undefined) {
return path.split('/').slice(-1)[0] ?? path
}
const command = str('command')
if (command !== undefined) {
// What runs, not where: a leading `cd …`, `VAR=…;` or `export …` says nothing at a glance.
const lead = /^(?:\s*(?:cd\s+(?:"[^"]*"|'[^']*'|\S+)(?:\s+\d?>{1,2}\s*\S+)*|(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*=(?:"[^"]*"|'[^']*'|\S*))\s*(?:&&|;)\s*)+/
return command.replace(/\s+/g, ' ').replace(lead, '').slice(0, 48)
}
return (str('pattern') ?? str('description') ?? str('query') ?? str('prompt') ?? '')
.replace(/\s+/g, ' ')
.slice(0, 48)
}
/** Records the start of a call by `agentId`. */
export function noteCall(
fleet: Fleet,
agentId: string,
tool: string,
subject: string,
nowMs: number,
unit?: string,
): AgentRow {
if (!fleet.agents.has(agentId)) {
fleet.seen += 1
}
fleet.calls += 1
const row = fleet.agents.get(agentId) ?? {
id: agentId,
firstMs: nowMs,
lastMs: nowMs,
calls: 0,
errors: 0,
lastTool: tool,
lastSubject: subject,
isDone: false,
}
const next: AgentRow = {
...row,
lastMs: nowMs,
calls: row.calls + 1,
lastTool: tool,
lastSubject: subject,
isDone: false,
}
if (unit !== undefined) {
const hits = fleet.unitHits.get(agentId) ?? new Map<string, number>()
hits.set(unit, (hits.get(unit) ?? 0) + 1)
fleet.unitHits.set(agentId, hits)
const top = [...hits.entries()].sort((a, b) => b[1] - a[1])[0]
if (top !== undefined) {
next.unit = top[0]
}
}
fleet.agents.set(agentId, next)
return next
}
const ERRORISH = /error|fail|exception|cannot|not found|denied|refused|blocked|unresolved|undefined|no such|does not exist|does not match/i
/** A path named by a "No such file" line, as the command printed it. */
export function missingPathOf(text: string): string | undefined {
return /([^\s:'"]*\/[^\s:'"]+): (?:open: )?No such file or directory/.exec(text)?.[1]
}
/**
* A failure text with what varies between agents taken out: paths, numbers,
* hex, quoted values, timestamps. Two agents hitting one cause then match.
*
* Empty when the text names no cause worth matching on: a bare exit code (a
* search that found nothing), or a malformed tool call, which is the model's
* slip and not something the agents share.
*/
export function normalizeFailure(text: string): string {
if (/InputValidationError|could not be parsed as JSON/.test(text)) {
return ''
}
const line =
text
.split('\n')
.map(part => part.trim().replace(/^Exit code \d+\s*/i, ''))
.find(part => part !== '' && ERRORISH.test(part)) ?? ''
if (line === '') {
return ''
}
// Families: one cause that words itself differently per command or per call.
if (/file does not exist|no such file or directory|path does not exist/i.test(line)) {
return 'a path that does not exist'
}
const denied = /^Permission to use (\w+) with command\s+(?:cd\s+\S+\s*(?:&&|;)\s*)?([\w./-]+)[\s\S]*has been denied/.exec(line)
if (denied !== null) {
return `Permission to use ${denied[1]} with command ${denied[2]} … was denied`
}
const blocked = /^([\w./-]+) to .+ was blocked by a deny rule/.exec(line)
if (blocked !== null) {
return `${blocked[1]} was blocked by a deny rule`
}
if (/does not match required schema/i.test(line)) {
// Which property broke which constraint, once each: `/rules/13/category` and `/rules/27/category` are one fact.
const clauses = new Set<string>()
for (const match of line.matchAll(/((?:\/[\w-]+)+): (must [a-z ]+?)(?=:|,|$)/g)) {
clauses.add(`${(match[1] ?? '').replace(/\/\d+(?=\/|$)/g, '/<n>')}: ${(match[2] ?? '').trim()}`)
}
return `Output does not match required schema: ${[...clauses].join('; ')}`.slice(0, 140)
}
if (/\(eval\):\d+: =+\S* not found/.test(line)) {
return '(eval):<n>: ==… not found'
}
const key = line
.replace(/\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?Z?\b/g, '<time>')
.replace(/(?:[A-Za-z]:)?(?:\/[\w.@~+-]+){2,}\/?/g, '<path>')
.replace(/\b0x[0-9a-fA-F]+\b/g, '<hex>')
.replace(/\b[0-9a-f]{12,}\b/g, '<id>')
.replace(/(["'`]).{1,60}?\1/g, '<q>')
.replace(/\b\d+(?:\.\d+)?\b/g, '<n>')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 140)
// What is left must still say something: `<n> <q>` and the like match everything.
return key.replace(/<\w+>|[^A-Za-z]+/g, ' ').trim().split(/\s+/).filter(word => word.length > 2).length >= 2 ? key : ''
}
/**
* Records a failed call. Returns the signature when this failure is one to
* call out now: the first time enough agents share it, and again each time it
* has spread `ESCALATE_FACTOR` times further. `example` is what the failing
* call was aimed at (a path, a command's head).
*/
export function noteFailure(
fleet: Fleet,
agentId: string,
errorText: string,
nowMs: number,
example?: string,
): Signature | null {
const row = fleet.agents.get(agentId)
fleet.errors += 1
if (row !== undefined) {
fleet.agents.set(agentId, { ...row, errors: row.errors + 1 })
}
const key = normalizeFailure(errorText)
if (key.length < 12) {
return null
}
const held = fleet.signatures.get(key)
const signature: Signature =
held !== undefined && nowMs - held.lastMs <= SIGNATURE_WINDOW_MS
? held
: { text: key, agents: new Set(), firstMs: nowMs, lastMs: nowMs, announcedAt: 0, announcedMs: 0, examples: [] }
signature.agents.add(agentId)
signature.lastMs = nowMs
if (example !== undefined && example !== '' && signature.examples.length < 3 && !signature.examples.includes(example)) {
signature.examples.push(example)
}
fleet.signatures.set(key, signature)
const size = signature.agents.size
const isDue =
signature.announcedAt === 0
? size >= announceThresholdOf(fleet)
: size >= signature.announcedAt * ESCALATE_FACTOR && nowMs - signature.announcedMs >= ESCALATE_QUIET_MS
if (isDue) {
signature.announcedAt = size
signature.announcedMs = nowMs
return signature
}
return null
}
/** A shared failure in one line, with what the failing calls were aimed at. */
export function lineOf(signature: Signature, form: 'full' | 'short' = 'full'): string {
const examples = signature.examples.slice(0, form === 'short' ? 1 : 2).join(', ')
const lead = form === 'short' ? `${signature.agents.size} agents: ` : `${signature.agents.size} agents hit the same failure: `
return `${lead}${signature.text}${examples !== '' ? ` (e.g. ${examples})` : ''}`
}
/** Marks an agent's loop as ended (its `turn.complete`). */
export function noteDone(fleet: Fleet, agentId: string, nowMs: number): void {
const row = fleet.agents.get(agentId)
if (row !== undefined) {
if (!row.isDone) {
fleet.ended += 1
}
fleet.agents.set(agentId, { ...row, isDone: true, lastMs: nowMs })
}
}
/** The fleet in numbers, as the pane's header line shows it. */
export function tallyOf(fleet: Fleet, nowMs: number) {
const rows = [...fleet.agents.values()]
const active = rows.filter(row => !row.isDone && nowMs - row.lastMs < IDLE_MS)
return {
total: fleet.seen,
active: active.length,
done: fleet.ended,
stalled: rows.filter(row => !row.isDone && nowMs - row.lastMs >= IDLE_MS).length,
calls: fleet.calls,
errors: fleet.errors,
recent: active.sort((a, b) => b.lastMs - a.lastMs),
// The widest-spread first: the pane has room for two or three.
shared: [...fleet.signatures.values()]
.filter(signature => signature.agents.size >= SIGNATURE_THRESHOLD)
.sort((a, b) => b.agents.size - a.agents.size),
}
}
/** Forgets agents that ended long ago, keeping the registry bounded on a long run. */
export function prune(fleet: Fleet, nowMs: number, keepMs = 30 * 60_000, max = 1500): void {
if (fleet.agents.size <= max) {
return
}
for (const [id, row] of fleet.agents) {
if ((row.isDone || nowMs - row.lastMs > keepMs) && fleet.agents.size > max) {
fleet.agents.delete(id)
fleet.unitHits.delete(id)
}
}
}
|