diff options
| author | nicdillon <nicholas.dillon@vercel.com> | 2026-10-28 11:26:14 -0400 |
|---|---|---|
| committer | nicdillon <nicholas.dillon@vercel.com> | 2026-10-28 11:26:14 -0400 |
| commit | 097f4c396d4d21ab26fef357a8bb07b3b084529a (patch) | |
| tree | 04d6d4f821c5fc0f45d2bb97fd540b0fc476de21 | |
| parent | 1a2c79d4e6c4a2283dc2c614b6538cbe1ed5ca62 (diff) | |
vercel-optimize: add build_machine_oversized gate and build signals collectorvercel-optimize/build-machine-oversized
- lib/build-signals.mjs: collect 14 days of deployments (v7 list) plus, for up
to 40 recent builds (v13 detail), duration.timeForBilling,
duration.cpuTimeForBilling, resourceConfig.buildMachine (purchaseType,
cores, memory, machineSelectionType), buildSkipped, and whether the build
came from a Deploy Hook (meta.deployHookName presence). meta.deployHookId is
the secret end of the hook URL and is never read or stored. Platform data
only; no Observability Plus required.
- lib/gates/build-machine-oversized.mjs: account-scoped gate for projects
pinned to Enhanced or Turbo. Recommends Elastic; reports the break-even
billed duration on the next smaller machine as context, not a savings
figure. Median uses successful builds only; skipped, Elastic-sized, and
other-machine builds are excluded from the build count; CPU minutes come
from cpuTimeForBilling when present; memory-related Elastic resize reasons
suppress the gate.
- Register the kind in gates/index (GATE_VERSION 1.9.0), deep-dive, display
labels, report category, and impact label.
- Citations for Managing Builds and build pricing; build-machine-right-sizing
support topic; signals schema 1.3 with the new builds field.
- SKILL.md and references/observability-plus.md: build-focused requests
continue without Observability Plus, since build gates use no route metrics.
- lib/grade-recommendation.mjs: grade `<account>` candidates on the account
rubric too (keeping the higher of route and account scores), and count
builds, vCPUs, and billed/CPU minutes as evidence. Without this the
build_machine_oversized rec was held back at quality 0.17.
- SKILL.md: the post-blocker rerun now merges with --force; without it the
merge failed and gates ran on the blocked collection (no usage or builds).
- impact-label: when a written build recommendation omits the gate o11ySignal,
re-derive it from signals.json so the report shows the real impact line
instead of a placeholder. A scanner-only build_minutes_fanout no longer claims a
billing threshold was exceeded.
- collect-signals: redact Deploy Hook ids and URLs from the stored project
config (link.deployHooks keeps name, ref, createdAt). They are bearer
secrets and were previously written to signals.json on every run.
- scan-codebase: skip .claude and .agents, where project-installed agent skills
live; scanning them flagged this skill's own source as findings.
- Tests: 36 new fixture tests (gate, collector, grading, impact label, redaction, scan); full
suite 805/805.
Validated on a live test project: moving a pinned-Turbo project to Elastic
cut CPU minutes 54% over five builds, and the gate stopped firing.
20 files changed, 886 insertions, 20 deletions
diff --git a/packages/vercel-optimize-tests/test/build-machine-grading.test.mjs b/packages/vercel-optimize-tests/test/build-machine-grading.test.mjs new file mode 100644 index 0000000..4da4116 --- /dev/null +++ b/packages/vercel-optimize-tests/test/build-machine-grading.test.mjs @@ -0,0 +1,84 @@ +// Regression: end-to-end run on the build fixture held back the build_machine_oversized rec at +// quality 0.17 because its `build_machine_oversized:<account>` candidateRef was graded as route scope +// (expecting file:line). Account-sentinel refs must be graded on the account rubric. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { gradeRecommendation, applyQualityFloor } from '../../../skills/vercel-optimize/lib/grade-recommendation.mjs'; + +const machineRec = { + what: 'Switch web from a fixed Turbo build machine to Elastic.', + why: 'web is pinned to Turbo (30 vCPUs). Its 9 builds in the last 14 days bill a median 2 billed minutes, about 60 CPU minutes each. Enhanced alone costs less unless builds take longer than 7 billed minutes.', + fix: 'Set the build machine to Elastic under Settings → Build and Deployment → Build Machines.', + verify: 'Compare CPU minutes per deployment over the next 5 builds.', + bucket: 'cost', + effort: 'low', + affectedFiles: [], + candidateRef: 'build_machine_oversized:<account>', + o11ySignal: 'machine=turbo builds=9 median_billed_min=2 fix=elastic smaller=enhanced break_even_min=7', + citations: ['https://vercel.com/docs/builds/managing-builds', 'https://vercel.com/docs/pricing#builds'], +}; + +test('grading: build_machine_oversized rec with an <account> ref clears the quality floor', () => { + const q = gradeRecommendation(machineRec); + assert.equal(q.scope, 'account'); + assert.ok(q.overall >= 0.55, `expected >= 0.55, got ${q.overall} (${JSON.stringify(q)})`); + const { kept, dropped } = applyQualityFloor([{ ...machineRec, quality: q }]); + assert.equal(kept.length, 1); + assert.equal(dropped.length, 0); +}); + +test('grading: a vague <account> rec still falls below the floor', () => { + const q = gradeRecommendation({ ...machineRec, why: 'It is expensive.', fix: 'Use Elastic.', verify: undefined, citations: [], o11ySignal: undefined }); + assert.ok(q.overall < 0.55, `vague rec should stay below the floor (got ${q.overall})`); +}); + +test('grading: an <account> rec with file:line evidence keeps its route-scope score when that is higher', () => { + const fanoutRec = { + what: 'Turn the Turborepo build cache back on for web.', + why: 'Builds are 100% of the bill and turbo.json:8 sets "cache": false on the build task.', + fix: 'Remove `"cache": false` from tasks.build in turbo.json:8 and enable skip-unaffected projects.', + bucket: 'cost', + affectedFiles: ['turbo.json'], + candidateRef: 'build_minutes_fanout:<account>', + o11ySignal: 'build_minutes_share=100% scanner_findings=1', + citations: ['https://vercel.com/docs/builds'], + }; + const both = gradeRecommendation(fanoutRec); + const routeOnly = gradeRecommendation({ ...fanoutRec, candidateRef: 'build_minutes_fanout:turbo.json' }); + assert.ok(both.overall >= routeOnly.overall, `sentinel grading must not lower the score (${both.overall} < ${routeOnly.overall})`); +}); + +test('grading: build units count as evidence', () => { + const withUnits = gradeRecommendation(machineRec); + const withoutUnits = gradeRecommendation({ ...machineRec, why: 'web is pinned to Turbo and builds are short.' }); + assert.ok(withUnits.evidence > withoutUnits.evidence); +}); + +import { synthesizeImpactFromSignal } from '../../../skills/vercel-optimize/lib/impact-label.mjs'; + +test('impact label: scanner-only fanout does not claim a billing threshold', () => { + const scannerOnly = synthesizeImpactFromSignal({ kind: 'build_minutes_fanout', candidateRef: 'build_minutes_fanout:<account>', o11ySignal: 'build_minutes_share=0% scanner_findings=1', impactTier: 'High' }); + assert.match(scannerOnly, /bypasses the Turborepo build cache/); + assert.doesNotMatch(scannerOnly, /exceeded the gate threshold/); + const measured = synthesizeImpactFromSignal({ kind: 'build_minutes_fanout', candidateRef: 'build_minutes_fanout:<account>', o11ySignal: 'build_minutes_share=100% scanner_findings=1', impactTier: 'High' }); + assert.match(measured, /100% of observed billed cost/); +}); + +test('impact label: re-derives build gate signals when the written rec omits o11ySignal', () => { + const MIN = 60_000; + const signals = { + plan: { plan: 'enterprise' }, + project: { id: 'prj_1', resourceConfig: { buildMachineType: 'turbo', buildMachineSelection: 'fixed' } }, + builds: { ok: true, windowDays: 14, deployments: Array.from({ length: 9 }, (_, i) => ({ id: `d${i}`, createdAt: i, state: 'READY', timeForBillingMs: 2 * MIN, cpuTimeForBillingMs: 60 * MIN, buildMachine: 'turbo', machineSelection: 'fixed', buildSkipped: false })) }, + usage: { services: [{ name: 'Build CPU Minutes', billedCost: 9.99 }, { name: 'Fast Data Transfer', billedCost: 0.01 }] }, + codebase: { findings: [] }, + }; + const machine = synthesizeImpactFromSignal({ candidateRef: 'build_machine_oversized:<account>', impactTier: 'High', why: 'Switch to Elastic.' }, signals); + assert.match(machine, /pinned Turbo machine bill a median 2 minutes/); + assert.match(machine, /longer than 7 billed minutes/); + const fanout = synthesizeImpactFromSignal({ candidateRef: 'build_minutes_fanout:<account>', impactTier: 'High', why: 'Re-enable caching.' }, signals); + assert.match(fanout, /100% of observed billed cost/); + // An explicit o11ySignal still wins over the fallback. + const explicit = synthesizeImpactFromSignal({ candidateRef: 'build_machine_oversized:<account>', impactTier: 'High', o11ySignal: 'machine=enhanced builds=9 median_billed_min=3 fix=elastic smaller=standard break_even_min=6' }, signals); + assert.match(explicit, /pinned Enhanced machine bill a median 3 minutes/); +}); diff --git a/packages/vercel-optimize-tests/test/build-machine-oversized.test.mjs b/packages/vercel-optimize-tests/test/build-machine-oversized.test.mjs new file mode 100644 index 0000000..a425500 --- /dev/null +++ b/packages/vercel-optimize-tests/test/build-machine-oversized.test.mjs @@ -0,0 +1,200 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { gate, metadata } from '../../../skills/vercel-optimize/lib/gates/build-machine-oversized.mjs'; +import { gates } from '../../../skills/vercel-optimize/lib/gates/index.mjs'; +import { validateCandidates } from '../../../skills/vercel-optimize/lib/gates/contract.mjs'; +import { synthesizeImpactFromSignal } from '../../../skills/vercel-optimize/lib/impact-label.mjs'; + +const MIN = 60_000; + +function builds(count, { billedMinutes = 3, machine = null, state = 'READY', skipped = false } = {}) { + return Array.from({ length: count }, (_, i) => ({ + id: `dpl_${String(i).padStart(3, '0')}`, + createdAt: 1_700_000_000_000 - i * 1000, + state, + target: i % 2 ? 'production' : null, + source: 'git', + branch: 'main', + timeForBillingMs: billedMinutes * MIN, + buildMachine: machine, + buildSkipped: skipped, + })); +} + +function signals({ machine = 'turbo', selection = 'fixed', deployments = builds(10), plan = 'pro', project } = {}) { + return { + plan: { plan }, + project: project ?? { id: 'prj_test', resourceConfig: { buildMachineType: machine, buildMachineSelection: selection } }, + builds: { ok: true, windowDays: 14, deployments }, + }; +} + +test('build_machine_oversized: metadata sane and registered', () => { + assert.equal(metadata.id, 'build_machine_oversized'); + assert.equal(metadata.scope, 'account'); + assert.equal(metadata.billingDimension, 'build'); + assert.ok(gates.some((g) => g.metadata.id === 'build_machine_oversized')); +}); + +test('build_machine_oversized: fires for fixed Turbo with break-even on Enhanced', () => { + const out = gate(signals()); + assert.equal(out.length, 1); + const [c] = out; + assert.equal(c.kind, 'build_machine_oversized'); + assert.equal(c.scope, 'account'); + assert.equal(c.evidence.smallerMachine, 'enhanced'); + assert.equal(c.evidence.medianBilledMinutes, 3); + // 3 billed minutes x 30 vCPUs / 8 vCPUs = 11.25 -> 11 whole minutes + assert.equal(c.evidence.breakEvenBilledMinutesOnSmaller, 11); + assert.equal(c.evidence.windowCpuMinutesEstimate, 900); + assert.equal(c.priority, 45); + assert.equal(c.confidence, 0.8); + assert.match(c.o11ySignal, /machine=turbo builds=10 median_billed_min=3 fix=elastic smaller=enhanced break_even_min=11/); + assert.equal(c.evidence.cpuMinutesSource, 'billed_x_vcpu'); + assert.doesNotThrow(() => validateCandidates(out)); +}); + +test('build_machine_oversized: Enhanced compares against Standard with a 2x ratio', () => { + const [c] = gate(signals({ machine: 'enhanced', deployments: builds(20, { billedMinutes: 3 }) })); + assert.equal(c.evidence.smallerMachine, 'standard'); + assert.equal(c.evidence.vcpuRatio, 2); + assert.equal(c.evidence.breakEvenBilledMinutesOnSmaller, 6); + assert.equal(c.confidence, 0.55); + assert.equal(c.evidence.smallerMachineMemoryGb, 8); +}); + +test('build_machine_oversized: scales the estimate to all built deployments when detail is sampled', () => { + const sampled = builds(10); + const listedOnly = builds(40).map((d, i) => ({ ...d, id: `dpl_l${i}`, timeForBillingMs: null })); + const [c] = gate(signals({ deployments: [...sampled, ...listedOnly] })); + assert.equal(c.evidence.builtCount, 50); + assert.equal(c.evidence.sampledBuilds, 10); + assert.equal(c.evidence.windowCpuMinutesEstimate, 4500); + assert.equal(c.evidence.detailSampleTruncated, true); +}); + +test('build_machine_oversized: raises priority at high CPU-minute volume', () => { + const [c] = gate(signals({ deployments: builds(60, { billedMinutes: 3 }) })); + assert.equal(c.evidence.windowCpuMinutesEstimate, 5400); + assert.equal(c.priority, 60); +}); + +test('build_machine_oversized: lowers confidence when selection mode is unknown', () => { + const [c] = gate(signals({ project: { id: 'prj_test', resourceConfig: { buildMachineType: 'turbo' } } })); + assert.equal(c.evidence.machineSelection, null); + assert.equal(c.confidence, 0.7); +}); + +test('build_machine_oversized: mentions contract machines on Enterprise', () => { + const [c] = gate(signals({ plan: 'enterprise' })); + assert.match(c.question, /contract/); + const [pro] = gate(signals({ plan: 'pro' })); + assert.doesNotMatch(pro.question, /contract/); +}); + +test('build_machine_oversized: ignores skipped builds and builds on another machine', () => { + const deployments = [ + ...builds(4), + ...builds(5, { skipped: true }).map((d, i) => ({ ...d, id: `dpl_s${i}` })), + ...builds(5, { machine: 'enhanced' }).map((d, i) => ({ ...d, id: `dpl_e${i}` })), + ]; + assert.deepEqual(gate(signals({ deployments })), []); +}); + +test('build_machine_oversized: excludes skipped and other-machine builds from the build count', () => { + const deployments = [ + ...builds(6), + ...builds(5, { skipped: true }).map((d, i) => ({ ...d, id: `dpl_s${i}` })), + ...builds(5, { machine: 'enhanced' }).map((d, i) => ({ ...d, id: `dpl_e${i}` })), + ]; + const [c] = gate(signals({ deployments })); + assert.equal(c.evidence.builtCount, 6); + // 3 billed minutes x 30 vCPUs x 6 builds + assert.equal(c.evidence.windowCpuMinutesEstimate, 540); +}); + +test('build_machine_oversized: leaves failed builds out of the median but counts them', () => { + const deployments = [ + ...builds(5, { billedMinutes: 3 }), + ...builds(4, { billedMinutes: 1, state: 'ERROR' }).map((d, i) => ({ ...d, id: `dpl_x${i}` })), + ]; + const [c] = gate(signals({ deployments })); + assert.equal(c.evidence.medianBilledMinutes, 3); + assert.equal(c.evidence.breakEvenBilledMinutesOnSmaller, 11); + assert.equal(c.evidence.builtCount, 9); + assert.equal(c.evidence.erroredBuildsExcludedFromMedian, 4); +}); + +test('build_machine_oversized: needs 5 successful builds, not 5 builds of any state', () => { + const deployments = [ + ...builds(4), + ...builds(6, { state: 'ERROR' }).map((d, i) => ({ ...d, id: `dpl_x${i}` })), + ]; + assert.deepEqual(gate(signals({ deployments })), []); +}); + +test('build_machine_oversized: silent when Elastic recorded a memory resize reason', () => { + for (const reason of ['oom-failure', 'high-peak-memory']) { + const project = { id: 'prj_test', resourceConfig: { buildMachineType: 'enhanced', buildMachineSelection: 'fixed', buildMachineElasticReason: reason } }; + assert.deepEqual(gate(signals({ project, deployments: builds(40) })), [], reason); + } +}); + +test('build_machine_oversized: steers toward Elastic because peak memory is not collected', () => { + const [c] = gate(signals({ machine: 'enhanced', deployments: builds(20) })); + assert.match(c.question, /recommend Elastic, not a fixed smaller machine/); + assert.match(c.question, /Standard has 8 GB/); + assert.equal(c.evidence.recommendedFix, 'elastic'); + assert.match(c.reason, /switch to Elastic/); + assert.equal(c.evidence.peakMemoryCollected, false); +}); + +test('build_machine_oversized: uses cpuTimeForBilling when every sampled build has it', () => { + const deployments = builds(10, { billedMinutes: 2, machine: 'turbo' }).map((d) => ({ ...d, cpuTimeForBillingMs: 60 * MIN })); + const [c] = gate(signals({ deployments })); + assert.equal(c.evidence.cpuMinutesSource, 'cpuTimeForBilling'); + assert.equal(c.evidence.medianCpuMinutes, 60); + assert.equal(c.evidence.windowCpuMinutesEstimate, 600); +}); + +test('build_machine_oversized: excludes builds Elastic sized after a switch, and counts hook builds', () => { + const pinned = builds(6, { machine: 'turbo' }).map((d, i) => ({ ...d, machineSelection: 'fixed', triggeredByDeployHook: i < 2 })); + const elasticEra = builds(5, { machine: 'turbo' }).map((d, i) => ({ ...d, id: `dpl_el${i}`, machineSelection: 'elastic' })); + const [c] = gate(signals({ deployments: [...pinned, ...elasticEra] })); + assert.equal(c.evidence.builtCount, 6); + assert.equal(c.evidence.hookTriggeredBuilds, 2); +}); + +test('build_machine_oversized: silent for Elastic, Standard, and Basic', () => { + assert.deepEqual(gate(signals({ selection: 'elastic' })), []); + assert.deepEqual(gate(signals({ machine: 'standard' })), []); + assert.deepEqual(gate(signals({ machine: 'basic' })), []); +}); + +test('build_machine_oversized: silent without project config or build signals', () => { + assert.deepEqual(gate({}), []); + assert.deepEqual(gate(signals({ project: { error: 'forbidden' } })), []); + assert.deepEqual(gate({ ...signals(), builds: { ok: false, code: 'FORBIDDEN' } }), []); + assert.deepEqual(gate({ ...signals(), builds: undefined }), []); +}); + +test('build_machine_oversized: silent below the sample and volume floors', () => { + assert.deepEqual(gate(signals({ deployments: builds(4) })), []); + // 5 builds x 1 billed minute x 8 vCPUs = 40 CPU minutes, under the 300 floor + assert.deepEqual(gate(signals({ machine: 'enhanced', deployments: builds(5, { billedMinutes: 1 }) })), []); +}); + +test('build_machine_oversized: deterministic', () => { + const s = signals(); + assert.deepEqual(gate(s), gate(s)); +}); + +test('build_machine_oversized: impact label states break-even, not savings', () => { + const [c] = gate(signals()); + const label = synthesizeImpactFromSignal({ kind: c.kind, o11ySignal: c.o11ySignal, impactTier: 'High' }); + assert.equal( + label, + 'High impact — builds on the pinned Turbo machine bill a median 3 minutes; switching to Elastic lets each build use a smaller machine when it fits (Enhanced alone costs less unless builds take longer than 11 billed minutes).', + ); + assert.doesNotMatch(label, /\$|sav/i); +}); diff --git a/packages/vercel-optimize-tests/test/build-signals.test.mjs b/packages/vercel-optimize-tests/test/build-signals.test.mjs new file mode 100644 index 0000000..615b125 --- /dev/null +++ b/packages/vercel-optimize-tests/test/build-signals.test.mjs @@ -0,0 +1,161 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + collectBuildSignals, + normalizeMachineType, + MAX_DETAIL_FETCHES, +} from '../../../skills/vercel-optimize/lib/build-signals.mjs'; + +const NOW = 1_700_000_000_000; + +function listRow(i, extra = {}) { + return { + uid: `dpl_${String(i).padStart(3, '0')}`, + created: NOW - i * 60_000, + readyState: 'READY', + target: 'production', + source: 'git', + meta: { githubCommitRef: 'main' }, + ...extra, + }; +} + +function fakeRun({ pages, details = {}, failFirst = false }) { + const calls = []; + const run = async (args) => { + calls.push(args[1]); + const path = args[1]; + if (path.startsWith('/v7/deployments')) { + if (failFirst) return { ok: false, code: 'FORBIDDEN' }; + const m = path.match(/until=(\d+)/); + const pageIndex = m ? Number(m[1]) : 0; + return { ok: true, data: pages[pageIndex] }; + } + const id = decodeURIComponent(path.split('/')[3].split('?')[0]); + return details[id] ?? { ok: true, data: { duration: { timeForBilling: 180_000 }, resourceConfig: { buildMachine: 'turbo' } } }; + }; + return { run, calls }; +} + +test('normalizeMachineType: accepts strings and object shapes, rejects unknown', () => { + assert.equal(normalizeMachineType('Turbo'), 'turbo'); + assert.equal(normalizeMachineType({ cores: 8, memory: 16384, purchaseType: 'enhanced', machineSelectionType: 'elastic' }), 'enhanced'); + assert.equal(normalizeMachineType({ type: 'enhanced' }), 'enhanced'); + assert.equal(normalizeMachineType({ default: 'standard' }), 'standard'); + assert.equal(normalizeMachineType('elastic'), 'elastic'); + assert.equal(normalizeMachineType('mega'), null); + assert.equal(normalizeMachineType(null), null); + assert.equal(normalizeMachineType({}), null); +}); + +test('collectBuildSignals: requires a project', async () => { + assert.deepEqual(await collectBuildSignals({}), { ok: false, code: 'NO_PROJECT' }); +}); + +test('collectBuildSignals: lists, sorts newest first, and attaches billing detail', async () => { + const { run, calls } = fakeRun({ + pages: [{ deployments: [listRow(2), listRow(0), listRow(1, { readyState: 'CANCELED' })], pagination: { next: null } }], + details: { dpl_002: { ok: true, data: { duration: { timeForBilling: 240_000 }, resourceConfig: { buildMachine: { type: 'turbo' } }, buildSkipped: false } } }, + }); + const out = await collectBuildSignals({ projectId: 'prj_1', orgId: 'team_1', now: NOW, run }); + assert.equal(out.ok, true); + assert.deepEqual(out.deployments.map((d) => d.id), ['dpl_000', 'dpl_001', 'dpl_002']); + assert.equal(out.builtCount, 2); + assert.equal(out.detailedCount, 2); + const d2 = out.deployments.find((d) => d.id === 'dpl_002'); + assert.equal(d2.timeForBillingMs, 240_000); + assert.equal(d2.buildMachine, 'turbo'); + assert.equal(d2.branch, 'main'); + // Canceled deployments are listed but never fetched in detail. + assert.equal(out.deployments.find((d) => d.id === 'dpl_001').timeForBillingMs, null); + assert.ok(calls[0].includes('projectId=prj_1') && calls[0].includes('teamId=team_1')); + assert.ok(calls.every((c) => !c.includes('dpl_001'))); +}); + +test('collectBuildSignals: omits teamId for user-owned projects', async () => { + const { run, calls } = fakeRun({ pages: [{ deployments: [listRow(0)], pagination: {} }] }); + await collectBuildSignals({ projectId: 'prj_1', orgId: 'usr_1', now: NOW, run }); + assert.ok(calls.every((c) => !c.includes('teamId'))); +}); + +test('collectBuildSignals: paginates with until and caps detail fetches', async () => { + const page0 = { deployments: Array.from({ length: 100 }, (_, i) => listRow(i)), pagination: { next: 1 } }; + const page1 = { deployments: Array.from({ length: 30 }, (_, i) => listRow(100 + i)), pagination: { next: null } }; + const { run, calls } = fakeRun({ pages: [page0, page1] }); + const out = await collectBuildSignals({ projectId: 'prj_1', now: NOW, run }); + assert.equal(out.listedCount, 130); + assert.equal(out.builtCount, 130); + assert.equal(out.detailedCount, MAX_DETAIL_FETCHES); + assert.equal(calls.filter((c) => c.startsWith('/v13/')).length, MAX_DETAIL_FETCHES); + assert.equal(out.listTruncated, false); +}); + +test('collectBuildSignals: returns the error when the first list page fails', async () => { + const { run } = fakeRun({ pages: [], failFirst: true }); + assert.deepEqual(await collectBuildSignals({ projectId: 'prj_1', now: NOW, run }), { ok: false, code: 'FORBIDDEN' }); +}); + +test('collectBuildSignals: records per-deployment detail failures without failing the run', async () => { + const { run } = fakeRun({ + pages: [{ deployments: [listRow(0), listRow(1)], pagination: {} }], + details: { dpl_000: { ok: false, code: 'NOT_FOUND' } }, + }); + const out = await collectBuildSignals({ projectId: 'prj_1', now: NOW, run }); + assert.equal(out.ok, true); + assert.equal(out.detailFailures, 1); + assert.equal(out.detailedCount, 1); + assert.equal(out.deployments[0].detailError, 'NOT_FOUND'); +}); + +test('collectBuildSignals: reads the live v13 shape and never keeps the hook secret', async () => { + const SECRET = 'hookSecretValue123'; + const { run } = fakeRun({ + pages: [{ deployments: [listRow(0), listRow(1)], pagination: {} }], + details: { + dpl_000: { ok: true, data: { + duration: { timeForBilling: 120_000, cpuTimeForBilling: 960_000, timeToContainerExit: 101_953 }, + resourceConfig: { buildMachine: { cores: 8, memory: 16384, defaultPurchaseType: 'standard', purchaseType: 'enhanced', machineSelectionType: 'elastic', selectionSource: 'elastic-algorithm' } }, + source: 'git', + meta: { deployHookName: 'cms-publish', deployHookRef: 'main', deployHookId: SECRET, githubCommitRef: 'main' }, + } }, + dpl_001: { ok: true, data: { + duration: { timeForBilling: 120_000, cpuTimeForBilling: 3_600_000 }, + resourceConfig: { buildMachine: { cores: 30, memory: 61440, purchaseType: 'turbo', machineSelectionType: 'fixed', selectionSource: 'project-setting' } }, + source: 'git', + meta: { repoPushedAt: '1793014958000', githubCommitRef: 'main' }, + } }, + }, + }); + const out = await collectBuildSignals({ projectId: 'prj_1', now: NOW, run }); + const [hook, push] = out.deployments; + assert.equal(hook.buildMachine, 'enhanced'); + assert.equal(hook.buildCores, 8); + assert.equal(hook.buildMemoryMb, 16384); + assert.equal(hook.machineSelection, 'elastic'); + assert.equal(hook.cpuTimeForBillingMs, 960_000); + assert.equal(hook.triggeredByDeployHook, true); + assert.equal(push.buildMachine, 'turbo'); + assert.equal(push.machineSelection, 'fixed'); + assert.equal(push.triggeredByDeployHook, false); + assert.ok(!JSON.stringify(out).includes(SECRET), 'deployHookId must never reach signals'); +}); + +import { redactProjectSecrets } from '../../../skills/vercel-optimize/lib/build-signals.mjs'; + +test('redactProjectSecrets: strips Deploy Hook ids and URLs, keeps names and refs', () => { + const cfg = { + id: 'prj_1', + resourceConfig: { buildMachineType: 'turbo' }, + link: { type: 'github', repo: 'r', deployHooks: [{ id: 'hookSecretA', name: 'cms-publish', ref: 'main', createdAt: 1, url: 'https://api.vercel.com/v1/integrations/deploy/prj_1/hookSecretA' }] }, + }; + const out = redactProjectSecrets(cfg); + assert.deepEqual(out.link.deployHooks, [{ name: 'cms-publish', ref: 'main', createdAt: 1 }]); + assert.equal(out.link.repo, 'r'); + assert.equal(out.resourceConfig.buildMachineType, 'turbo'); + assert.ok(!JSON.stringify(out).includes('hookSecretA')); + // Input is not mutated, and configs without hooks pass through unchanged. + assert.equal(cfg.link.deployHooks[0].id, 'hookSecretA'); + const plain = { id: 'prj_2', link: { type: 'github' } }; + assert.equal(redactProjectSecrets(plain), plain); + assert.equal(redactProjectSecrets(null), null); +}); diff --git a/packages/vercel-optimize-tests/test/scan-skips-agent-dirs.test.mjs b/packages/vercel-optimize-tests/test/scan-skips-agent-dirs.test.mjs new file mode 100644 index 0000000..89bfb34 --- /dev/null +++ b/packages/vercel-optimize-tests/test/scan-skips-agent-dirs.test.mjs @@ -0,0 +1,22 @@ +// Regression: an end-to-end run scanned the project-installed skill under .claude/skills and flagged the +// skill's own scanner source as a finding. Agent skill directories must be skipped. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync, cpSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const SKILL = join(dirname(fileURLToPath(import.meta.url)), '../../../skills/vercel-optimize'); + +test('scan-codebase: skips .claude and .agents skill directories', () => { + const repo = mkdtempSync(join(tmpdir(), 'scan-skip-')); + writeFileSync(join(repo, 'package.json'), JSON.stringify({ name: 'x', dependencies: { next: '16.0.0' } })); + for (const dir of ['.claude/skills/vercel-optimize/lib', '.agents/skills/vercel-optimize/lib']) { + mkdirSync(join(repo, dir), { recursive: true }); + cpSync(join(SKILL, 'lib/scanners'), join(repo, dir, 'scanners'), { recursive: true }); + } + const out = execFileSync(process.execPath, [join(SKILL, 'scripts/scan-codebase.mjs'), repo], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }); + assert.doesNotMatch(out, /\.claude|\.agents/); +}); diff --git a/skills/vercel-optimize/SKILL.md b/skills/vercel-optimize/SKILL.md index c8a6d6d..2526531 100644 --- a/skills/vercel-optimize/SKILL.md +++ b/skills/vercel-optimize/SKILL.md @@ -23,7 +23,7 @@ Core doctrine: read [references/doctrine.md](references/doctrine.md) if any rule - Authenticated CLI session: `vercel login`. - Linked app directory: `vercel link`. `VERCEL_PROJECT_ID` can help resolve project config, but `vercel metrics` still requires directory linkage. The link or environment must include the intended project org/team/user scope so the collector can resolve a CLI-safe `--scope` and keep `vercel metrics`, `vercel usage`, and `vercel contract` on the same account. - Node.js 20+. -- Observability Plus for route-level metric-backed recommendations. +- Observability Plus for route-level metric-backed recommendations. Build-cost checks don't need it: build gates read project config, deployments, billing, and build logs, which limited runs still collect. Never put auth tokens in shell commands. Do not type `VERCEL_TOKEN=...`, `--token ...`, or `Authorization: Bearer ...` into commands that may be echoed in chat. @@ -112,14 +112,15 @@ vercel link --yes --project <project-name-or-id> --cwd <app-dir> - `forbidden` or `project_not_found`: fix auth/team scope. Do not pitch Observability Plus. - `all_failed_other`: show the raw error code and ask whether to continue in limited code-only mode. +- Build-focused requests (Build CPU Minutes, build machines, slow or expensive builds): an Observability Plus blocker doesn't limit the build audit. Scope, auth, and link blockers still apply. Tell the user route metrics are unavailable using the build-focused template under Chat Templates, word for word, then rerun with `--continue-without-observability` without waiting for a choice. -Do not silently fall back to code-only mode. If the user accepts a limited audit, rerun collection with: +Do not silently fall back to code-only mode. Except for build-focused requests, if the user accepts a limited audit, rerun collection with: ```bash node scripts/collect-signals.mjs [projectId] --continue-without-observability > "$RUN_DIR/vercel-signals.json" 2> "$RUN_DIR/collect.stderr" ``` -Then scan and merge again. +Then rerun the scan and the merge, adding `--force` to `merge-signals.mjs`. The first merge already wrote `signals.json`; without `--force` the merge fails and the gates run on the blocked collection, which has no usage or build data. ### 2. Gate candidates @@ -313,6 +314,10 @@ Use these messages without adding sales copy or process detail. > Use the verbatim choice template in [references/observability-plus.md](references/observability-plus.md). Do not silently fall back to code-only mode; present the two-path choice: enable Observability Plus and rerun the metric-backed audit, or accept a limited code-only run. +**Route-level metrics unavailable on a build-focused request:** + +> Route-level metrics aren't available for this run. Build-cost checks don't use them, so I'm continuing with the full build audit. This run won't include runtime recommendations. + **Project is not linked:** > This worktree is not linked to a Vercel project. Run `vercel link --yes --project <project-name-or-id> --cwd <app-dir>` and rerun the audit. If the team is known, add `--team <team-id-or-slug>`. diff --git a/skills/vercel-optimize/lib/build-signals.mjs b/skills/vercel-optimize/lib/build-signals.mjs new file mode 100644 index 0000000..62548bf --- /dev/null +++ b/skills/vercel-optimize/lib/build-signals.mjs @@ -0,0 +1,140 @@ +// Build-cost signals for build-scoped gates. Platform data only, so it works without Observability Plus. +// Sources: +// GET /v7/deployments list deployments in the window (state, target, source, branch) +// GET /v13/deployments/:id duration.timeForBilling and duration.cpuTimeForBilling (ms), resourceConfig.buildMachine +// ({ purchaseType, cores, memory, machineSelectionType, selectionSource }), meta.deployHookName +// meta.deployHookId is the secret last segment of the hook URL. Never read, store, or log it. +import { runVercelJson } from './vercel.mjs'; + +export const BUILD_WINDOW_DAYS = 14; +export const LIST_PAGE_SIZE = 100; +export const MAX_LIST_PAGES = 5; +// Detail calls are one request per deployment; cap them to stay well under the 100 req / 60s team limit. +export const MAX_DETAIL_FETCHES = 40; + +export const VCPU_BY_MACHINE = Object.freeze({ basic: 2, standard: 4, enhanced: 8, turbo: 30 }); + +const BUILT_STATES = new Set(['READY', 'ERROR']); + +export function normalizeMachineType(value) { + if (value == null) return null; + const raw = typeof value === 'string' + ? value + : (value.purchaseType ?? value.type ?? value.machineType ?? value.default ?? value.name ?? null); + if (typeof raw !== 'string') return null; + const key = raw.trim().toLowerCase(); + return Object.hasOwn(VCPU_BY_MACHINE, key) || key === 'elastic' ? key : null; +} + +export async function collectBuildSignals({ projectId, orgId, now = Date.now(), run = runVercelJson } = {}) { + if (!projectId) return { ok: false, code: 'NO_PROJECT' }; + const teamQs = orgId && !String(orgId).startsWith('usr_') ? `teamId=${encodeURIComponent(orgId)}` : ''; + const since = now - BUILD_WINDOW_DAYS * 86_400_000; + + const listed = []; + let until = null; + let truncated = false; + for (let page = 0; page < MAX_LIST_PAGES; page += 1) { + const qs = [ + `projectId=${encodeURIComponent(projectId)}`, + `since=${since}`, + `limit=${LIST_PAGE_SIZE}`, + until ? `until=${until}` : null, + teamQs || null, + ].filter(Boolean).join('&'); + const r = await run(['api', `/v7/deployments?${qs}`]); + if (!r?.ok) { + if (page === 0) return { ok: false, code: r?.code ?? 'UNKNOWN' }; + truncated = true; + break; + } + const rows = Array.isArray(r.data?.deployments) ? r.data.deployments : []; + for (const d of rows) listed.push(normalizeListRow(d)); + const next = r.data?.pagination?.next; + if (!next || rows.length < LIST_PAGE_SIZE) break; + if (page === MAX_LIST_PAGES - 1) truncated = true; + until = next; + } + + const deployments = dedupeAndSort(listed); + const toDetail = deployments.filter((d) => BUILT_STATES.has(d.state)).slice(0, MAX_DETAIL_FETCHES); + let detailFailures = 0; + for (const d of toDetail) { + const r = await run(['api', `/v13/deployments/${encodeURIComponent(d.id)}${teamQs ? `?${teamQs}` : ''}`]); + if (!r?.ok) { + detailFailures += 1; + d.detailError = r?.code ?? 'UNKNOWN'; + continue; + } + const duration = r.data?.duration ?? {}; + const machine = r.data?.resourceConfig?.buildMachine; + d.timeForBillingMs = positiveNumber(duration.timeForBilling); + d.cpuTimeForBillingMs = positiveNumber(duration.cpuTimeForBilling); + d.buildMachine = normalizeMachineType(machine); + d.buildCores = positiveNumber(machine?.cores); + d.buildMemoryMb = positiveNumber(machine?.memory); + d.machineSelection = typeof machine?.machineSelectionType === 'string' ? machine.machineSelectionType : null; + d.buildSkipped = r.data?.buildSkipped === true; + // Presence only: source is "git" for both pushes and hook builds, so the hook name is the reliable marker. + d.triggeredByDeployHook = typeof r.data?.meta?.deployHookName === 'string'; + } + + return { + ok: true, + windowDays: BUILD_WINDOW_DAYS, + listedCount: deployments.length, + builtCount: deployments.filter((d) => BUILT_STATES.has(d.state)).length, + detailedCount: toDetail.length - detailFailures, + detailFailures, + listTruncated: truncated, + deployments, + }; +} + +function normalizeListRow(d = {}) { + const meta = d.meta ?? {}; + return { + id: String(d.uid ?? d.id ?? ''), + createdAt: Number(d.created ?? d.createdAt ?? 0) || 0, + state: String(d.readyState ?? d.state ?? '').toUpperCase() || null, + target: d.target ?? null, + source: d.source ?? null, + branch: meta.githubCommitRef ?? meta.gitlabCommitRef ?? meta.bitbucketCommitRef ?? null, + timeForBillingMs: null, + cpuTimeForBillingMs: null, + buildMachine: null, + buildCores: null, + buildMemoryMb: null, + machineSelection: null, + buildSkipped: null, + triggeredByDeployHook: null, + }; +} + +// Project config is stored in signals.json verbatim. link.deployHooks[] carries each hook's id and full URL, +// which are bearer secrets, so replace them with name/ref/createdAt only. +export function redactProjectSecrets(cfg) { + if (!cfg || typeof cfg !== 'object' || !Array.isArray(cfg.link?.deployHooks)) return cfg; + return { + ...cfg, + link: { + ...cfg.link, + deployHooks: cfg.link.deployHooks.map((h) => ({ + name: h?.name ?? null, + ref: h?.ref ?? null, + createdAt: h?.createdAt ?? null, + })), + }, + }; +} + +function positiveNumber(value) { + const n = Number(value); + return Number.isFinite(n) && n > 0 ? n : null; +} + +function dedupeAndSort(rows) { + const byId = new Map(); + for (const row of rows) if (row.id && !byId.has(row.id)) byId.set(row.id, row); + return [...byId.values()].sort((a, b) => (b.createdAt - a.createdAt) || a.id.localeCompare(b.id)); +} diff --git a/skills/vercel-optimize/lib/deep-dive.mjs b/skills/vercel-optimize/lib/deep-dive.mjs index 3d1c02e..cedcd84 100644 --- a/skills/vercel-optimize/lib/deep-dive.mjs +++ b/skills/vercel-optimize/lib/deep-dive.mjs @@ -286,6 +286,11 @@ export const SPEC_GENERATORS = { return []; }, + build_machine_oversized(_c) { + // Project config + sampled deployment billing durations carry the evidence; no per-candidate query. + return []; + }, + region_misconfig(_c) { // Branch 2 (scanner-only) — per-region TTFB metric unavailable today, so no deep-dive query. return []; diff --git a/skills/vercel-optimize/lib/display-labels.mjs b/skills/vercel-optimize/lib/display-labels.mjs index e8f90f8..072cb42 100644 --- a/skills/vercel-optimize/lib/display-labels.mjs +++ b/skills/vercel-optimize/lib/display-labels.mjs @@ -13,6 +13,7 @@ const KIND_LABELS = new Map([ ['cwv_poor', 'Poor Core Web Vitals'], ['platform_fluid_compute', 'Fluid Compute usage'], ['platform_bot_protection', 'Bot traffic'], + ['build_machine_oversized', 'Oversized build machine'], ['rendering_candidate', 'Rendering opportunity'], ['missing_cache_headers', 'Missing cache headers'], ['max_age_without_s_maxage', 'Browser-only cache header'], diff --git a/skills/vercel-optimize/lib/gates/build-machine-oversized.mjs b/skills/vercel-optimize/lib/gates/build-machine-oversized.mjs new file mode 100644 index 0000000..d34de8b --- /dev/null +++ b/skills/vercel-optimize/lib/gates/build-machine-oversized.mjs @@ -0,0 +1,132 @@ +// A project pinned to Enhanced or Turbo pays for every vCPU on every billed build minute. +// A smaller machine is cheaper unless the same build gets slower by more than the vCPU ratio, +// so the gate reports a break-even duration instead of a savings figure. +// Peak memory is not collected, so the headline fix is Elastic, which resizes after out-of-memory failures and +// after consistently short builds. The break-even is supporting context, not the recommendation. +// Account-scoped: the lever is the project's build machine setting, not code. +import { VCPU_BY_MACHINE, normalizeMachineType } from '../build-signals.mjs'; + +export const metadata = { + id: 'build_machine_oversized', + threshold: 'fixed build machine is enhanced or turbo AND >= 5 successful billed builds sampled AND estimated window CPU minutes >= 300 AND no memory-related Elastic resize reason', + billingDimension: 'build', + scope: 'account', + sourceCitation: 'vercel-optimize gate threshold', + description: + 'Project is pinned to a larger build machine. Build CPU Minutes are billed duration times vCPUs, so the recommended fix is Elastic, which sizes each build. The gate reports the break-even billed duration on the next smaller machine as context.', +}; + +const NEXT_SMALLER = Object.freeze({ turbo: 'enhanced', enhanced: 'standard' }); +const MIN_SAMPLED_BUILDS = 5; +const MIN_WINDOW_CPU_MINUTES = 300; +const HIGH_PRIORITY_CPU_MINUTES = 5000; +const BUILT_STATES = new Set(['READY', 'ERROR']); +const MEMORY_REASONS = new Set(['oom-failure', 'high-peak-memory']); +// Standard has 8 GB, half of Enhanced; memory-bound builds fail there, so that step carries less certainty. +const SMALLER_MEMORY_GB = Object.freeze({ enhanced: 16, standard: 8 }); + +export function gate(signals) { + // Without project config we can't tell which machine is pinned; recommending a change blind erodes trust. + const project = signals?.project; + if (!project || project.error) return []; + + const rc = project.resourceConfig ?? {}; + const selection = typeof rc.buildMachineSelection === 'string' ? rc.buildMachineSelection : null; + if (selection === 'elastic') return []; + + const machine = normalizeMachineType(rc.buildMachineType); + const smaller = NEXT_SMALLER[machine]; + if (!smaller) return []; + + const builds = signals?.builds; + if (!builds?.ok || !Array.isArray(builds.deployments)) return []; + + // Eligible = built on the pinned machine (or machine unknown) and not skipped. + // The median uses successful builds only: early failures bill a short minimum and would drag the break-even down. + const eligible = builds.deployments.filter((d) => + BUILT_STATES.has(d?.state) + && d.buildSkipped !== true + && d.machineSelection !== 'elastic' + && (d.buildMachine == null || d.buildMachine === machine)); + const billed = eligible.filter((d) => Number.isFinite(d?.timeForBillingMs) && d.timeForBillingMs > 0); + const sampled = billed.filter((d) => d.state === 'READY'); + if (sampled.length < MIN_SAMPLED_BUILDS) return []; + + // Elastic records why it resized a project. Memory reasons mean a smaller fixed machine is the wrong advice. + const elasticReason = typeof rc.buildMachineElasticReason === 'string' ? rc.buildMachineElasticReason : null; + if (MEMORY_REASONS.has(elasticReason)) return []; + + const builtCount = eligible.length; + const erroredSampled = billed.filter((d) => d.state === 'ERROR').length; + const medianBilledMinutes = median(sampled.map((d) => d.timeForBillingMs / 60_000)); + const vcpu = VCPU_BY_MACHINE[machine]; + const smallerVcpu = VCPU_BY_MACHINE[smaller]; + // Prefer Vercel's own CPU-minute figure when every sampled build has it; fall back to billed minutes x vCPUs. + const cpuSource = sampled.every((d) => Number.isFinite(d.cpuTimeForBillingMs)) ? 'cpuTimeForBilling' : 'billed_x_vcpu'; + const medianCpuMinutes = cpuSource === 'cpuTimeForBilling' + ? median(sampled.map((d) => d.cpuTimeForBillingMs / 60_000)) + : medianBilledMinutes * vcpu; + const windowCpuMinutesEstimate = Math.round(medianCpuMinutes * builtCount); + const hookTriggeredBuilds = eligible.filter((d) => d.triggeredByDeployHook === true).length; + if (windowCpuMinutesEstimate < MIN_WINDOW_CPU_MINUTES) return []; + + // Billed minutes are whole minutes, so the break-even on the smaller machine is floored. + const breakEvenBilledMinutes = Math.floor((medianBilledMinutes * vcpu) / smallerVcpu); + const isEnterprise = signals?.plan?.plan === 'enterprise'; + + return [{ + kind: metadata.id, + scope: 'account', + files: [], + priority: windowCpuMinutesEstimate >= HIGH_PRIORITY_CPU_MINUTES ? 60 : 45, + confidence: confidenceFor(machine, selection), + o11ySignal: `machine=${machine} builds=${builtCount} median_billed_min=${round1(medianBilledMinutes)} fix=elastic smaller=${smaller} break_even_min=${breakEvenBilledMinutes}`, + reason: `Project is pinned to ${capitalize(machine)} (${vcpu} vCPUs) with a median ${round1(medianBilledMinutes)} billed minutes per build; switch to Elastic`, + question: [ + `Builds bill a median ${round1(medianBilledMinutes)} minutes on ${capitalize(machine)}. Would Elastic, which sizes each build, cover this project?`, + `For context, ${capitalize(smaller)} alone costs less unless the same builds take longer than ${breakEvenBilledMinutes} billed minutes there.`, + `Peak build memory is not collected, and ${capitalize(smaller)} has ${SMALLER_MEMORY_GB[smaller]} GB. Elastic moves a project to a larger machine after an out-of-memory failure, so recommend Elastic, not a fixed smaller machine.`, + 'After switching, expect a few builds at the old size while Elastic adjusts.', + isEnterprise ? 'On Enterprise, confirm the machine is not set by contract; contract machines change through the account manager.' : null, + ].filter(Boolean).join(' '), + evidence: { + metric: 'builds.deployments', + machineType: machine, + machineSelection: selection, + smallerMachine: smaller, + vcpu, + smallerVcpu, + vcpuRatio: round2(vcpu / smallerVcpu), + builtCount, + sampledBuilds: sampled.length, + erroredBuildsExcludedFromMedian: erroredSampled, + smallerMachineMemoryGb: SMALLER_MEMORY_GB[smaller], + peakMemoryCollected: false, + medianBilledMinutes: round1(medianBilledMinutes), + medianCpuMinutes: round1(medianCpuMinutes), + cpuMinutesSource: cpuSource, + hookTriggeredBuilds, + recommendedFix: 'elastic', + windowCpuMinutesEstimate, + breakEvenBilledMinutesOnSmaller: breakEvenBilledMinutes, + detailSampleTruncated: billed.length < builtCount, + windowDays: builds.windowDays ?? null, + }, + }]; +} + +function confidenceFor(machine, selection) { + // Turbo -> Enhanced keeps 16 GB and a 3.75x vCPU ratio; Enhanced -> Standard halves memory to 8 GB. + const base = machine === 'turbo' ? 0.8 : 0.55; + return selection === 'fixed' ? base : round2(base - 0.1); +} + +function median(values) { + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +} + +function round1(n) { return Math.round(n * 10) / 10; } +function round2(n) { return Math.round(n * 100) / 100; } +function capitalize(s) { return s.charAt(0).toUpperCase() + s.slice(1); } diff --git a/skills/vercel-optimize/lib/gates/index.mjs b/skills/vercel-optimize/lib/gates/index.mjs index 345c1e7..a481f49 100644 --- a/skills/vercel-optimize/lib/gates/index.mjs +++ b/skills/vercel-optimize/lib/gates/index.mjs @@ -12,6 +12,7 @@ import * as scannerDriven from './scanner-driven.mjs'; import * as observabilityEventsAttribution from './observability-events-attribution.mjs'; import * as usageSpikeTriage from './usage-spike-triage.mjs'; import * as buildMinutesFanout from './build-minutes-fanout.mjs'; +import * as buildMachineOversized from './build-machine-oversized.mjs'; import * as regionMisconfig from './region-misconfig.mjs'; // Intentionally NOT registered: @@ -33,6 +34,7 @@ export const gates = [ observabilityEventsAttribution, usageSpikeTriage, buildMinutesFanout, + buildMachineOversized, regionMisconfig, ]; @@ -42,4 +44,4 @@ export const DEFAULT_MAX_CODE_CANDIDATES = 6; export const MAX_CODE_CANDIDATES = DEFAULT_MAX_CODE_CANDIDATES; // Bump on any threshold change so report + iteration baselines can detect gate-logic drift. -export const GATE_VERSION = '1.8.0'; +export const GATE_VERSION = '1.9.0'; diff --git a/skills/vercel-optimize/lib/grade-recommendation.mjs b/skills/vercel-optimize/lib/grade-recommendation.mjs index ca577e9..cdd2ba9 100644 --- a/skills/vercel-optimize/lib/grade-recommendation.mjs +++ b/skills/vercel-optimize/lib/grade-recommendation.mjs @@ -1,10 +1,11 @@ // 4-axis rubric (specificity, actionability, grounding, evidence) → bucket. See references/recommendations.md. -// Account-scope (platform_*) recs use a separate grounding/evidence pair — they structurally cannot produce file:line. +// Account-scope recs (platform_* kinds, scope: 'account', or an `<account>` candidateRef) use a separate +// grounding/evidence pair — they structurally cannot produce file:line. const HEDGE_WORDS = /\b(consider|might|may|could|perhaps|maybe|likely|probably)\b/gi; const VERB_OPENERS = /^\s*(?:[-*]\s+|\d+[.)]\s+|[*_]+)?(?:add|set|enable|disable|replace|remove|move|wrap|cache|defer|parallelize|introduce|configure|update|change|switch|opt[-\s]?in|opt[-\s]?out|export|import|install|run|delete|rename)/im; -const COUNT_WORDS_RE = /\b(errors?|queries|invocations|requests|reads|writes|bytes|fetch(?:es)?|calls?|hits?|misses?|seconds?|images?|deployments?|cold[- ]?starts?|users?)\b/gi; -const UNIT_RE = /\b\d[\d.,]*\s*(?:%|ms|s|sec|seconds?|min|minutes?|h|hours?|GB|MB|KB|K|M|B|rps|qps|req\/s|reqs?\/min)\b/gi; +const COUNT_WORDS_RE = /\b(errors?|queries|invocations|requests|reads|writes|bytes|fetch(?:es)?|calls?|hits?|misses?|seconds?|images?|deployments?|builds?|cold[- ]?starts?|users?)\b/gi; +const UNIT_RE = /\b\d[\d.,]*\s*(?:%|ms|s|sec|seconds?|min|minutes?|(?:billed|CPU)\s+minutes?|vCPUs?|h|hours?|GB|MB|KB|K|M|B|rps|qps|req\/s|reqs?\/min)\b/gi; const CODE_FENCE_RE = /```[\s\S]*?```/g; const INLINE_CODE_RE = /`[^`\n]{10,}`/g; const FILE_LINE_RE = /[\w/.\-()\[\]]+\.\w+:\d+/g; @@ -13,7 +14,18 @@ const FILE_LINE_RE = /[\w/.\-()\[\]]+\.\w+:\d+/g; const W = { grounding: 0.35, evidence: 0.30, specificity: 0.20, actionability: 0.15 }; export function gradeRecommendation(rec, ctx = {}) { - const accountScope = isAccountScope(rec); + if (isAccountScope(rec)) return gradeAs(rec, ctx, true); + // Gate candidates without a route carry the `<account>` sentinel (investigation-brief.mjs). Some, like + // build_minutes_fanout, also cite scanner file:line evidence, so grade both ways and keep the stronger. + if (hasAccountSentinel(rec)) { + const asRoute = gradeAs(rec, ctx, false); + const asAccount = gradeAs(rec, ctx, true); + return asAccount.overall > asRoute.overall ? asAccount : asRoute; + } + return gradeAs(rec, ctx, false); +} + +function gradeAs(rec, ctx, accountScope) { const specificity = scoreSpecificity(rec); const actionability = scoreActionability(rec); const grounding = accountScope ? scoreGroundingAccount(rec) : scoreGrounding(rec, ctx); @@ -36,6 +48,10 @@ function isAccountScope(rec) { return false; } +function hasAccountSentinel(rec) { + return typeof rec?.candidateRef === 'string' && rec.candidateRef.endsWith(':<account>'); +} + function grade(overall) { if (overall >= 0.85) return 'Excellent'; if (overall >= 0.70) return 'Good'; @@ -108,7 +124,7 @@ function scoreEvidence(rec) { // No findingRefs/code fences possible — grade structural tie to gate + signal-quoting. function scoreGroundingAccount(rec) { let s = 0; - if (typeof rec.candidateRef === 'string' && rec.candidateRef.startsWith('platform_')) s += 0.4; + if (typeof rec.candidateRef === 'string' && (rec.candidateRef.startsWith('platform_') || hasAccountSentinel(rec))) s += 0.4; else if (typeof rec.candidateRef === 'string' && rec.candidateRef.length > 0) s += 0.2; // Quoting deep-dive data in why/fix is the account-scope equivalent of citing file:line. const text = [rec.why, rec.fix, rec.verify].filter((x) => typeof x === 'string').join('\n'); diff --git a/skills/vercel-optimize/lib/impact-label.mjs b/skills/vercel-optimize/lib/impact-label.mjs index b9bb61e..201e98b 100644 --- a/skills/vercel-optimize/lib/impact-label.mjs +++ b/skills/vercel-optimize/lib/impact-label.mjs @@ -1,3 +1,5 @@ +import { gate as buildMachineGate } from './gates/build-machine-oversized.mjs'; +import { gate as buildFanoutGate } from './gates/build-minutes-fanout.mjs'; import { impactMagnitude } from './impact-magnitude.mjs'; const SLOW_ROUTE_P95_THRESHOLD_MS = 500; @@ -28,6 +30,7 @@ export function synthesizeImpactFromSignal(rec, signals = {}) { rec?.evidence?.o11ySignal, rec?.why, rec?.what, + gateSignalFallback(rec, signals), ].filter((v) => typeof v === 'string' && v.trim()).join('\n') || null; if (!sig || !tier) return null; const m = String(sig); @@ -78,9 +81,21 @@ export function synthesizeImpactFromSignal(rec, signals = {}) { if (coldPct != null) { return `${tier} impact — current cold-start share is ${coldPct}%; the gate fires above ${COLD_START_THRESHOLD_PCT}%.`; } + const pinnedMachine = m.match(/machine=(enhanced|turbo)\b/i)?.[1] ?? null; + const smallerMachine = m.match(/smaller=(standard|enhanced)\b/i)?.[1] ?? null; + const medianBilledMin = parseSigNumber(m, /median_billed_min=([\d.]+)/); + const breakEvenMin = parseSigNumber(m, /break_even_min=([\d]+)/); + if (pinnedMachine && smallerMachine && medianBilledMin != null && breakEvenMin != null) { + return `${tier} impact — builds on the pinned ${capitalizeWord(pinnedMachine)} machine bill a median ${medianBilledMin} minutes; switching to Elastic lets each build use a smaller machine when it fits (${capitalizeWord(smallerMachine)} alone costs less unless builds take longer than ${breakEvenMin} billed minutes).`; + } if (rec?.candidateRef?.startsWith('build_minutes_fanout:') || rec?.kind === 'build_minutes_fanout') { - return buildSharePct != null - ? `${tier} impact — Build CPU Minutes account for ${buildSharePct}% of observed billed cost in this window.` + if (buildSharePct != null && buildSharePct > 0) { + return `${tier} impact — Build CPU Minutes account for ${buildSharePct}% of observed billed cost in this window.`; + } + // Fired by the repo scanner alone (usage missing or zero): don't imply a billing threshold was measured. + const scannerFindings = parseSigNumber(m, /scanner_findings=(\d+)/); + return scannerFindings + ? `${tier} impact — the repo disables or bypasses the Turborepo build cache; build usage for this window wasn't available to size it.` : `${tier} impact — Build CPU Minutes exceeded the gate threshold in this window.`; } return `${tier} impact — see follow-up metrics for magnitude.`; @@ -124,3 +139,27 @@ function formatInteger(n) { if (!Number.isFinite(n)) return String(n); return Math.round(n).toLocaleString('en-US'); } + +function capitalizeWord(s) { + return s.charAt(0).toUpperCase() + s.slice(1).toLowerCase(); +} + +// Account-scoped build gates are deterministic over signals.json, so when a written recommendation +// omits the gate's o11ySignal, re-derive it instead of falling back to a placeholder impact line. +const BUILD_GATES = Object.freeze({ + build_machine_oversized: { gate: buildMachineGate, has: /machine=(?:enhanced|turbo)\b/i }, + build_minutes_fanout: { gate: buildFanoutGate, has: /build_minutes_share=/i }, +}); + +function gateSignalFallback(rec, signals) { + const kind = rec?.kind ?? (typeof rec?.candidateRef === 'string' ? rec.candidateRef.split(':')[0] : null); + const entry = BUILD_GATES[kind]; + if (!entry || !signals || typeof signals !== 'object') return null; + const existing = [rec?.o11ySignal, rec?.evidence?.o11ySignal].filter((v) => typeof v === 'string').join('\n'); + if (entry.has.test(existing)) return null; + try { + return entry.gate(signals)?.[0]?.o11ySignal ?? null; + } catch { + return null; + } +} diff --git a/skills/vercel-optimize/references/candidates.md b/skills/vercel-optimize/references/candidates.md index 9bcc87f..784ffac 100644 --- a/skills/vercel-optimize/references/candidates.md +++ b/skills/vercel-optimize/references/candidates.md @@ -6,10 +6,21 @@ The deterministic threshold expressions that turn observability signals into investigation candidates. Pure JS, no LLM. Thresholds live in `lib/gates/*.mjs`. -Total gates: 15. Budget cap: `MAX_CODE_CANDIDATES = 6`. Gate version: `1.8.0`. +Total gates: 16. Budget cap: `MAX_CODE_CANDIDATES = 6`. Gate version: `1.9.0`. ## Gates +### `build_machine_oversized` + +- **Threshold**: `fixed build machine is enhanced or turbo AND >= 5 successful billed builds sampled AND estimated window CPU minutes >= 300 AND no memory-related Elastic resize reason` +- **Billing dimension**: build +- **Scope**: account +- **Source citation**: `vercel-optimize gate threshold` + +Project is pinned to a larger build machine. Build CPU Minutes are billed duration times vCPUs, so the recommended fix is Elastic, which sizes each build. The gate reports the break-even billed duration on the next smaller machine as context. + +--- + ### `build_minutes_fanout` - **Threshold**: `Build Minutes share > 0.15 OR turbo-force-bypass finding present` diff --git a/skills/vercel-optimize/references/data-collection.md b/skills/vercel-optimize/references/data-collection.md index 140c787..153e7a0 100644 --- a/skills/vercel-optimize/references/data-collection.md +++ b/skills/vercel-optimize/references/data-collection.md @@ -20,7 +20,7 @@ The merged `signals.json` has this top-level shape: ```json { - "schemaVersion": "1.2", + "schemaVersion": "1.3", "collectedAt": "2026-05-12T20:48:44.123Z", "timeWindow": "14d", "projectId": "prj_xxx", @@ -49,9 +49,10 @@ The merged `signals.json` has this top-level shape: "observabilityPlusBlocker": null | "no_oplus_probe" | "project_disabled" | "payment_required" | "forbidden" | "daily_quota_exceeded" | "project_not_found" | "not_linked" | "all_failed_other" | "no_traffic", "observabilityPlusBlockerDetail": "...", "plan": { "plan": "hobby" | "pro" | "enterprise" | "uncertain", "reason": "..." }, - "project": { /* /v9/projects/:id response; team-owned projects include ?teamId */ }, + "project": { /* /v9/projects/:id response; team-owned projects include ?teamId. link.deployHooks[] is reduced to {name, ref, createdAt}: hook ids and URLs are secrets */ }, "contract": { "context": "...", "commitments": [], "totalCommitments": 0 }, "usage": { /* vercel usage --format json --breakdown daily, or null */ }, + "builds": { "ok": true, "windowDays": 14, "listedCount": 0, "builtCount": 0, "detailedCount": 0, "detailFailures": 0, "listTruncated": false, "deployments": [ /* {id, createdAt, state, target, source, branch, timeForBillingMs, cpuTimeForBillingMs, buildMachine, buildCores, buildMemoryMb, machineSelection, buildSkipped, triggeredByDeployHook} */ ] } | { "ok": false, "code": "..." }, "usageError": null | "USAGE_UNAVAILABLE" | "USAGE_CONTEXT_MISMATCH" | "NOT_COLLECTED_OBSERVABILITY_BLOCKED" | "NOT_COLLECTED_UNSUPPORTED_FRAMEWORK" | "EXIT_<n>" | "UNKNOWN", "stack": { /* framework + version + router + ORM + monorepo */ }, "codebase": { /* scan-codebase output: stack + routes + findings */ }, @@ -76,6 +77,7 @@ Downstream consumers reference `signals.<field>` paths verbatim. Bumping `schema | Framework support | local `package.json` via `detectStack()` + `classifyFrameworkSupport()` | Code-backed route recommendations | Stop before metric fan-out on unsupported frameworks unless the user chooses `--continue-unsupported-framework` | | CLI command scope | `vercel whoami --format json`, then `vercel api /v2/teams/:orgId` when a linked `team_...` ID must be converted to a slug | Keeps `vercel metrics`, `vercel usage`, and `vercel contract` on the linked project's account instead of the user's current/personal scope | `PROJECT_SCOPE_UNRESOLVED` or `SCOPE_UNRESOLVED`; stop and ask the user to clarify the intended project/account, then re-link under the intended team or personal account | | Project/scope verification | `vercel api /v9/projects/:id?teamId=<orgId>` for team-owned projects; omit `teamId` for `usr_...` user-owned projects | Proves the resolved account can read the resolved project before Observability Plus or billing conclusions | `PROJECT_SCOPE_MISMATCH`; stop and ask the user to confirm the exact project and team/personal scope. Do not report Observability Plus as missing until this check passes | +| Build deployments | `vercel api /v7/deployments?projectId=<id>&since=<14d>&limit=100` (up to 5 pages), then `vercel api /v13/deployments/:id` for at most 40 most recent READY/ERROR deployments (`duration.timeForBilling`, `duration.cpuTimeForBilling`, `resourceConfig.buildMachine.{purchaseType,cores,memory,machineSelectionType}`, `buildSkipped`, presence of `meta.deployHookName`). `source` is `git` for both pushes and Deploy Hook builds, so the hook name is the marker. `meta.deployHookId` is the secret end of the hook URL and is never read or stored. Team-owned projects add `teamId`. Needs no Observability Plus. | `build_machine_oversized` | `builds.ok=false` with the CLI error code; build gates emit no candidates | | Observability Plus configuration | Vercel CLI/API probe plus one metric access check; user-owned projects skip the team configuration endpoint and rely on the scoped metrics probe | All `metrics.*` signals | Stop early when the account lacks Observability Plus or this project is disabled | | Observability Plus metrics access | One canary `vercel metrics vercel.request.count --since 14d --limit 1`, then full fan-out only if it succeeds | All `metrics.*` signals | Set `observabilityPlusUsable=false` with blocker detail; emit a blocker document after project/scope verification but before billing collection unless `--continue-without-observability` is passed | | Project config | Verified project API response from project/scope verification | Fluid Compute, BotID, Speed Insights, security flags | Stop on ownership mismatch; otherwise gates that need missing optional fields skip | diff --git a/skills/vercel-optimize/references/docs-library.json b/skills/vercel-optimize/references/docs-library.json index ad4e330..cacb219 100644 --- a/skills/vercel-optimize/references/docs-library.json +++ b/skills/vercel-optimize/references/docs-library.json @@ -152,7 +152,19 @@ { "url": "https://vercel.com/docs/builds", "topic": "Builds — build infrastructure, monorepo build controls, concurrency, and queues", - "appliesTo": ["build_minutes_fanout"], + "appliesTo": ["build_minutes_fanout", "build_machine_oversized"], + "applicableFrameworks": ["*"] + }, + { + "url": "https://vercel.com/docs/builds/managing-builds", + "topic": "Managing builds — build machine types, Elastic sizing, on-demand concurrent builds", + "appliesTo": ["build_machine_oversized"], + "applicableFrameworks": ["*"] + }, + { + "url": "https://vercel.com/docs/pricing#builds", + "topic": "Build pricing — Build CPU Minutes as billed duration times vCPUs", + "appliesTo": ["build_machine_oversized"], "applicableFrameworks": ["*"] }, { diff --git a/skills/vercel-optimize/references/observability-plus.md b/skills/vercel-optimize/references/observability-plus.md index 908ca8c..6fdcc52 100644 --- a/skills/vercel-optimize/references/observability-plus.md +++ b/skills/vercel-optimize/references/observability-plus.md @@ -19,6 +19,8 @@ This is a data dependency, not an upgrade pitch. The skill ranks work by observe Scanner-only mode can still catch traffic-independent code issues, but it cannot rank the hottest routes or prove cost impact. Make that tradeoff explicit before continuing. +Build gates (`build_minutes_fanout`, `build_machine_oversized`) are not in this table: they use project config, deployments, billing, and build logs. For a build-focused request, skip this template and use the build-focused template in SKILL.md instead. + ## User Template Render this template first, then wait for the user's choice. Replace only `<detail>`. Do not add a preface; the heading is the opening line. diff --git a/skills/vercel-optimize/references/support-topics/build-machine-right-sizing.md b/skills/vercel-optimize/references/support-topics/build-machine-right-sizing.md new file mode 100644 index 0000000..c77d1b3 --- /dev/null +++ b/skills/vercel-optimize/references/support-topics/build-machine-right-sizing.md @@ -0,0 +1,22 @@ +--- +id: build-machine-right-sizing +title: Build machine right-sizing +status: active +candidateKinds: ["build_machine_oversized"] +frameworks: ["*"] +priority: 90 +citations: ["https://vercel.com/docs/builds/managing-builds", "https://vercel.com/docs/pricing#builds"] +maxBriefChars: 900 +--- + +## Investigation Brief +Build CPU Minutes are billed duration times vCPUs. A pinned Enhanced or Turbo machine pays off only when it cuts build time by at least the vCPU ratio. Compare the median billed duration with the break-even on the next smaller machine. + +## Evidence To Check +Confirm the selection is fixed and the sample ran on the pinned machine. Look for out-of-memory failures or memory-heavy steps before suggesting a smaller machine. + +## Do Not Recommend When +Skip when builds ran out of memory on a smaller machine before, or the sample is small. On Enterprise, do not recommend a change the contract sets; point to the account manager. + +## Verification +Recommend Elastic first. Quote the build count, median billed minutes, vCPUs, and CPU minutes per build; give the break-even as context, and suggest comparing billed duration over an equal window afterward. Never quote savings. diff --git a/skills/vercel-optimize/scripts/collect-signals.mjs b/skills/vercel-optimize/scripts/collect-signals.mjs index 2831fb9..f365db8 100755 --- a/skills/vercel-optimize/scripts/collect-signals.mjs +++ b/skills/vercel-optimize/scripts/collect-signals.mjs @@ -3,6 +3,7 @@ // usage + codebase stack + metric queries. Status → stderr, JSON → stdout. // Degrades gracefully when capabilities are missing. +import { collectBuildSignals, redactProjectSecrets } from '../lib/build-signals.mjs'; import { checkCliVersion, checkAuth, @@ -24,7 +25,7 @@ import { import { classifyFrameworkSupport } from '../lib/framework-support.mjs'; import { QUERIES, TIME_WINDOW, normalizerFor } from '../lib/queries.mjs'; -const SCHEMA_VERSION = '1.2'; +const SCHEMA_VERSION = '1.3'; const log = (...args) => console.error('[collect-signals]', ...args); @@ -131,7 +132,8 @@ async function main() { log(`command scope resolved (source=${commandScope.source}; scoped=${scope ? 'yes' : 'no'})`); log('validating linked project belongs to the resolved scope…'); - const projectCfg = await getProjectConfig(project.projectId, project.orgId); + // Deploy Hook ids and URLs are secrets: anyone holding one can trigger builds. Keep only names and refs. + const projectCfg = redactProjectSecrets(await getProjectConfig(project.projectId, project.orgId)); const projectScope = validateProjectScope(projectCfg, project); if (!projectScope.ok) { throw new Error(`PROJECT_SCOPE_MISMATCH: ${projectScope.detail} Ask the user to confirm the exact Vercel project and team/personal scope, then rerun after \`vercel link --yes --project <project-name-or-id> --team <team-slug>\` or after setting both VERCEL_PROJECT_ID and VERCEL_ORG_ID for the intended scope.`); @@ -236,12 +238,17 @@ async function main() { log('continuing after Observability Plus blocker because --continue-without-observability was set'); } - log('pulling account plan + contract + usage in parallel…'); - const [accountPlan, contract, usageResult] = await Promise.all([ + log('pulling account plan + contract + usage + builds in parallel…'); + const [accountPlan, contract, usageResult, builds] = await Promise.all([ getAccountPlan(project.orgId || scope), getContract(scope), getUsage({ days: 14, scope }), + collectBuildSignals({ projectId: project.projectId, orgId: project.orgId }) + .catch((err) => ({ ok: false, code: 'BUILD_SIGNALS_FAILED', detail: String(err?.message ?? err) })), ]); + log(builds?.ok + ? `builds: ${builds.builtCount} built deployments in ${builds.windowDays}d; billing detail for ${builds.detailedCount}${builds.listTruncated ? ' (list truncated)' : ''}` + : `builds: unavailable (${builds?.code ?? 'unknown'}) — build gates will skip`); let usage = null; let usageContextMismatch = false; @@ -338,6 +345,7 @@ async function main() { usageError: usageResult?.ok ? (usageContextMismatch ? 'USAGE_CONTEXT_MISMATCH' : null) : (usageResult?.code ?? 'UNKNOWN'), + builds, stack, metrics, metricsSchema: schema, diff --git a/skills/vercel-optimize/scripts/render-report.mjs b/skills/vercel-optimize/scripts/render-report.mjs index 542a992..8f7b481 100644 --- a/skills/vercel-optimize/scripts/render-report.mjs +++ b/skills/vercel-optimize/scripts/render-report.mjs @@ -327,6 +327,7 @@ function candidateFamily(kind) { case 'middleware_heavy': return 'middleware'; case 'build_minutes_fanout': + case 'build_machine_oversized': return 'build'; default: return kind || null; diff --git a/skills/vercel-optimize/scripts/scan-codebase.mjs b/skills/vercel-optimize/scripts/scan-codebase.mjs index 9d416b5..645e3f7 100755 --- a/skills/vercel-optimize/scripts/scan-codebase.mjs +++ b/skills/vercel-optimize/scripts/scan-codebase.mjs @@ -15,7 +15,8 @@ import { } from '../lib/workspace-resolver.mjs'; const SCHEMA_VERSION = '1.0'; -const SKIP_DIRS = new Set(['node_modules', '.next', '.vercel', 'dist', 'build', '.git', 'coverage', '.turbo', '__tests__', 'cypress']); +// .claude and .agents hold project-installed agent skills (including this one); scanning them flags the skill's own source. +const SKIP_DIRS = new Set(['node_modules', '.next', '.vercel', 'dist', 'build', '.git', 'coverage', '.turbo', '__tests__', 'cypress', '.claude', '.agents']); const SKIP_FILE_PATTERNS = [/\.test\./, /\.spec\./, /\.d\.ts$/]; async function main() { |
