diff options
| author | John Pham <john.pham@vercel.com> | 2026-05-26 15:53:17 -0700 |
|---|---|---|
| committer | John Pham <john.pham@vercel.com> | 2026-05-26 18:39:35 -0700 |
| commit | 583754a2b2d022e61396b14dc46db5f20c3c1d2b (patch) | |
| tree | 6fd51225135e7f8794353b916ba65ac0c0e1f7a2 | |
| parent | 18a24346600009dc3fcb99e4b2cd83b301601775 (diff) | |
Harden vercel optimize report safetyjohnphamous/vercel-optimize-observation-safety
21 files changed, 822 insertions, 168 deletions
diff --git a/packages/vercel-optimize-tests/test/citations.test.mjs b/packages/vercel-optimize-tests/test/citations.test.mjs index eb6e897..8e6619a 100644 --- a/packages/vercel-optimize-tests/test/citations.test.mjs +++ b/packages/vercel-optimize-tests/test/citations.test.mjs @@ -92,6 +92,18 @@ test("libraryForStack: 'use cache' surfaces for Next 15", async () => { assert.ok(useCache, "'use cache' must appear for Next 15+"); }); +test('libraryForStack: unstable_cache is available for Next 15 but not Next 16', async () => { + const next15 = await libraryForStack('next', '15.4.10'); + assert.ok(next15.urls.find(u => + u.url === 'https://nextjs.org/docs/app/api-reference/functions/unstable_cache' + ), 'unstable_cache citation should remain available for Next 15'); + + const next16 = await libraryForStack('next', '16.2.2'); + assert.equal(next16.urls.find(u => + u.url === 'https://nextjs.org/docs/app/api-reference/functions/unstable_cache' + ), undefined, 'unstable_cache citation should not be offered for Next 16'); +}); + test('sanitizeCitations strips unknown URL', async () => { const rec = { citations: [ 'https://vercel.com/docs/caching/cdn-cache', diff --git a/packages/vercel-optimize-tests/test/grade-and-verify.test.mjs b/packages/vercel-optimize-tests/test/grade-and-verify.test.mjs index 98041ca..9cd4f4e 100644 --- a/packages/vercel-optimize-tests/test/grade-and-verify.test.mjs +++ b/packages/vercel-optimize-tests/test/grade-and-verify.test.mjs @@ -260,6 +260,85 @@ test('extractClaims + verifyClaim: geolocation Vary proof must be an actual Vary assert.match(failed.reason, /X-Vercel-IP-Country/); }); +test('extractClaims + verifyClaim: high-cardinality latitude/longitude Vary headers are unsafe', async () => { + const repoRoot = await mkdtemp(join(tmpdir(), 'vo-geo-cardinality-')); + await mkdir(join(repoRoot, 'app/api/geocode'), { recursive: true }); + await writeFile( + join(repoRoot, 'app/api/geocode/route.ts'), + "export function GET(req) { return Response.json({ lat: req.headers.get('x-vercel-ip-latitude') }); }\n" + ); + + const rec = { + candidateRef: 'uncached_route:/api/geocode', + what: 'Cache geocode suggestions with s-maxage.', + fix: "Return `Cache-Control: public, s-maxage=300` and `Vary: X-Vercel-IP-Latitude, X-Vercel-IP-Longitude`.", + affectedFiles: ['app/api/geocode/route.ts'], + }; + const claims = extractClaims(rec, { repoRoot, framework: 'next', version: '15.4.10' }); + const cardinality = claims.find((c) => c.type === 'cache_vary_cardinality_safe'); + assert.ok(cardinality); + const failed = await verifyClaim(cardinality); + assert.equal(failed.disposition, 'failed'); + assert.match(failed.reason, /high-cardinality/); + + const coarse = await verifyClaim({ + ...cardinality, + rec: { + ...rec, + fix: "Return `Cache-Control: public, s-maxage=300` and `Vary: X-Vercel-IP-City` after confirming city-level results are acceptable.", + }, + }); + assert.equal(coarse.disposition, 'verified'); +}); + +test('extractClaims + verifyClaim: Vercel geo Vary uses documented coarse header names', async () => { + const repoRoot = await mkdtemp(join(tmpdir(), 'vo-geo-country-region-')); + await mkdir(join(repoRoot, 'app/api/region'), { recursive: true }); + await writeFile( + join(repoRoot, 'app/api/region/route.ts'), + "export function GET(req) { return Response.json({ region: req.headers.get('x-vercel-ip-country-region') }); }\n" + ); + + const rec = { + candidateRef: 'uncached_route:/api/region', + what: 'Cache region-specific data with s-maxage.', + fix: "Return `Cache-Control: public, s-maxage=300` and `Vary: X-Vercel-IP-Region`.", + affectedFiles: ['app/api/region/route.ts'], + }; + const claim = extractClaims(rec, { repoRoot, framework: 'next', version: '15.4.10' }) + .find((c) => c.type === 'cache_vary_matches_dynamic_inputs'); + assert.ok(claim); + + const failed = await verifyClaim(claim); + assert.equal(failed.disposition, 'failed'); + assert.match(failed.reason, /X-Vercel-IP-Country-Region/); + + const documented = await verifyClaim({ + ...claim, + rec: { + ...rec, + fix: "Return `Cache-Control: public, s-maxage=300` and `Vary: X-Vercel-IP-Country-Region`.", + }, + }); + assert.equal(documented.disposition, 'verified'); +}); + +test('extractClaims + verifyClaim: postal-code Vary headers are unsafe', async () => { + const rec = { + candidateRef: 'uncached_route:/api/local', + what: 'Cache postal-code-specific offers.', + fix: "Return `Cache-Control: public, s-maxage=300` and `Vary: X-Vercel-IP-Postal-Code`.", + affectedFiles: ['app/api/local/route.ts'], + }; + const cardinality = extractClaims(rec, { framework: 'next', version: '15.4.10' }) + .find((c) => c.type === 'cache_vary_cardinality_safe'); + assert.ok(cardinality); + + const failed = await verifyClaim(cardinality); + assert.equal(failed.disposition, 'failed'); + assert.match(failed.reason, /Postal-Code/); +}); + test('extractClaims + verifyClaim: cache header values cannot contain empty directives', async () => { const rec = { candidateRef: 'cache_header_gap:/api/docs-og', @@ -539,6 +618,38 @@ test('extractClaims + verifyClaim: Cache Components rejects overbroad Route Hand assert.match(failed.reason, /Route Segment Config still has Route Handler options/); }); +test('extractClaims + verifyClaim: route-level revalidate is blocked by dynamic parent layout', async () => { + const repoRoot = await mkdtemp(join(tmpdir(), 'vo-revalidate-dynamic-parent-')); + await mkdir(join(repoRoot, 'apps/site/app/(landing)/prices'), { recursive: true }); + await writeFile( + join(repoRoot, 'apps/site/app/(landing)/layout.tsx'), + "import { withAuth } from '@/lib/auth';\nexport default async function Layout({ children }) { await withAuth(); return children; }\n" + ); + await writeFile( + join(repoRoot, 'apps/site/app/(landing)/prices/page.tsx'), + "export default function Prices() { return <main>Prices</main>; }\n" + ); + + const rec = { + candidateRef: 'uncached_route:/prices', + what: 'Add route-level revalidate to /prices.', + fix: 'Add `export const revalidate = 60` to `app/(landing)/prices/page.tsx`.', + affectedFiles: ['app/(landing)/prices/page.tsx'], + }; + const claim = extractClaims(rec, { + repoRoot, + projectRootDirectory: 'apps/site', + framework: 'next', + version: '15.4.10', + }).find((c) => c.type === 'next_route_revalidate_static_prereq'); + assert.ok(claim); + + const failed = await verifyClaim(claim); + assert.equal(failed.disposition, 'failed'); + assert.match(failed.reason, /withAuth/); + assert.match(failed.reason, /next build/); +}); + test('extractClaims + verifyClaim: cacheTag invalidation claims need matching revalidateTag/updateTag evidence', async () => { const repoRoot = await mkdtemp(join(tmpdir(), 'vo-tags-')); await mkdir(join(repoRoot, 'apps/site'), { recursive: true }); diff --git a/packages/vercel-optimize-tests/test/project-facts.test.mjs b/packages/vercel-optimize-tests/test/project-facts.test.mjs index ea283a5..32f9741 100644 --- a/packages/vercel-optimize-tests/test/project-facts.test.mjs +++ b/packages/vercel-optimize-tests/test/project-facts.test.mjs @@ -33,6 +33,7 @@ test('renderReport: surfaces Fluid Compute enabled from project.defaultResourceC assert.match(md, /In-function concurrency is enabled/); assert.match(md, /## Configuration notes/); assert.match(md, /memory tier: Standard/); + assert.match(md, /CPU-bound, or latency-sensitive route evidence/); assert.match(md, /fra1, iad1/); }); diff --git a/packages/vercel-optimize-tests/test/render-report.test.mjs b/packages/vercel-optimize-tests/test/render-report.test.mjs index c0beb66..faae7b2 100644 --- a/packages/vercel-optimize-tests/test/render-report.test.mjs +++ b/packages/vercel-optimize-tests/test/render-report.test.mjs @@ -35,7 +35,7 @@ const baseSignals = { }, }; -async function renderCli(recsRaw, extraArgs = [], gateRaw = { toLaunch: [], platform: [], gated: [] }) { +async function renderCli(recsRaw, extraArgs = [], gateRaw = { toLaunch: [], platform: [], gated: [] }, signalsRaw = baseSignals) { const root = await mkdtemp(join(tmpdir(), 'vo-render-report-')); try { const recsPath = join(root, 'recs.json'); @@ -44,7 +44,7 @@ async function renderCli(recsRaw, extraArgs = [], gateRaw = { toLaunch: [], plat await Promise.all([ writeFile(recsPath, JSON.stringify(recsRaw), 'utf-8'), writeFile(gatePath, JSON.stringify(gateRaw), 'utf-8'), - writeFile(signalsPath, JSON.stringify(baseSignals), 'utf-8'), + writeFile(signalsPath, JSON.stringify(signalsRaw), 'utf-8'), ]); return await exec('node', [RENDER_SCRIPT, recsPath, gatePath, signalsPath, '--project', 'x', '--no-timestamp', ...extraArgs]); } finally { @@ -731,6 +731,81 @@ test('renderReport: keeps Usage column when at least one row has a real unit', ( assert.ok(md.includes('| Service | Usage | Billed cost |')); }); +test('renderReport: omits zero-cost usage service rows', () => { + const md = renderReport({ + recommendations: [], + gated: [], + signals: { + ...baseSignals, + usage: { + ...baseSignals.usage, + services: [ + { name: 'Function Duration', usage: '1.20M GB-hr', billedCost: 142.50 }, + { name: 'Edge Requests', billedCost: 0 }, + { name: 'Image Optimization Cache Reads', billedCost: 0 }, + ], + totals: { billedCost: 142.50 }, + }, + }, + opts: { projectName: 'x' }, + }); + assert.ok(md.includes('| Function Duration | 1.20M GB-hr | $142.50 |')); + assert.ok(md.includes('_2 zero-cost service rows were omitted._')); + assert.ok(!md.includes('Edge Requests')); + assert.ok(!md.includes('Image Optimization Cache Reads')); +}); + +test('renderReport: replaces all-zero usage payload with concise note', () => { + const md = renderReport({ + recommendations: [], + gated: [], + signals: { + ...baseSignals, + usageScope: 'team', + usage: { + ...baseSignals.usage, + services: [ + { name: 'Fast Origin Transfer', billedCost: 0 }, + { name: 'Function Invocations', billedCost: 0 }, + { name: 'Image Optimization Transformation', billedCost: 0 }, + ], + totals: { billedCost: 0 }, + }, + }, + opts: { projectName: 'x' }, + }); + assert.ok(md.includes('every reported service cost was $0.00 for this window')); + assert.ok(md.includes('team-wide billing payload')); + assert.ok(!md.includes('| Service | Billed cost |')); + assert.ok(!md.includes('Fast Origin Transfer')); +}); + +test('renderReport: shows effective cost when included credits zero out billed cost', () => { + const md = renderReport({ + recommendations: [], + gated: [], + signals: { + ...baseSignals, + usageScope: 'project', + usage: { + ...baseSignals.usage, + services: [ + { name: 'Fast Origin Transfer', pricingUnit: 'USD', effectiveCost: 0.00056132736, billedCost: 2.0000000000194732e-10 }, + { name: 'Function Duration', pricingUnit: 'USD', effectiveCost: 0.08251739315, billedCost: 1.4999999853326784e-10 }, + { name: 'Edge Requests', pricingUnit: 'USD', effectiveCost: 0, billedCost: 0 }, + ], + totals: { billedCost: 4.4999999854348125e-10, effectiveCost: 0.08307872051 }, + }, + }, + opts: { projectName: 'x' }, + }); + assert.ok(md.includes('Net billed cost is $0.00 after included credits or allotments')); + assert.ok(md.includes('| Service | Effective cost |')); + assert.ok(md.includes('| Function Duration | $0.08 |')); + assert.ok(md.includes('**Total effective cost: $0.08**')); + assert.ok(!md.includes('| Service | Billed cost |')); +}); + test('renderReport: cost-header labels team-wide vs project scope', () => { const team = renderReport({ recommendations: [], gated: [], signals: { ...baseSignals, usageScope: 'team' }, opts: { projectName: 'x' } }); assert.ok(team.includes('## Cost breakdown (team-wide — `vercel usage` has no per-project filter)')); @@ -967,6 +1042,43 @@ test('renderReport: observation evidence expands raw metric shorthand', () => { assert.doesNotMatch(md, /(?:^|[,\s])p95=/); }); +test('renderReport: observation evidence expands dotted metric shorthand', () => { + const md = renderReport({ + recommendations: [], + gated: [], + observations: [{ + candidateRef: 'slow_route:/docs', + summary: 'cpu.p95=120ms vs latency.p95=900ms and ttfb.p95=880ms', + evidence: 'deepDive.cpu.p95=120ms; deepDive.latency.p95=900ms', + suggestedAction: 'Inspect runtime traces before recommending a code change.', + kind: 'other', + }], + signals: baseSignals, + }); + assert.match(md, /95th percentile CPU time: 120ms/); + assert.match(md, /95th percentile latency: 900ms/); + assert.match(md, /95th percentile TTFB: 880ms/); + assert.doesNotMatch(md, /\b(?:cpu|latency|ttfb)\.p95=/); +}); + +test('renderReport: direct renderer holds back unsafe observations', () => { + const md = renderReport({ + recommendations: [], + gated: [], + observations: [{ + candidateRef: 'platform_bot_protection:<account>', + summary: 'Bot traffic dominates bandwidth.', + evidence: 'browser_impersonation traffic is high.', + suggestedAction: 'Enable WAF managed bot rules.', + kind: 'config_gap', + }], + signals: baseSignals, + }); + assert.ok(!md.includes('Enable WAF managed bot rules')); + assert.match(md, /## Needs more evidence/); + assert.match(md, /staged safe-rollout plan/); +}); + test('renderReport: throws when observation summary is missing', () => { assert.throws( () => renderReport({ @@ -1106,6 +1218,143 @@ test('render-report CLI: holds back unsupported framework-causal observations', assert.match(stdout, /framework-specific cause claim that verification could not support/); }); +test('render-report CLI: holds back implementation-grade observation actions', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'platform_bot_protection:<account>', + summary: 'Bot traffic accounts for 92% of edge bandwidth.', + evidence: 'browser_impersonation dominates Fast Data Transfer.', + suggestedAction: 'Enable WAF managed bot rules to challenge or deny browser_impersonation traffic.', + kind: 'config_gap', + }], + }); + assert.ok(!stdout.includes('Enable WAF managed bot rules'), 'implementation-grade observation action should not ship'); + assert.match(stdout, /verified platform recommendation/); +}); + +test('render-report CLI: counts held-back observations in coverage', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1, abstentions: 0 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'slow_route:/api/ribbon', + summary: 'The route waits on shared origin data.', + evidence: 'latency.p95=900ms', + suggestedAction: 'wrap the shared fetch helper before returning the route response.', + kind: 'upstream_dependency', + }], + }, [], { + toLaunch: [{ kind: 'slow_route', route: '/api/ribbon' }], + platform: [], + gated: [], + }); + assert.match(stdout, /0 recommendations ready\s+·\s+1 need more evidence/); + assert.match(stdout, /## Needs more evidence/); + assert.ok(!stdout.includes('wrap the shared fetch helper'), 'lower-case implementation action should not ship as an observation'); +}); + +test('render-report CLI: holds back alternate implementation verbs in observations', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'slow_route:/api/search', + summary: 'The route has independent upstream calls.', + evidence: 'cpu.p95=100ms vs latency.p95=1200ms', + suggestedAction: 'use Promise.all and raise the TTL for the shared search lookup.', + kind: 'upstream_dependency', + }], + }); + assert.ok(!stdout.includes('use Promise.all'), 'alternate implementation action should not ship as an observation'); + assert.match(stdout, /ready-to-apply recommendation evidence bar/); +}); + +test('render-report CLI: holds back root-cause claims even when action asks to inspect logs', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'route_errors:/api/checkout', + summary: '500 error rate is caused by an upstream checkout gateway response.', + evidence: 'http_status=500 count=92; route.ts:57 parses the upstream response.', + suggestedAction: 'Inspect logs before recommending a code change.', + kind: 'error_storm', + }], + }); + assert.ok(!stdout.includes('caused by an upstream checkout gateway response')); + assert.match(stdout, /runtime root-cause claim/); +}); + +test('render-report CLI: holds back stale Next cache APIs in observations', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'slow_route:/api/ribbon', + summary: 'The route fetches shared Sanity data on every request.', + evidence: 'next@16 app route with repeated shared fetches.', + suggestedAction: 'Wrap the fetch in unstable_cache with a 60s TTL.', + kind: 'upstream_dependency', + }], + }, [], { toLaunch: [], platform: [], gated: [] }, { + ...baseSignals, + stack: { ...baseSignals.stack, frameworkVersion: '16.2.6', cacheComponents: true }, + }); + assert.ok(!stdout.includes('unstable_cache'), 'Next 16 stale cache API should not ship in observations'); + assert.match(stdout, /framework-version evidence/); +}); + +test('render-report CLI: does not hold back unstable_cache observations for Next 15', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'slow_route:/api/ribbon', + summary: 'The route uses unstable_cache on a shared lookup.', + evidence: 'Next 15 route with runtime-cache evidence.', + suggestedAction: 'Inspect cache key cardinality and freshness before recommending a code change.', + kind: 'upstream_dependency', + }], + }, [], { toLaunch: [], platform: [], gated: [] }, { + ...baseSignals, + stack: { ...baseSignals.stack, frameworkVersion: '15.4.10' }, + }); + assert.match(stdout, /The route uses unstable_cache/); + assert.doesNotMatch(stdout, /cache API that does not match/); +}); + +test('render-report CLI: holds back WAF bot-category custom-rule claims', async () => { + const { stdout } = await renderCli({ + schemaVersion: '1.0', + summary: { totalRecs: 0, observations: 1 }, + recsGraded: [], + abstentions: [], + observations: [{ + candidateRef: 'platform_bot_protection:<account>', + summary: 'browser_impersonation traffic dominates bandwidth.', + evidence: 'bot_category=browser_impersonation accounts for most Fast Data Transfer.', + suggestedAction: 'Add a WAF custom rule targeting bot_category=browser_impersonation with a Challenge action.', + kind: 'config_gap', + }], + }); + assert.ok(!stdout.includes('bot_category=browser_impersonation'), 'unsupported WAF condition should not ship'); + assert.match(stdout, /WAF-rule condition/); +}); + test('render-report CLI: suppresses same-route same-family observations covered by a ready rec', async () => { const readyCacheRec = { ...sampleRec, diff --git a/packages/vercel-optimize-tests/test/sanitizers.test.mjs b/packages/vercel-optimize-tests/test/sanitizers.test.mjs index f4f81af..25a4f7d 100644 --- a/packages/vercel-optimize-tests/test/sanitizers.test.mjs +++ b/packages/vercel-optimize-tests/test/sanitizers.test.mjs @@ -330,7 +330,8 @@ test('bot-protection-certainty: softens unsupported rule-state claims and adds r assert.match(rec.why, /likely contributor/); assert.match(rec.why, /false-positive risk monitored/); assert.doesNotMatch(rec.why, /without false-positive risk/); - assert.match(rec.fix, /Log -> Challenge -> Deny/); + assert.match(rec.fix, /starts in Log mode/); + assert.match(rec.fix, /appropriate Challenge or Deny action/); assert.ok(r.tags.includes('bot-protection-certainty:why')); assert.ok(r.tags.includes('bot-protection-certainty:staged-rollout')); assert.equal(r.needsReview, true); diff --git a/skills/vercel-optimize/SKILL.md b/skills/vercel-optimize/SKILL.md index 492652d..c8a6d6d 100644 --- a/skills/vercel-optimize/SKILL.md +++ b/skills/vercel-optimize/SKILL.md @@ -2,7 +2,7 @@ name: vercel-optimize description: "Use for Vercel cost and performance optimization on deployed projects, especially Next.js, SvelteKit, Nuxt, and limited Astro apps. Collect Vercel metrics, usage, project config, and code scan results first; investigate only metric-backed candidates; produce ranked recommendations grounded in verified files and version-aware Vercel/framework docs. Trigger for Vercel bill reduction, slow or expensive routes, caching opportunities, Function Invocations, Build Minutes, Fast Data Transfer, Core Web Vitals, Bot Management, Fluid compute, or cost breakdown requests." metadata: - version: "1.1.0" + version: "1.2.0" --- # Vercel Optimize diff --git a/skills/vercel-optimize/lib/display-labels.mjs b/skills/vercel-optimize/lib/display-labels.mjs index 9e1c4d8..e8f90f8 100644 --- a/skills/vercel-optimize/lib/display-labels.mjs +++ b/skills/vercel-optimize/lib/display-labels.mjs @@ -59,6 +59,11 @@ const REQUEST_COUNT_KINDS = new Set([ const PUBLIC_ASSIGNMENT_LABELS = new Map([ ...SIGNAL_LABELS, ['deepDive.latency.p95', 'deepDive latency p95'], + ['deepDive.cpu.p95', 'deepDive CPU p95'], + ['deepDive.ttfb.p95', 'deepDive TTFB p95'], + ['cpu.p95', 'CPU p95'], + ['latency.p95', 'latency p95'], + ['ttfb.p95', 'TTFB p95'], ['cache_result', 'cache result'], ['http_status', 'HTTP status'], ['error_code', 'error code'], diff --git a/skills/vercel-optimize/lib/extract-claims.mjs b/skills/vercel-optimize/lib/extract-claims.mjs index a5d9122..6ddecfd 100644 --- a/skills/vercel-optimize/lib/extract-claims.mjs +++ b/skills/vercel-optimize/lib/extract-claims.mjs @@ -67,6 +67,13 @@ export function extractClaims(rec, ctx = {}) { projectRootDirectory, sourceField: 'cache-safety', }); + if (mentionsVaryHeader(rec)) { + claims.push({ + type: 'cache_vary_cardinality_safe', + rec, + sourceField: 'cache-vary-cardinality', + }); + } claims.push({ type: 'cache_rec_not_error_dominated_or_acknowledged', rec, @@ -207,6 +214,19 @@ export function extractClaims(rec, ctx = {}) { }); } + if (mentionsRouteLevelRevalidate(rec)) { + claims.push({ + type: 'next_route_revalidate_static_prereq', + rec, + framework, + frameworkVersion, + cacheComponents, + repoRoot, + projectRootDirectory, + sourceField: 'next-route-revalidate-static-prereq', + }); + } + if (mentionsExistingCacheTagInvalidation(rec)) { claims.push({ type: 'next_cache_tag_invalidation_supported', @@ -335,6 +355,10 @@ function recommendsSharedCache(rec) { return /\b(?:s-maxage|CDN-Cache-Control|Vercel-CDN-Cache-Control|Cache-Control)\b/i.test(haystack); } +function mentionsVaryHeader(rec) { + return /\bVary\b/i.test(recText(rec)); +} + function mentionsNextCachedNotFound(rec) { const haystack = recText(rec); return /\bnotFound\b/.test(haystack) && @@ -419,6 +443,13 @@ function mentionsNextCacheComponentsRouteSegmentConfig(rec) { /\b(?:revalidate|dynamic|fetchCache)\b[^.\n]{0,80}\broute segment (?:config|export)\b/i.test(haystack); } +function mentionsRouteLevelRevalidate(rec) { + const haystack = recText(rec); + return /\bexport\s+const\s+revalidate\b/.test(haystack) || + /\broute[- ]level\s+revalidate\b/i.test(haystack) || + /\brevalidate\s*(?:=|:)\s*\d+\b[^.\n]{0,120}\b(?:page|layout|route segment|segment export)\b/i.test(haystack); +} + function mentionsExistingCacheTagInvalidation(rec) { const haystack = recText(rec); if (!/\bcacheTag\s*\(/.test(haystack)) return false; diff --git a/skills/vercel-optimize/lib/investigation-brief.mjs b/skills/vercel-optimize/lib/investigation-brief.mjs index 1b9ef1e..d526723 100644 --- a/skills/vercel-optimize/lib/investigation-brief.mjs +++ b/skills/vercel-optimize/lib/investigation-brief.mjs @@ -595,14 +595,14 @@ function cachePolicyGuidance(kind, stack = {}) { const framework = stack.framework ?? 'unknown'; const cacheComponents = stack.cacheComponents === true; const hints = [ - 'Whole public GET response: recommend `Cache-Control` / `CDN-Cache-Control` with `s-maxage` and `stale-while-revalidate`; name the TTL/freshness window and required `Vary` headers.', + 'Whole public GET response: recommend `Cache-Control` / `CDN-Cache-Control` with `s-maxage` and `stale-while-revalidate`; name the TTL/freshness window and required `Vary` headers. Avoid high-cardinality `Vary` headers such as `X-Vercel-IP-Latitude` or `X-Vercel-IP-Longitude`; use coarser geography only when the product can tolerate it.', 'Fallback, 404, auth, preview, webhook, mutation, and per-user branches: keep them uncached or short-lived while caching only the safe success branch.', ]; if (framework === 'next') { if (cacheComponents) { hints.push('Next.js with Cache Components: for reusable data inside the render path, prefer `use cache` / `use cache: remote` plus `cacheLife()` and `cacheTag()` when invalidation evidence exists.'); } else { - hints.push('Next.js data fetch path: use `fetch(..., { next: { revalidate: seconds } })` or route-level `revalidate` only when it matches the project version and route semantics.'); + hints.push('Next.js data fetch path: use `fetch(..., { next: { revalidate: seconds } })` or route-level `revalidate` only when it matches the project version and route semantics. Before recommending route-level `export const revalidate`, inspect the page/layout route chain for `cookies()`, `headers()`, `draftMode()`, `connection()`, and auth helpers; if any parent layout is request-time dynamic, require `next build` or manifest proof that the route is still ISR/static, otherwise abstain.'); } } hints.push('Reusable server data where whole-response CDN caching is unsafe: recommend Runtime Cache only when the same result is reused across requests and the freshness/invalidation story is explicit.'); diff --git a/skills/vercel-optimize/lib/observation-safety.mjs b/skills/vercel-optimize/lib/observation-safety.mjs new file mode 100644 index 0000000..1a628cd --- /dev/null +++ b/skills/vercel-optimize/lib/observation-safety.mjs @@ -0,0 +1,174 @@ +export function splitCustomerSafeObservations(observations, abstentions = [], signals = {}) { + const safe = []; + const heldBack = []; + for (const observation of Array.isArray(observations) ? observations : []) { + const unsafeReason = unsupportedObservationReason(observation, abstentions, signals); + if (unsafeReason) { + heldBack.push({ + candidateRef: observation?.candidateRef ?? null, + reason: unsafeReason, + needsEvidence: true, + }); + } else { + safe.push(observation); + } + } + return { observations: safe, heldBackObservations: heldBack }; +} + +function unsupportedObservationReason(observation, abstentions = [], signals = {}) { + if (contradictsNoChangeReason(observation, abstentions)) { + return 'This observation repeated an action that another investigation rejected. Re-run with a single scoped candidate before applying it.'; + } + if (hasUnsupportedWafBotCategoryClaim(observation)) { + return 'This observation described an Observability bot category as a WAF-rule condition without supported rule evidence. Re-run with documented WAF condition evidence before applying it.'; + } + if (hasUnsafeBotProtectionObservation(observation)) { + return 'This observation recommended Bot Protection or WAF changes without a staged safe-rollout plan and allowlist review. Promote it to a verified platform recommendation before applying it.'; + } + if (hasStaleNextCacheApiObservation(observation, signals)) { + return 'This observation used a cache API that does not match the detected framework-version evidence. Re-run with the current Next.js cache evidence before applying it.'; + } + if (hasUnsupportedFrameworkCausalClaim(observation)) { + return 'This observation made a framework-specific cause claim that verification could not support. Re-run with runtime logs or official framework evidence before applying it.'; + } + if (hasUnsupportedStaticGenerationClaim(observation)) { + return 'This observation made a static-generation behavior claim that verification could not support. Re-run with route-manifest or runtime evidence before applying it.'; + } + if (hasUnsupportedSourceAbsenceClaim(observation)) { + return 'This observation made a source-file absence claim that verification could not support. Re-run with a file-existence check or runtime logs before applying it.'; + } + if (hasUnsupportedCacheLifeCdnClaim(observation)) { + return 'This observation depended on an unsupported cacheLife-to-CDN claim. Re-run with production header evidence before applying it.'; + } + if (hasUnsupportedRuntimeRootCauseClaim(observation)) { + return 'This observation made a runtime root-cause claim that needs log, stack, or upstream response evidence before it can ship.'; + } + if (hasImplementationGradeObservationAction(observation)) { + return 'This observation described an implementation change that needs the ready-to-apply recommendation evidence bar before it can ship.'; + } + return null; +} + +function observationText(observation) { + return [ + observation?.summary, + observation?.evidence, + observation?.suggestedAction, + ].filter(Boolean).join(' '); +} + +function evidenceText(observation) { + return [ + observation?.summary, + observation?.evidence, + ].filter(Boolean).join(' '); +} + +function hasUnsupportedWafBotCategoryClaim(observation) { + const text = observationText(observation); + if (!/\bWAF\b/i.test(text)) return false; + return /\bbot_category\s*=/i.test(text) || + /\btarget(?:ing)?\s+(?:browser_impersonation|automated_browser|ecommerce|monitor)\b/i.test(text); +} + +function hasUnsafeBotProtectionObservation(observation) { + const text = observationText(observation); + if (!/\b(?:Bot Protection|BotID|bot_filter|WAF|managed bot rules?)\b/i.test(text)) return false; + const recommendsAction = /\b(?:enable|add|create|configure|challenge|deny|block|rate-limit|target)\b/i.test(String(observation?.suggestedAction ?? '')); + if (!recommendsAction) return false; + const hasSafeRollout = /\b(?:staged|log mode|log action|dry run)\b/i.test(text); + const hasAllowlist = /\ballowlist|exclusions?\b/i.test(text); + return !(hasSafeRollout && hasAllowlist); +} + +function hasStaleNextCacheApiObservation(observation, signals = {}) { + const text = observationText(observation); + if (!/\bunstable_cache\b/.test(text)) return false; + if (signals?.stack?.framework !== 'next') return false; + const major = parseInt(String(signals?.stack?.frameworkVersion ?? '').match(/\d+/)?.[0] ?? '', 10); + return Number.isFinite(major) && major >= 16; +} + +function hasImplementationGradeObservationAction(observation) { + const action = String(observation?.suggestedAction ?? ''); + if (action.trim() === '') return false; + if (/\b(?:use cache:\s*remote|unstable_cache|Cache-Control|s-maxage|cacheLife|export const revalidate|checkBotId|BotID)\b/i.test(action)) return true; + return /\b(?:enable|add|wrap|apply|move|parallelize|set|create|configure|deny|challenge|block|fix|replace|refactor|rewrite|upgrade|downgrade)\b/i.test(action) || + /\bcache\s+(?:the|this|that|shared|public|origin|response|route|data|lookup|fetch|helper)\b/i.test(action) || + /\bturn\s+(?:on|off)\b/i.test(action) || + /\bswitch\s+(?:to|from|the|this)\b/i.test(action) || + /\buse\s+Promise\.all\b/i.test(action) || + /\b(?:raise|lower|increase|decrease)\s+(?:the\s+)?(?:TTL|timeout|memory|CPU|cache|cache lifetime|duration)\b/i.test(action); +} + +function contradictsNoChangeReason(observation, abstentions) { + const target = candidateTarget(observation?.candidateRef); + if (!target) return false; + const lowerObservationText = observationText(observation).toLowerCase(); + const relevantReasons = abstentions + .filter((a) => candidateTarget(a?.candidateRef) === target) + .map((a) => String(a?.reason ?? '').toLowerCase()); + if (relevantReasons.length === 0) return false; + + if (/\bparalleliz(?:e|ing)\b/.test(lowerObservationText) && + /\bgetsession\b/.test(lowerObservationText) && + relevantReasons.some((reason) => /\bgetsession\b/.test(reason) && /\b(?:gates?|redirect|auth-preserving|blocked)\b/.test(reason))) { + return true; + } + return false; +} + +function candidateTarget(ref) { + if (typeof ref !== 'string') return null; + const idx = ref.indexOf(':'); + if (idx === -1) return null; + return ref.slice(idx + 1); +} + +function hasUnsupportedFrameworkCausalClaim(observation) { + const text = observationText(observation).toLowerCase(); + if (!text.includes('notfound') || !text.includes('use cache')) return false; + return ( + /known next\.js cache components edge case/.test(text) || + /next\.js\s+\d+(?:\.\d+)?[^.]{0,120}treats[^.]{0,120}dynamic api/.test(text) || + /can surface as 5xx/.test(text) || + /surface as 500/.test(text) || + /instead of throwing inside (?:the )?cache/.test(text) || + /cache boundary/.test(text) + ); +} + +function hasUnsupportedStaticGenerationClaim(observation) { + const text = observationText(observation).toLowerCase(); + if (!/\bgeneratestaticparams\b/.test(text)) return false; + return /\b(?:returns?\s*(?:an\s+)?empty|\[\])\b[^.\n]{0,240}\b(?:every request|on[- ]demand|no params? (?:are )?prebuilt|populate generatestaticparams|served from (?:the )?cdn|hit bucket|cachebreakdown)\b/i.test(text) || + /\b(?:every request|on[- ]demand|no params? (?:are )?prebuilt|populate generatestaticparams|served from (?:the )?cdn|hit bucket|cachebreakdown)\b[^.\n]{0,240}\b(?:returns?\s*(?:an\s+)?empty|\[\])\b/i.test(text) || + /\bdynamic\s*=\s*['"`]error['"`]\b[^.\n]{0,240}\b(?:generatestaticparams|dynamicparams|every request|on[- ]demand)\b/i.test(text); +} + +function hasUnsupportedSourceAbsenceClaim(observation) { + const ref = String(observation?.candidateRef ?? ''); + if (!ref.startsWith('route_errors:')) return false; + const text = observationText(observation).toLowerCase(); + return /\b(?:enoent|no\s+(?:matching|corresponding)\s+(?:mdx|file|post)|missing\s+(?:mdx|file|post)|does\s+not\s+exist|not\s+found\s+on\s+disk)\b/.test(text); +} + +function hasUnsupportedCacheLifeCdnClaim(observation) { + return hasUnsupportedCacheLifeCdnText(observationText(observation)); +} + +export function hasUnsupportedCacheLifeCdnText(text) { + if (typeof text !== 'string' || !/\bcacheLife\b/i.test(text)) return false; + if (/\btoLaunch-\d+\b/i.test(text)) return true; + return /\bcacheLife\b[^.\n]{0,240}\b(?:Cache-Control|s-maxage|CDN|edge cache|cache breakdown|x-vercel-cache|HIT|MISS|function (?:still )?runs per request|every request invokes the function|canonical|toLaunch-\d+)\b/i.test(text) || + /\b(?:Cache-Control|s-maxage|CDN|edge cache|cache breakdown|x-vercel-cache|HIT|MISS|function (?:still )?runs per request|every request invokes the function|canonical|toLaunch-\d+)\b[^.\n]{0,240}\bcacheLife\b/i.test(text) || + /\b(?:no|never|without|missing)\s+cacheLife\b[^.\n]{0,240}\b(?:no|not|never|0%|every|per request|function)\b[^.\n]{0,120}\b(?:cache|cached|hit|runs?|invoke)/i.test(text); +} + +function hasUnsupportedRuntimeRootCauseClaim(observation) { + const text = observationText(observation); + if (!/\b(?:caused by|root cause|responsible for failures|would produce)\b/i.test(text)) return false; + if (!/\b(?:5xx|500|error|failures?)\b/i.test(text)) return false; + return !/\b(?:logs?\s+(?:show|confirm|include|contain)|stack\s+(?:shows|trace|evidence)|trace\s+(?:shows|confirms)|exception\s+(?:shows|confirms)|response body\s+(?:shows|confirms)|runtime evidence)\b/i.test(evidenceText(observation)); +} diff --git a/skills/vercel-optimize/lib/project-facts.mjs b/skills/vercel-optimize/lib/project-facts.mjs index 0f2442d..279635a 100644 --- a/skills/vercel-optimize/lib/project-facts.mjs +++ b/skills/vercel-optimize/lib/project-facts.mjs @@ -39,8 +39,8 @@ export function deriveProjectFacts(signals) { if (cfg.functionDefaultMemoryType === 'standard') { out.push({ id: 'memory_standard', - strength: 'Function memory tier: Standard (2GB) — the cost-efficient default; upgrade to Performance (4GB) only if you have evidence of memory-bound routes.', - briefLine: 'Function memory tier is Standard (2GB), the cost-efficient default. Recommending an upgrade to Performance (4GB) requires memory-saturation evidence.', + strength: 'Function memory tier: Standard (2GB) — the cost-efficient default; upgrade to Performance (4GB) only with memory, CPU-bound, or latency-sensitive route evidence.', + briefLine: 'Function memory tier is Standard (2GB), the cost-efficient default. Recommending an upgrade to Performance (4GB) requires memory, CPU-bound, or latency-sensitive route evidence.', contradictPhrases: [], }); } else if (cfg.functionDefaultMemoryType === 'performance') { @@ -68,8 +68,8 @@ export function deriveProjectFacts(signals) { if (cfg.functionZeroConfigFailover === true) { out.push({ id: 'zero_config_failover', - strength: 'Multi-region zero-config failover is enabled.', - briefLine: 'Zero-config multi-region failover is ENABLED. Do not recommend enabling it.', + strength: 'Function failover is enabled in project config.', + briefLine: 'Function failover is ENABLED in project config. Do not recommend enabling it.', contradictPhrases: [ 'enable zero-config failover', 'enable multi-region failover', diff --git a/skills/vercel-optimize/lib/render-report.mjs b/skills/vercel-optimize/lib/render-report.mjs index 32132cc..864fd0b 100644 --- a/skills/vercel-optimize/lib/render-report.mjs +++ b/skills/vercel-optimize/lib/render-report.mjs @@ -7,11 +7,15 @@ import { canonicalizeRoute } from './route-normalize.mjs'; import { computeCostCoverage, renderCostCoverageMarkdown } from './cost-coverage.mjs'; import { gates as registeredGates } from './gates/index.mjs'; import { formatCandidateLabel, formatKind, formatPublicText, formatRoute, formatSignal } from './display-labels.mjs'; +import { splitCustomerSafeObservations } from './observation-safety.mjs'; const PLATFORM_CAP = 3; const GATED_TARGET_PREVIEW = 5; export function renderReport({ recommendations = [], gated = [], abstentions = [], observations = [], signals = {}, candidates = [], opts = {} } = {}) { + const safety = splitCustomerSafeObservations(observations, abstentions, signals); + observations = safety.observations; + abstentions = [...abstentions, ...safety.heldBackObservations]; assertValidObservations(observations); const projectName = opts.projectName ?? signals.project?.name ?? '<project>'; @@ -291,7 +295,9 @@ function renderCoverageLine(candidates, recommendations, signals, opts = {}) { } const recCount = (recommendations ?? []).filter((r) => !r.abstain && !isPlatformScope(r)).length; parts.push(`${recCount} recommendation${recCount === 1 ? '' : 's'} ready`); - const rawHeldBackCount = Number.isInteger(opts.heldBackCount) ? opts.heldBackCount : 0; + const rawHeldBackCount = Number.isInteger(opts.heldBackCount) + ? opts.heldBackCount + : (Array.isArray(opts.abstentions) ? opts.abstentions.filter((a) => a?.needsEvidence === true).length : 0); const heldBackCount = Math.min(rawHeldBackCount, Math.max(0, launched.length - recCount)); if (heldBackCount > 0) { parts.push(`${heldBackCount} need more evidence`); @@ -372,32 +378,43 @@ function renderCostBreakdown(usage, signals) { const lines = []; const services = Array.isArray(usage?.services) ? usage.services : null; if (services && services.length > 0) { - const rows = services.slice().sort((a, b) => (b.billedCost ?? 0) - (a.billedCost ?? 0)); - // Drop Usage column when every cell is "(unspecified)" — happens when CLI emits pricingUnit=USD. - const usageCells = rows.map((s) => formatUsage(s)); - const hasRealUsage = usageCells.some((c) => c !== '(unspecified)'); - if (hasRealUsage) { - lines.push('| Service | Usage | Billed cost |'); - lines.push('|---|---|---|'); - for (let i = 0; i < rows.length; i++) { - const s = rows[i]; - const cost = typeof s.billedCost === 'number' ? `$${s.billedCost.toFixed(2)}` : '(n/a)'; - lines.push(`| ${escape(s.name ?? '(unnamed)')} | ${escape(usageCells[i])} | ${cost} |`); - } - } else { - lines.push('| Service | Billed cost |'); - lines.push('|---|---|'); - for (const s of rows) { - const cost = typeof s.billedCost === 'number' ? `$${s.billedCost.toFixed(2)}` : '(n/a)'; - lines.push(`| ${escape(s.name ?? '(unnamed)')} | ${cost} |`); - } + const chargedRows = services.filter((s) => { + const cost = serviceCost(s); + return cost === null || costRoundsToCents(cost) > 0; + }); + if (chargedRows.length > 0) { + return renderServiceCostRows(chargedRows, { + costLabel: 'Billed cost', + costOf: serviceCost, + omittedZeroRows: services.length - chargedRows.length, + total: usage.totals?.billedCost, + totalLabel: 'Total billed', + totalSuffix: ' _(precise observed cost; future-savings framing is magnitude, never precise)_', + }); } - const total = usage.totals?.billedCost; - if (typeof total === 'number') { + + const effectiveRows = services.filter((s) => costRoundsToCents(serviceEffectiveCost(s)) > 0); + if (effectiveRows.length > 0) { + lines.push('_Net billed cost is $0.00 after included credits or allotments. Showing effective usage cost so active cost drivers are still visible._'); lines.push(''); - lines.push(`**Total billed: $${total.toFixed(2)}** _(precise observed cost; future-savings framing is magnitude, never precise)_`); + return [ + ...lines, + ...renderServiceCostRows(effectiveRows, { + costLabel: 'Effective cost', + costOf: serviceEffectiveCost, + omittedZeroRows: services.length - effectiveRows.length, + total: usage.totals?.effectiveCost, + totalLabel: 'Total effective cost', + totalSuffix: ' _(usage cost before included-credit or allotment offsets)_', + }), + ]; + } + + if (chargedRows.length === 0) { + const scope = signals.usageScope === 'team' ? 'team-wide ' : ''; + lines.push(`_\`vercel usage\` returned a ${scope}billing payload, but every reported service cost was $0.00 for this window._`); + return lines; } - return lines; } // Fallback to o11y-derived ranking when usage payload missing. @@ -420,6 +437,58 @@ function renderCostBreakdown(usage, signals) { return lines; } +function renderServiceCostRows(services, { costLabel, costOf, omittedZeroRows = 0, total = null, totalLabel, totalSuffix = '' }) { + const lines = []; + const rows = services.slice().sort((a, b) => (costOf(b) ?? 0) - (costOf(a) ?? 0)); + // Drop Usage column when every cell is "(unspecified)" — happens when CLI emits pricingUnit=USD. + const usageCells = rows.map((s) => formatUsage(s)); + const hasRealUsage = usageCells.some((c) => c !== '(unspecified)'); + if (hasRealUsage) { + lines.push(`| Service | Usage | ${costLabel} |`); + lines.push('|---|---|---|'); + for (let i = 0; i < rows.length; i++) { + const s = rows[i]; + const costValue = costOf(s); + const cost = typeof costValue === 'number' ? `$${costValue.toFixed(2)}` : '(n/a)'; + lines.push(`| ${escape(s.name ?? '(unnamed)')} | ${escape(usageCells[i])} | ${cost} |`); + } + } else { + lines.push(`| Service | ${costLabel} |`); + lines.push('|---|---|'); + for (const s of rows) { + const costValue = costOf(s); + const cost = typeof costValue === 'number' ? `$${costValue.toFixed(2)}` : '(n/a)'; + lines.push(`| ${escape(s.name ?? '(unnamed)')} | ${cost} |`); + } + } + if (omittedZeroRows > 0) { + lines.push(''); + lines.push(`_${omittedZeroRows} zero-cost service ${omittedZeroRows === 1 ? 'row was' : 'rows were'} omitted._`); + } + if (typeof total === 'number') { + lines.push(''); + lines.push(`**${totalLabel}: $${total.toFixed(2)}**${totalSuffix}`); + } + return lines; +} + +function serviceCost(service) { + if (typeof service?.billedCost === 'number') return service.billedCost; + if (typeof service?.cost === 'number') return service.cost; + return null; +} + +function serviceEffectiveCost(service) { + if (typeof service?.effectiveCost === 'number') return service.effectiveCost; + if (typeof service?.pricingQuantity === 'number' && service?.pricingUnit === 'USD') return service.pricingQuantity; + return 0; +} + +function costRoundsToCents(cost) { + if (typeof cost !== 'number' || !Number.isFinite(cost)) return 0; + return Math.round(cost * 100) / 100; +} + function renderRecTable(recs, signals = {}) { if (recs.length === 0) return ['_(none)_']; const lines = []; diff --git a/skills/vercel-optimize/lib/sanitizers/bot-protection-certainty.mjs b/skills/vercel-optimize/lib/sanitizers/bot-protection-certainty.mjs index 51bb06b..5de9a71 100644 --- a/skills/vercel-optimize/lib/sanitizers/bot-protection-certainty.mjs +++ b/skills/vercel-optimize/lib/sanitizers/bot-protection-certainty.mjs @@ -27,8 +27,8 @@ export function apply(rec) { const text = STRING_FIELDS.map((field) => rec?.[field]).filter((s) => typeof s === 'string').join('\n'); if (/\b(?:Bot Protection|BotID|bot_filter|WAF)\b/i.test(text) && - !/\bLog\s*(?:->|\u2192)\s*Challenge\s*(?:->|\u2192)\s*Deny\b/i.test(text)) { - const caveat = ' Use a staged Log -> Challenge -> Deny rollout with allowlist/exclusions for known monitoring and partner clients.'; + !/\bstaged\b[\s\S]{0,80}\b(?:log|allowlist|exclusions?)\b/i.test(text)) { + const caveat = ' Use a staged rollout that starts in Log mode where available, then moves to the appropriate Challenge or Deny action only after allowlist/exclusion review for known monitoring and partner clients.'; if (typeof rec.fix === 'string') rec.fix += caveat; else rec.fix = caveat.trim(); tags.push('bot-protection-certainty:staged-rollout'); diff --git a/skills/vercel-optimize/lib/verify-claim.mjs b/skills/vercel-optimize/lib/verify-claim.mjs index 42d9e95..b8e68a9 100644 --- a/skills/vercel-optimize/lib/verify-claim.mjs +++ b/skills/vercel-optimize/lib/verify-claim.mjs @@ -26,6 +26,7 @@ export async function verifyClaim(claim) { case 'citation_in_library': return verifyCitationInLibrary(claim); case 'citation_applies_to_version': return verifyCitationAppliesToVersion(claim); case 'cache_vary_matches_dynamic_inputs': return verifyCacheVaryMatchesDynamicInputs(claim); + case 'cache_vary_cardinality_safe': return verifyCacheVaryCardinalitySafe(claim); case 'next_cached_not_found_causal_support': return verifyNextCachedNotFoundCausalSupport(claim); case 'next_stable_cache_api_for_version': return verifyNextStableCacheApiForVersion(claim); case 'next_runtime_cache_api_for_version': return verifyNextRuntimeCacheApiForVersion(claim); @@ -37,6 +38,7 @@ export async function verifyClaim(claim) { case 'image_response_headers_citation': return verifyImageResponseHeadersCitation(claim); case 'next_image_priority_api_for_version': return verifyNextImagePriorityApiForVersion(claim); case 'next_cache_components_route_segment_config': return verifyNextCacheComponentsRouteSegmentConfig(claim); + case 'next_route_revalidate_static_prereq': return verifyNextRouteRevalidateStaticPrereq(claim); case 'next_cache_tag_invalidation_supported': return verifyNextCacheTagInvalidationSupported(claim); case 'cache_rec_not_error_dominated_or_acknowledged': return verifyCacheRecNotErrorDominatedOrAcknowledged(claim); case 'cache_control_header_syntax': return verifyCacheControlHeaderSyntax(claim); @@ -200,7 +202,9 @@ async function verifyCacheVaryMatchesDynamicInputs({ rec, files, repoRoot = '.', for (const file of files) { try { const { content } = await readClaimFile({ file, repoRoot, projectRootDirectory }); - if (/\bgeolocation\s*\(/.test(content) || /\b\w+\.geo\??\./.test(content) || /['"]x-vercel-ip-country['"]/i.test(content)) { + if (/\bgeolocation\s*\(/.test(content) || + /\b\w+\.geo\??\./.test(content) || + /['"]x-vercel-ip-(?:country|country-region|city|latitude|longitude|postal-code|timezone)['"]/i.test(content)) { usesVercelGeo = true; break; } @@ -213,16 +217,32 @@ async function verifyCacheVaryMatchesDynamicInputs({ rec, files, repoRoot = '.', const text = [rec.what, rec.why, rec.fix, rec.currentBehavior, rec.desiredBehavior, rec.verify] .filter(Boolean) .join('\n'); - const hasCountryVary = hasHeaderValue(text, 'Vary', /(?:^|,\s*)X-Vercel-IP-Country(?:\s*,|$)/i); - if (hasCountryVary) { - return { disposition: 'verified', reason: 'cache rec varies by X-Vercel-IP-Country for geolocation-dependent output' }; + const hasCoarseGeoVary = hasHeaderValue(text, 'Vary', /(?:^|,\s*)X-Vercel-IP-(?:Country|Country-Region|City)(?:\s*,|$)/i); + if (hasCoarseGeoVary) { + return { disposition: 'verified', reason: 'cache rec varies by a coarse Vercel geolocation header for geolocation-dependent output' }; } return { disposition: 'failed', - reason: 'cache rec touches Vercel geolocation but does not vary by X-Vercel-IP-Country', + reason: 'cache rec touches Vercel geolocation but does not vary by a coarse Vercel geolocation header such as X-Vercel-IP-Country, X-Vercel-IP-Country-Region, or X-Vercel-IP-City', }; } +async function verifyCacheVaryCardinalitySafe({ rec }) { + if (!rec) return { disposition: 'unsupported', reason: 'cache_vary_cardinality_safe requires rec' }; + const text = recText(rec); + const varyValues = extractHeaderValues(text, 'Vary').join(', '); + if (!varyValues) { + return { disposition: 'verified', reason: 'no concrete Vary header value detected' }; + } + if (/\bX-Vercel-IP-(?:Latitude|Longitude|Postal-Code)\b/i.test(varyValues)) { + return { + disposition: 'failed', + reason: 'Vary on X-Vercel-IP-Latitude, X-Vercel-IP-Longitude, or X-Vercel-IP-Postal-Code creates very high-cardinality CDN cache keys; use a coarser geography header when safe, or leave the response uncached', + }; + } + return { disposition: 'verified', reason: 'Vary header avoids known high-cardinality geolocation headers' }; +} + async function verifyNextCachedNotFoundCausalSupport({ rec }) { if (!rec) return { disposition: 'unsupported', reason: 'next_cached_not_found_causal_support requires rec' }; const text = recText(rec); @@ -486,6 +506,39 @@ async function verifyNextCacheComponentsRouteSegmentConfig({ rec, framework, fra }; } +async function verifyNextRouteRevalidateStaticPrereq({ rec, framework, cacheComponents, repoRoot = '.', projectRootDirectory = null }) { + if (!rec) return { disposition: 'unsupported', reason: 'next_route_revalidate_static_prereq requires rec' }; + if (framework !== 'next') return { disposition: 'verified', reason: 'not a Next.js project' }; + if (cacheComponents === true) { + return { disposition: 'verified', reason: 'Cache Components route-segment restrictions are handled separately' }; + } + const files = recommendationFilesFromRec(rec) + .filter((file) => /(^|\/)app\/.+\/(?:page|layout|template)\.(?:tsx?|jsx?)$/.test(String(file)) || + /(^|\/)(?:page|layout|template)\.(?:tsx?|jsx?)$/.test(String(file))); + if (files.length === 0) { + return { disposition: 'verified', reason: 'route-level revalidate recommendation does not target a page/layout/template file' }; + } + + const dynamicHits = []; + for (const file of files) { + const routeChain = await readNextRouteChainFiles(file, repoRoot, projectRootDirectory); + if (routeChain.length === 0) { + return { disposition: 'unverifiable', reason: `could not inspect route chain for ${file}` }; + } + for (const entry of routeChain) { + const hit = firstDynamicRouteChainReason(entry.content); + if (hit) dynamicHits.push(`${entry.relative}:${hit}`); + } + } + if (dynamicHits.length > 0) { + return { + disposition: 'failed', + reason: `route-level revalidate can be defeated by request-time APIs or auth helpers in the route chain (${dynamicHits.slice(0, 3).join(', ')}); prove the route is ISR/static from next build output or move the dynamic read out before recommending revalidate`, + }; + } + return { disposition: 'verified', reason: 'no request-time API or common auth helper detected in the recommended route chain' }; +} + async function verifyNextCacheTagInvalidationSupported({ rec, repoRoot = '.', projectRootDirectory = null }) { if (!rec) return { disposition: 'unsupported', reason: 'next_cache_tag_invalidation_supported requires rec' }; const tags = extractCacheTags(recText(rec)); @@ -1048,6 +1101,50 @@ function recommendationFilesFromRec(rec) { ])); } +async function readNextRouteChainFiles(file, repoRoot, projectRootDirectory) { + const normalized = normalizeProjectRootDirectory(file); + if (!normalized) return []; + const appIdx = normalized.split('/').lastIndexOf('app'); + if (appIdx === -1) { + try { + const { path, content } = await readClaimFile({ file, repoRoot, projectRootDirectory }); + return [{ path, relative: normalized, content }]; + } catch { + return []; + } + } + + const parts = normalized.split('/'); + const appParts = parts.slice(0, appIdx + 1); + const routeDirs = parts.slice(appIdx + 1, -1); + const candidates = new Set([normalized]); + for (let depth = 0; depth <= routeDirs.length; depth++) { + const dir = [...appParts, ...routeDirs.slice(0, depth)].join('/'); + for (const base of ['layout', 'template']) { + for (const ext of ['tsx', 'ts', 'jsx', 'js']) candidates.add(`${dir}/${base}.${ext}`); + } + } + + const out = []; + for (const candidate of candidates) { + try { + const { path, content } = await readClaimFile({ file: candidate, repoRoot, projectRootDirectory }); + out.push({ path, relative: candidate, content }); + } catch {} + } + return out; +} + +function firstDynamicRouteChainReason(content) { + const text = String(content ?? ''); + const direct = text.match(/\b(cookies|headers|draftMode|connection)\s*\(/); + if (direct) return `${direct[1]}()`; + const helper = text.match(/\b(withAuth|getServerSession|auth|currentUser)\s*\(/); + if (helper) return `${helper[1]}()`; + if (/from\s+['"]next\/headers['"]/.test(text)) return 'next/headers import'; + return null; +} + function pathSuffixMatches(candidateFile, routeFile) { const candidate = normalizeProjectRootDirectory(candidateFile); const route = normalizeProjectRootDirectory(routeFile); diff --git a/skills/vercel-optimize/metadata.json b/skills/vercel-optimize/metadata.json index 3d8eaff..8287bad 100644 --- a/skills/vercel-optimize/metadata.json +++ b/skills/vercel-optimize/metadata.json @@ -1,5 +1,5 @@ { - "version": "1.1.0", + "version": "1.2.0", "organization": "Vercel Engineering", "date": "May 2026", "abstract": "Deep cost and performance optimization for supported Vercel projects. Pulls observability metrics, billing, and project config from the Vercel CLI, then investigates the codebase only where those metrics point. Produces ranked, verified recommendations with before/after code grounded in version-aware documentation citations. Avoids precise dollar projections (cost framed as order-of-magnitude) and avoids invented documentation URLs (curated allow-list).", diff --git a/skills/vercel-optimize/references/docs-library.json b/skills/vercel-optimize/references/docs-library.json index 0c45992..ad4e330 100644 --- a/skills/vercel-optimize/references/docs-library.json +++ b/skills/vercel-optimize/references/docs-library.json @@ -1,7 +1,7 @@ { "$schema": "Curated documentation allow-list for the vercel-optimize skill. Entries are version-aware via applicableFrameworks (semver). The recommender prompt receives only the subset valid for the user's stack. Citations outside this allow-list are stripped by the unknown-citation sanitizer.", "version": "1.1.1", - "lastVerified": "2026-05-21", + "lastVerified": "2026-05-26", "schemaVersion": "1.0", "applicableFrameworksSyntax": "Semver-style. '*' = any framework. 'next@*' = any Next.js. 'next@14' = Next 14.x. 'next@>=15.0.0' = Next 15+. Multiple via '||': 'next@14 || next@>=15.0.0'.", "urls": [ @@ -12,6 +12,12 @@ "applicableFrameworks": ["*"] }, { + "url": "https://vercel.com/docs/headers/request-headers", + "topic": "Vercel request headers — documented geolocation header names including country, country-region, city, latitude, longitude, timezone, and postal code", + "appliesTo": ["uncached_route", "cache_header_gap"], + "applicableFrameworks": ["*"] + }, + { "url": "https://vercel.com/docs/caching/cache-control-headers", "topic": "Cache-Control headers on Vercel — s-maxage, CDN-Cache-Control, Vercel-CDN-Cache-Control, stale-while-revalidate", "appliesTo": ["uncached_route", "cache_header_gap"], @@ -67,7 +73,7 @@ }, { "url": "https://vercel.com/docs/vercel-firewall/vercel-waf/managed-rulesets", - "topic": "Vercel WAF managed rulesets — managed bot rules and staged Log or Deny actions", + "topic": "Vercel WAF managed rulesets — managed bot and AI-bot actions by ruleset", "appliesTo": ["platform_bot_protection", "uncached_route"], "applicableFrameworks": ["*"] }, @@ -325,9 +331,9 @@ }, { "url": "https://nextjs.org/docs/app/api-reference/functions/unstable_cache", - "topic": "unstable_cache — Next.js 14 cache primitive (deprecated in 15 in favor of 'use cache')", + "topic": "unstable_cache — Next.js 14/15 cache primitive (replaced in 16 by 'use cache')", "appliesTo": ["uncached_route", "function-duration"], - "applicableFrameworks": ["next@14"] + "applicableFrameworks": ["next@14 || next@15"] }, { "url": "https://nextjs.org/docs/app/api-reference/directives/use-cache", diff --git a/skills/vercel-optimize/references/recommendations.md b/skills/vercel-optimize/references/recommendations.md index 96a3aa1..cd1ae07 100644 --- a/skills/vercel-optimize/references/recommendations.md +++ b/skills/vercel-optimize/references/recommendations.md @@ -192,7 +192,7 @@ The recommender's citation library is filtered by `signals.json.stack.framework@ | App Router | Next ≥ 13.0 | Default since 14 | | `generateStaticParams` | Next ≥ 13.0 | Replaces getStaticPaths for App Router | | Fetch `next: { revalidate }` | Next ≥ 13.0 | Note: default fetch caching flipped in Next 15 | -| `unstable_cache` | Next 14 only | Replaced by 'use cache' in 15+ | +| `unstable_cache` | Next 14-15 | Replaced by 'use cache' in 16 | | `'use cache'` directive | Next ≥ 15.0 | Persistent cache primitive | | `cacheLife()`, `cacheTag()` | Next ≥ 15.0 | Pairs with 'use cache' | | `after()` | Next ≥ 15.0 | Non-blocking post-response work | diff --git a/skills/vercel-optimize/references/scoring.md b/skills/vercel-optimize/references/scoring.md index 8e18447..a988493 100644 --- a/skills/vercel-optimize/references/scoring.md +++ b/skills/vercel-optimize/references/scoring.md @@ -104,10 +104,12 @@ The agent renders this as the final output of Step 4. The shape is fixed; the co | Service | Usage | Billed Cost | |---|---|---| -| (rows from usage.services, sorted by billedCost desc) | +| (non-zero rows from usage.services, sorted by billedCost desc) | Total billed: {usage.totals.billedCost} (we render the precise current cost — we just don't project future precise savings) +Omit zero-cost service rows from the table at the same cent precision shown to customers. If every row has `$0.00` billed cost but `effectiveCost` / USD `pricingQuantity` is non-zero, explain that net billed cost is `$0.00` after included credits or allotments and show the effective usage cost table instead. If both billed and effective costs are `$0.00`, replace the table with a concise note that `vercel usage` returned a billing payload but every reported service cost was `$0.00` for the window. + If `vercel usage` was queried and unavailable, the cost breakdown is replaced by an observability-derived cost ranking from `metrics.fnGbHrByRoute` + `metrics.fnCpuMsByRoute` + `metrics.fdtByRoute` when those metrics exist. These don't translate directly to dollars — they show *which routes consume the billable units* so the user knows what to attack first. If `usageError` is `NOT_COLLECTED_OBSERVABILITY_BLOCKED` or another `NOT_COLLECTED_*` value, say usage was not collected; do not describe it as a billing-plan or Costs-feature finding. Render Observability status from the actual collection state: @@ -146,6 +148,8 @@ For each high-priority rec, in order: Non-recommendation findings from reconciliation or investigation: deployment regressions, route-error storms, metric mismatches, and other real signals that should not become speculative performance recommendations. +Observations must not contain implementation-grade actions. If the suggested action says to enable, add, wrap, apply, move, configure, challenge, deny, or otherwise change code or project settings, the renderer must hold it back until it passes the ready-to-apply recommendation evidence bar. Customer-visible observations can ask for narrower evidence collection: inspect logs, compare deployments, check headers, or confirm cacheability. + ## Investigated, no change recommended Candidates that were checked but did not produce a supported recommendation. Use plain reasons; do not use "abstain" in customer-facing copy. diff --git a/skills/vercel-optimize/references/verification.md b/skills/vercel-optimize/references/verification.md index ea06d2e..fbb5024 100644 --- a/skills/vercel-optimize/references/verification.md +++ b/skills/vercel-optimize/references/verification.md @@ -34,11 +34,13 @@ The verifier extracts claims from `why`, `fix`, `currentBehavior`, `desiredBehav | 10 | `cited_count_literal` | "60+ icons in packages/ui/src/icons" | glob directory, count by extension | | 11 | `citation_in_library` | Any URL in `citations[]` | URL ∈ `references/docs-library.json` | | 12 | `citation_applies_to_version` | Any URL in `citations[]` | URL's `applicableFrameworks` matches `signals.json.stack.framework@frameworkVersion` | -| 13 | `cache_vary_matches_dynamic_inputs` | CDN cache rec touches route files that read Vercel geolocation | Fails unless the rec varies by `X-Vercel-IP-Country` | +| 13 | `cache_vary_matches_dynamic_inputs` | CDN cache rec touches route files that read Vercel geolocation | Fails unless the rec varies by a coarse Vercel geolocation header such as `X-Vercel-IP-Country`, `X-Vercel-IP-Country-Region`, or `X-Vercel-IP-City` | +| 13a | `cache_vary_cardinality_safe` | CDN cache rec sets `Vary` on request-specific geography | Fails on high-cardinality `X-Vercel-IP-Latitude` / `X-Vercel-IP-Longitude` / `X-Vercel-IP-Postal-Code` | | 14 | `next_cached_not_found_causal_support` | Rec claims `notFound()` inside `'use cache'` caused 5xx | Fails unless backed by Next-specific docs or runtime stack evidence | | 15 | `next_stable_cache_api_for_version` | Next.js 16 cache rec includes code examples | Fails on `unstable_cacheLife` / `unstable_cacheTag` or one-arg `revalidateTag()` | | 16 | `next_cache_components_runtime_cache_preference` | Next.js rec uses Runtime Cache APIs while `cacheComponents=true` | Fails unless `use cache: remote` is used or Runtime Cache is framed as a fallback | | 17 | `next_cache_components_route_segment_config` | Next.js 16 rec suggests removed route segment config while `cacheComponents=true` | Fails on `dynamicParams`, `dynamic`, `revalidate`, or `fetchCache` recommendations | +| 17a | `next_route_revalidate_static_prereq` | Rec suggests route-level `export const revalidate` for a Next.js page/layout route | Fails when the route chain contains request-time APIs or common auth helpers that can force dynamic rendering | | 18 | `next_cache_lifetime_freshness_supported` | Rec lengthens a tagged Cache Components lifetime with `cacheLife()` | Fails unless every affected `cacheTag()` has matching `revalidateTag()` / `updateTag()` evidence | | 19 | `next_cache_life_cdn_header_semantics` | Rec claims `cacheLife()` emits CDN/Cache-Control headers or that missing `cacheLife()` alone makes a route run per request | Fails unless rewritten to the documented Cache Components lifetime behavior or backed by production header evidence | | 20 | `next_cache_tag_invalidation_supported` | Cache-lifetime rec claims existing tag invalidation | Fails unless every claimed `cacheTag()` has matching `revalidateTag()` / `updateTag()` evidence | diff --git a/skills/vercel-optimize/scripts/render-report.mjs b/skills/vercel-optimize/scripts/render-report.mjs index 68ff4ae..542a992 100644 --- a/skills/vercel-optimize/scripts/render-report.mjs +++ b/skills/vercel-optimize/scripts/render-report.mjs @@ -7,6 +7,7 @@ import { dirname, resolve } from 'node:path'; import { buildFinalReportMessage, renderReport } from '../lib/render-report.mjs'; import { dedupeRecommendations } from '../lib/dedup-recs.mjs'; import { canonicalizeRoute } from '../lib/route-normalize.mjs'; +import { hasUnsupportedCacheLifeCdnText, splitCustomerSafeObservations } from '../lib/observation-safety.mjs'; const log = (...a) => console.error('[render-report]', ...a); const HARD_REGEN_TRIGGERS = new Set([ @@ -113,7 +114,7 @@ async function main() { ...(Array.isArray(recsRaw.observations) ? recsRaw.observations : []), ...(Array.isArray(recsRaw.abstentions) ? recsRaw.abstentions : []), ]); - const { observations: safeObservations, heldBackObservations } = splitCustomerSafeObservations(flattenedObservations, baseAbstentions); + const { observations: safeObservations, heldBackObservations } = splitCustomerSafeObservations(flattenedObservations, baseAbstentions, signalsRaw); const observations = suppressReadyCoveredObservations(safeObservations, recommendations); const abstentions = [ @@ -159,7 +160,9 @@ async function main() { ? recsRaw.summary.withheldRecommendations : (Array.isArray(recsRaw.regenPlan) ? recsRaw.regenPlan.length : 0) + (Array.isArray(recsRaw.qualityDropped) ? recsRaw.qualityDropped.length : 0)) + - (Array.isArray(recsRaw.sanitizerDropped) ? recsRaw.sanitizerDropped.length : 0), + (Array.isArray(recsRaw.sanitizerDropped) ? recsRaw.sanitizerDropped.length : 0) + + (Array.isArray(recsRaw.heldBackObservations) ? recsRaw.heldBackObservations.length : 0) + + heldBackObservations.length, noChangeCount: Number.isInteger(recsRaw.summary?.abstentions) ? Math.min(recsRaw.summary.abstentions, publicBaseAbstentions.length) : publicBaseAbstentions.length, @@ -283,24 +286,6 @@ function candidateMatchesRef(candidate, ref) { return a === b || canonicalizeRoute(a) === canonicalizeRoute(b); } -function splitCustomerSafeObservations(observations, abstentions = []) { - const safe = []; - const heldBack = []; - for (const observation of observations) { - const unsafeReason = unsupportedObservationReason(observation, abstentions); - if (unsafeReason) { - heldBack.push({ - candidateRef: observation.candidateRef ?? null, - reason: unsafeReason, - needsEvidence: true, - }); - } else { - safe.push(observation); - } - } - return { observations: safe, heldBackObservations: heldBack }; -} - function suppressReadyCoveredObservations(observations, recommendations = []) { if (!Array.isArray(observations) || observations.length === 0) return []; const readyFamiliesByTarget = new Map(); @@ -348,46 +333,6 @@ function candidateFamily(kind) { } } -function unsupportedObservationReason(observation, abstentions = []) { - if (contradictsNoChangeReason(observation, abstentions)) { - return 'This observation repeated an action that another investigation rejected. Re-run with a single scoped candidate before applying it.'; - } - if (hasUnsupportedFrameworkCausalClaim(observation)) { - return 'This observation made a framework-specific cause claim that verification could not support. Re-run with runtime logs or official framework evidence before applying it.'; - } - if (hasUnsupportedStaticGenerationClaim(observation)) { - return 'This observation made a static-generation behavior claim that verification could not support. Re-run with route-manifest or runtime evidence before applying it.'; - } - if (hasUnsupportedSourceAbsenceClaim(observation)) { - return 'This observation made a source-file absence claim that verification could not support. Re-run with a file-existence check or runtime logs before applying it.'; - } - if (hasUnsupportedCacheLifeCdnClaim(observation)) { - return 'This observation depended on an unsupported cacheLife-to-CDN claim. Re-run with production header evidence before applying it.'; - } - return null; -} - -function contradictsNoChangeReason(observation, abstentions) { - const target = candidateTarget(observation?.candidateRef); - if (!target) return false; - const observationText = [ - observation?.summary, - observation?.evidence, - observation?.suggestedAction, - ].filter(Boolean).join(' ').toLowerCase(); - const relevantReasons = abstentions - .filter((a) => candidateTarget(a?.candidateRef) === target) - .map((a) => String(a?.reason ?? '').toLowerCase()); - if (relevantReasons.length === 0) return false; - - if (/\bparalleliz(?:e|ing)\b/.test(observationText) && - /\bgetsession\b/.test(observationText) && - relevantReasons.some((reason) => /\bgetsession\b/.test(reason) && /\b(?:gates?|redirect|auth-preserving|blocked)\b/.test(reason))) { - return true; - } - return false; -} - function candidateTarget(ref) { if (typeof ref !== 'string') return null; const idx = ref.indexOf(':'); @@ -402,63 +347,6 @@ function publicNoChangeReason(reason) { return reason; } -function hasUnsupportedFrameworkCausalClaim(observation) { - const text = [ - observation?.summary, - observation?.evidence, - observation?.suggestedAction, - ].filter(Boolean).join(' ').toLowerCase(); - if (!text.includes('notfound') || !text.includes('use cache')) return false; - return ( - /known next\.js cache components edge case/.test(text) || - /next\.js\s+\d+(?:\.\d+)?[^.]{0,120}treats[^.]{0,120}dynamic api/.test(text) || - /can surface as 5xx/.test(text) || - /surface as 500/.test(text) || - /instead of throwing inside (?:the )?cache/.test(text) || - /cache boundary/.test(text) - ); -} - -function hasUnsupportedStaticGenerationClaim(observation) { - const text = [ - observation?.summary, - observation?.evidence, - observation?.suggestedAction, - ].filter(Boolean).join(' ').toLowerCase(); - if (!/\bgeneratestaticparams\b/.test(text)) return false; - return /\b(?:returns?\s*(?:an\s+)?empty|\[\])\b[^.\n]{0,240}\b(?:every request|on[- ]demand|no params? (?:are )?prebuilt|populate generatestaticparams|served from (?:the )?cdn|hit bucket|cachebreakdown)\b/i.test(text) || - /\b(?:every request|on[- ]demand|no params? (?:are )?prebuilt|populate generatestaticparams|served from (?:the )?cdn|hit bucket|cachebreakdown)\b[^.\n]{0,240}\b(?:returns?\s*(?:an\s+)?empty|\[\])\b/i.test(text) || - /\bdynamic\s*=\s*['"`]error['"`]\b[^.\n]{0,240}\b(?:generatestaticparams|dynamicparams|every request|on[- ]demand)\b/i.test(text); -} - -function hasUnsupportedSourceAbsenceClaim(observation) { - const ref = String(observation?.candidateRef ?? ''); - if (!ref.startsWith('route_errors:')) return false; - const text = [ - observation?.summary, - observation?.evidence, - observation?.suggestedAction, - ].filter(Boolean).join(' ').toLowerCase(); - return /\b(?:enoent|no\s+(?:matching|corresponding)\s+(?:mdx|file|post)|missing\s+(?:mdx|file|post)|does\s+not\s+exist|not\s+found\s+on\s+disk)\b/.test(text); -} - -function hasUnsupportedCacheLifeCdnClaim(observation) { - const text = [ - observation?.summary, - observation?.evidence, - observation?.suggestedAction, - ].filter(Boolean).join(' '); - return hasUnsupportedCacheLifeCdnText(text); -} - -function hasUnsupportedCacheLifeCdnText(text) { - if (typeof text !== 'string' || !/\bcacheLife\b/i.test(text)) return false; - if (/\btoLaunch-\d+\b/i.test(text)) return true; - return /\bcacheLife\b[^.\n]{0,240}\b(?:Cache-Control|s-maxage|CDN|edge cache|cache breakdown|x-vercel-cache|HIT|MISS|function (?:still )?runs per request|every request invokes the function|canonical|toLaunch-\d+)\b/i.test(text) || - /\b(?:Cache-Control|s-maxage|CDN|edge cache|cache breakdown|x-vercel-cache|HIT|MISS|function (?:still )?runs per request|every request invokes the function|canonical|toLaunch-\d+)\b[^.\n]{0,240}\bcacheLife\b/i.test(text) || - /\b(?:no|never|without|missing)\s+cacheLife\b[^.\n]{0,240}\b(?:no|not|never|0%|every|per request|function)\b[^.\n]{0,120}\b(?:cache|cached|hit|runs?|invoke)/i.test(text); -} - function buildDebugArtifact({ recsRaw, recommendationsRaw, diff --git a/skills/vercel-optimize/scripts/verify-and-regen.mjs b/skills/vercel-optimize/scripts/verify-and-regen.mjs index 6cbf03a..45973c3 100644 --- a/skills/vercel-optimize/scripts/verify-and-regen.mjs +++ b/skills/vercel-optimize/scripts/verify-and-regen.mjs @@ -174,7 +174,10 @@ async function main() { ); const triggeredByPassRate = verifiable >= REGEN_MIN_CLAIMS && passRate < REGEN_PASS_RATE_THRESHOLD; const cacheSafetyFailures = claimsWithResults.filter( - (r) => r.disposition === 'failed' && r.claim?.type === 'cache_vary_matches_dynamic_inputs' + (r) => r.disposition === 'failed' && ( + r.claim?.type === 'cache_vary_matches_dynamic_inputs' || + r.claim?.type === 'cache_vary_cardinality_safe' + ) ); const semanticSafetyFailures = claimsWithResults.filter( (r) => r.disposition === 'failed' && ( @@ -188,6 +191,7 @@ async function main() { r.claim?.type === 'image_response_headers_citation' || r.claim?.type === 'next_image_priority_api_for_version' || r.claim?.type === 'next_cache_components_route_segment_config' || + r.claim?.type === 'next_route_revalidate_static_prereq' || r.claim?.type === 'next_cache_tag_invalidation_supported' || r.claim?.type === 'cache_rec_not_error_dominated_or_acknowledged' || r.claim?.type === 'cache_control_header_syntax' || @@ -234,7 +238,7 @@ async function main() { regenBriefHint: triggeredByContradiction ? 'Sub-agent recommended toggling on a project setting that is already enabled. Re-spawn with the project-config Strengths block highlighted; the rec must drop the contradictory step and keep only the actionable parts.' : triggeredByCacheSafety - ? 'Sub-agent recommended CDN caching for output that varies by Vercel geolocation. Re-spawn with the cache safety failure highlighted; the rec must include the correct Vary header (for example X-Vercel-IP-Country) or abstain.' + ? 'Sub-agent recommended CDN caching with unsafe or missing Vary behavior. Re-spawn with the cache safety failure highlighted; the rec must use a low-cardinality Vary header that matches the dynamic inputs, or abstain.' : triggeredBySemanticSafety ? 'Sub-agent made a framework-semantic claim that failed deterministic checks. Re-spawn with the failure highlighted; the rec must either add version-correct code/citations/runtime evidence or abstain.' : 'Re-spawn the sub-agent with this rec\'s topFailures injected as feedback. Re-emit the rec only if regenPassRate >= originalPassRate AND citation count not gutted.', |
