diff options
| author | nicdillon <nicholas.dillon@vercel.com> | 2026-10-28 11:40:52 -0400 |
|---|---|---|
| committer | nicdillon <nicholas.dillon@vercel.com> | 2026-10-28 11:40:52 -0400 |
| commit | a2f5f8e863ce1de53915bddffbbf9a08b2ac327f (patch) | |
| tree | 16be89057e655ce532aabc9b298d711e5ddc0edc | |
| parent | 1a2c79d4e6c4a2283dc2c614b6538cbe1ed5ca62 (diff) | |
vercel-optimize: make metric queries work on current Vercel CLIsvercel-optimize/count-aggregation-fallback
Metric queries were written against CLI v53.4.0. Current CLIs (seen on 62.5.0
and 65.2.0) changed three things, so 16 of 32 queries failed with
invalid_request, including the Observability Plus canary in collect-signals.
Audits then reported Observability Plus as unusable (blocker=all_failed_other)
on teams that have it.
1. *.count metrics allow only count/unique; the skill used sum.
2. Dimensions are camelCase (originHostname, cacheResult, botCategory,
requestPath, sourceImageHostname, wafAction, ...).
3. Metrics were renamed: speed_insights_metric.{lcp,inp,cls,ttfb_ms} ->
speed_insights.{lcp_ms,inp_ms,cls,ttfb_ms}; speed_insights_metric.count was
removed (sample volume now comes from the count of LCP samples);
middleware_invocation.duration_ms -> request_duration_ms.
queryMetric now retries a rejected query in the current form
(modernizeMetricQuery) and maps the response back to the legacy metric,
aggregation, and dimension names (restoreLegacyMetricShape). Callers, gates,
and the v53.4.0 fixtures are unchanged, and older CLIs keep the original
query.
Verified against `vercel metrics schema <metric>` on CLI 65.2.0. With the
first part of this fix, the canary passed and request-count queries returned
56 rows on a team with Observability Plus.
Repro:
vercel metrics vercel.request.count --format json -a sum --since 14d --limit 1
-> {"error":{"code":"invalid_request"}}
vercel metrics vercel.request.count --format json --since 14d --limit 1
-> summary [{ vercel_request_count_count: 257 }]
| -rw-r--r-- | packages/vercel-optimize-tests/test/count-aggregation-fallback.test.mjs | 79 | ||||
| -rw-r--r-- | skills/vercel-optimize/lib/vercel.mjs | 91 |
2 files changed, 168 insertions, 2 deletions
diff --git a/packages/vercel-optimize-tests/test/count-aggregation-fallback.test.mjs b/packages/vercel-optimize-tests/test/count-aggregation-fallback.test.mjs new file mode 100644 index 0000000..445df15 --- /dev/null +++ b/packages/vercel-optimize-tests/test/count-aggregation-fallback.test.mjs @@ -0,0 +1,79 @@ +// Queries were written against CLI v53.4.0. Current CLIs (seen on 62.5.0 and 65.2.0) renamed some metrics, +// use camelCase dimensions, and allow only count/unique on *.count metrics. queryMetric retries a rejected +// query in the current form and maps the response back. These cases come from `vercel metrics schema`. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { modernizeMetricQuery, restoreLegacyMetricShape, normalizeSummary } from '../../../skills/vercel-optimize/lib/vercel.mjs'; +import { QUERIES } from '../../../skills/vercel-optimize/lib/queries.mjs'; + +// Current catalog (CLI 65.2.0): metric -> { dims, aggs } +const CATALOG = { + 'vercel.request.count': { dims: ['route', 'cacheResult', 'httpStatus', 'requestMethod', 'botCategory'], aggs: ['count', 'unique'] }, + 'vercel.external_api_request.count': { dims: ['originHostname', 'requestPath'], aggs: ['count', 'unique'] }, + 'vercel.external_api_request.request_duration_ms': { dims: ['originHostname'], aggs: ['count', 'sum', 'avg', 'p50', 'p75', 'p95'] }, + 'vercel.external_api_request.transfer_bytes': { dims: ['originHostname'], aggs: ['count', 'sum'] }, + 'vercel.request.fdt_total_bytes': { dims: ['route', 'cacheResult', 'botCategory'], aggs: ['count', 'sum', 'avg', 'p95'] }, + 'vercel.middleware_invocation.count': { dims: ['route', 'requestPath'], aggs: ['count', 'unique'] }, + 'vercel.middleware_invocation.request_duration_ms': { dims: ['route', 'requestPath'], aggs: ['count', 'sum', 'p95'] }, + 'vercel.image_transformation.count': { dims: ['sourceImageHostname'], aggs: ['count', 'unique'] }, + 'vercel.firewall_action.count': { dims: ['wafAction', 'action', 'route'], aggs: ['count', 'unique'] }, + 'vercel.speed_insights.lcp_ms': { dims: ['route'], aggs: ['count', 'p75'] }, + 'vercel.speed_insights.inp_ms': { dims: ['route'], aggs: ['count', 'p75'] }, + 'vercel.speed_insights.cls': { dims: ['route'], aggs: ['count', 'p75'] }, + 'vercel.speed_insights.ttfb_ms': { dims: ['route'], aggs: ['count', 'p75'] }, +}; + +const FAILED_ON_65 = ['externalApiP75', 'fdtByBot', 'fdtByCache', 'middlewareCount', 'middlewareDurationP95', 'imageByHost', + 'cwvLcpByRoute', 'cwvInpByRoute', 'cwvClsByRoute', 'cwvTtfbByRoute', 'cwvCount', 'cwvCountByRoute', 'firewallByAction', + 'externalApiCount', 'externalApiBytes']; + +test('modernizeMetricQuery: every query that failed on CLI 65.2.0 maps to a valid current query', () => { + for (const id of FAILED_ON_65) { + const q = QUERIES.find((x) => x.id === id); + assert.ok(q, `query ${id} exists`); + const m = modernizeMetricQuery(q.metricId, q); + const entry = CATALOG[m.metricId]; + assert.ok(entry, `${id}: ${m.metricId} is in the current catalog`); + assert.ok(entry.aggs.includes(m.aggregation), `${id}: aggregation ${m.aggregation} is allowed`); + for (const dim of m.groupBy) assert.ok(entry.dims.includes(dim), `${id}: dimension ${dim} is allowed`); + } +}); + +test('modernizeMetricQuery: renames, camelCases, and switches sum to count where needed', () => { + assert.deepEqual(modernizeMetricQuery('vercel.request.count', { aggregation: 'sum', groupBy: ['route', 'cache_result'] }), + { metricId: 'vercel.request.count', aggregation: 'count', groupBy: ['route', 'cacheResult'], filter: undefined, changed: true }); + const cwv = modernizeMetricQuery('vercel.speed_insights_metric.count', { aggregation: 'sum', groupBy: ['route'] }); + assert.equal(cwv.metricId, 'vercel.speed_insights.lcp_ms'); + assert.equal(cwv.aggregation, 'count'); + const mw = modernizeMetricQuery('vercel.middleware_invocation.duration_ms', { aggregation: 'p95', groupBy: ['request_path'] }); + assert.deepEqual([mw.metricId, mw.aggregation, mw.groupBy], ['vercel.middleware_invocation.request_duration_ms', 'p95', ['requestPath']]); + assert.equal(modernizeMetricQuery('vercel.function_invocation.function_duration_ms', { aggregation: 'p95', groupBy: ['route'] }).changed, false); +}); + +test('modernizeMetricQuery: rewrites grouped dimension names inside a filter', () => { + const m = modernizeMetricQuery('vercel.request.fdt_total_bytes', { aggregation: 'sum', groupBy: ['cache_result'], filter: "cache_result eq 'MISS'" }); + assert.equal(m.filter, "cacheResult eq 'MISS'"); +}); + +test('restoreLegacyMetricShape: maps fields and dimensions back so normalizeSummary is unchanged', () => { + const legacy = { metricId: 'vercel.speed_insights_metric.lcp', aggregation: 'p75', groupBy: ['route'] }; + const modern = modernizeMetricQuery(legacy.metricId, legacy); + const out = restoreLegacyMetricShape({ + query: { metric: modern.metricId, aggregation: 'p75' }, + summary: [{ route: '/a', vercel_speed_insights_lcp_ms_p75: 2100 }], + data: [{ timestamp: 't', vercel_speed_insights_lcp_ms_p75: 2000 }], + }, legacy, modern); + assert.deepEqual(normalizeSummary(out, legacy.metricId, legacy.aggregation, legacy.groupBy), [{ value: 2100, route: '/a' }]); + assert.equal(out.data[0].vercel_speed_insights_metric_lcp_p75, 2000); + assert.equal(out.query.metric, legacy.metricId); + + const legacy2 = { metricId: 'vercel.request.count', aggregation: 'sum', groupBy: ['route', 'cache_result'] }; + const modern2 = modernizeMetricQuery(legacy2.metricId, legacy2); + const out2 = restoreLegacyMetricShape({ summary: [{ route: '/a', cacheResult: 'HIT', vercel_request_count_count: 257 }] }, legacy2, modern2); + assert.deepEqual(normalizeSummary(out2, legacy2.metricId, 'sum', legacy2.groupBy), [{ value: 257, route: '/a', cache_result: 'HIT' }]); + assert.ok(!JSON.stringify(out2).includes('cacheResult')); +}); + +test('restoreLegacyMetricShape: passes through non-objects', () => { + assert.equal(restoreLegacyMetricShape(null, { metricId: 'a', aggregation: 'sum', groupBy: [] }, { metricId: 'a', aggregation: 'count', groupBy: [] }), null); +}); diff --git a/skills/vercel-optimize/lib/vercel.mjs b/skills/vercel-optimize/lib/vercel.mjs index 0810c70..1566bc0 100644 --- a/skills/vercel-optimize/lib/vercel.mjs +++ b/skills/vercel-optimize/lib/vercel.mjs @@ -507,11 +507,98 @@ export async function queryMetric(metricId, opts = {}) { const onRetry = (attempt, delayMs) => { console.error(`[queryMetric] ${metricId} hit RATE_LIMITED; retry ${attempt}/3 after ${(delayMs / 1000).toFixed(0)}s`); }; - return await throttle.run(() => - retryOnRateLimit(() => runVercelJson(scopedArgs(args, opts.scope)), { onRetry }) + const runOnce = (a) => throttle.run(() => + retryOnRateLimit(() => runVercelJson(scopedArgs(a, opts.scope)), { onRetry }) ); + const result = await runOnce(args); + if (result?.ok || !AGGREGATION_REJECTIONS.has(String(result?.code))) return result; + + // Queries here were written against CLI v53.4.0. Later CLIs (seen on 62.5.0 and 65.2.0) renamed some + // metrics, switched dimensions to camelCase, and allow only count/unique on *.count metrics. On a + // rejection, retry in the current form and map the response back to the legacy shape, so callers, + // gates, and fixtures keep their field names. + const modern = modernizeMetricQuery(metricId, opts); + if (!modern.changed) return result; + const modernArgs = ['metrics', modern.metricId, '--format', 'json']; + if (modern.aggregation) modernArgs.push('-a', modern.aggregation); + for (const dim of modern.groupBy) modernArgs.push('--group-by', dim); + if (modern.filter) modernArgs.push('-f', modern.filter); + if (opts.since) modernArgs.push('--since', opts.since); + if (opts.until) modernArgs.push('--until', opts.until); + if (opts.limit) modernArgs.push('--limit', String(opts.limit)); + const retry = await runOnce(modernArgs); + if (!retry?.ok) return result; + return { + ...retry, + data: restoreLegacyMetricShape(retry.data, { metricId, aggregation: opts.aggregation, groupBy: opts.groupBy ?? [] }, modern), + compatFallback: { metricId: modern.metricId, aggregation: modern.aggregation, groupBy: modern.groupBy }, + }; +} + +const AGGREGATION_REJECTIONS = new Set(['invalid_aggregation', 'invalid_request']); + +// Renamed metrics. `aggregation` overrides the requested one when the replacement measures it differently: +// there is no Speed Insights count metric any more, so sample volume comes from the count of LCP samples. +const METRIC_RENAMES = Object.freeze({ + 'vercel.speed_insights_metric.lcp': { id: 'vercel.speed_insights.lcp_ms' }, + 'vercel.speed_insights_metric.inp': { id: 'vercel.speed_insights.inp_ms' }, + 'vercel.speed_insights_metric.cls': { id: 'vercel.speed_insights.cls' }, + 'vercel.speed_insights_metric.ttfb_ms': { id: 'vercel.speed_insights.ttfb_ms' }, + 'vercel.speed_insights_metric.count': { id: 'vercel.speed_insights.lcp_ms', aggregation: 'count' }, + 'vercel.middleware_invocation.duration_ms': { id: 'vercel.middleware_invocation.request_duration_ms' }, +}); + +const snakeToCamel = (dim) => String(dim).replace(/_([a-z0-9])/g, (_, c) => c.toUpperCase()); + +export function modernizeMetricQuery(metricId, opts = {}) { + const rename = METRIC_RENAMES[metricId]; + const id = rename?.id ?? metricId; + let aggregation = rename?.aggregation ?? opts.aggregation; + if (aggregation === 'sum' && /\.count$/.test(id)) aggregation = 'count'; + const legacyGroupBy = opts.groupBy ?? []; + const groupBy = legacyGroupBy.map(snakeToCamel); + let filter = opts.filter; + if (typeof filter === 'string') { + for (const dim of legacyGroupBy) filter = filter.replace(new RegExp(`\\b${dim}\\b`, 'g'), snakeToCamel(dim)); + } + const changed = id !== metricId + || aggregation !== opts.aggregation + || groupBy.some((d, i) => d !== legacyGroupBy[i]) + || filter !== opts.filter; + return { metricId: id, aggregation, groupBy, filter, changed }; +} + +export function restoreLegacyMetricShape(data, legacy, modern) { + if (!data || typeof data !== 'object') return data; + const fromKey = `${modern.metricId.replace(/\./g, '_')}_${modern.aggregation}`; + const toKey = `${legacy.metricId.replace(/\./g, '_')}_${legacy.aggregation}`; + const dimPairs = legacy.groupBy.map((d, i) => [modern.groupBy[i], d]).filter(([m, l]) => m && m !== l); + const restoreRow = (row) => { + if (!row || typeof row !== 'object') return row; + const out = { ...row }; + if (fromKey !== toKey && Object.hasOwn(out, fromKey)) { + out[toKey] = out[fromKey]; + delete out[fromKey]; + } + for (const [m, l] of dimPairs) { + if (Object.hasOwn(out, m)) { + out[l] = out[m]; + delete out[m]; + } + } + return out; + }; + return { + ...data, + query: data.query && typeof data.query === 'object' + ? { ...data.query, metric: legacy.metricId, aggregation: legacy.aggregation, groupBy: legacy.groupBy } + : data.query, + summary: Array.isArray(data.summary) ? data.summary.map(restoreRow) : data.summary, + data: Array.isArray(data.data) ? data.data.map(restoreRow) : data.data, + }; } + // Team-owned projects need `?teamId=<orgId>` to avoid current-team drift. User- // owned projects use the authenticated user context and should not pass teamId. export async function getProjectConfig(projectId, orgId) { |
