fix(stats): use capability fallbacks in comparison radar (#51161)

2026-09-24 14:26:39 -05:00 · 2026-09-24 14:26:39 -05:00 · 6df0d5d951
commit 6df0d5d951
parent 0f549842ee
5 changed files with 164 additions and 39 deletions
--- a/.github/actions/setup-bun/action.yml
+++ b/.github/actions/setup-bun/action.yml
@ -15,12 +15,15 @@ runs:
      with:
        node-version: "24"

-    - name: Get baseline download URL
+    - name: Get Bun version and baseline download URL
      id: bun-url
      shell: bash
      run: |
+        V=$(node -p "require('./package.json').packageManager.split('@')[1]")
+        # Bun 1.4.2 includes the patched peer-variant fix in oven-sh/bun#33646.
+        if [ "$RUNNER_OS" = "Windows" ]; then V=1.4.2; fi
+        echo "version=$V" >> "$GITHUB_OUTPUT"
        if [ "$RUNNER_ARCH" = "X64" ]; then
-          V=$(node -p "require('./package.json').packageManager.split('@')[1]")
          case "$RUNNER_OS" in
            macOS)   OS=darwin ;;
            Linux)   OS=linux ;;
@ -30,9 +33,10 @@ runs:
        fi

    - name: Setup Bun
+      id: setup-bun
      uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
      with:
-        bun-version-file: ${{ !steps.bun-url.outputs.url && 'package.json' || '' }}
+        bun-version: ${{ !steps.bun-url.outputs.url && steps.bun-url.outputs.version || '' }}
        bun-download-url: ${{ steps.bun-url.outputs.url }}

    - name: Get cache directory
@ -45,24 +49,16 @@ runs:
      uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
      with:
        path: ${{ steps.cache.outputs.dir }}
-        key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+        key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-${{ hashFiles('**/bun.lock') }}
        restore-keys: |
-          ${{ runner.os }}-bun-
+          ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-

    - name: Install setuptools for distutils compatibility
      run: python3 -m pip install setuptools || pip install setuptools || true
      shell: bash

    - name: Install dependencies
-      run: |
-        # Workaround for patched peer variants
-        # e.g. ./patches/ for standard-openapi
-        # https://github.com/oven-sh/bun/issues/28147
-        if [ "$RUNNER_OS" = "Windows" ]; then
-          bun install --linker hoisted ${{ inputs.install-flags }}
-        else
-          bun install ${{ inputs.install-flags }}
-        fi
+      run: bun install ${{ inputs.install-flags }}
      shell: bash

    - name: Save Bun dependencies
@ -70,4 +66,4 @@ runs:
      uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
      with:
        path: ${{ steps.cache.outputs.dir }}
-        key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+        key: ${{ runner.os }}-bun-${{ steps.setup-bun.outputs.bun-version }}-${{ hashFiles('**/bun.lock') }}
--- a/packages/stats/app/src/routes/compare-radar.tsx
+++ b/packages/stats/app/src/routes/compare-radar.tsx
@ -22,6 +22,12 @@ type RadarAxis = {
  label: string
  description: string
  score: (model: ModelCatalogEntry) => number | undefined
+  capability?: "reasoning" | "toolCall"
+}
+
+type RadarScore = {
+  value: number
+  fallback?: string
 }

 type RadarPoint = {
@ -37,7 +43,7 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
      name: model.name,
      labName: model.labName,
      color: radarColors[index % radarColors.length],
-      scores: axes().map((axis) => (model.catalog ? axis.score(model.catalog) : undefined)),
+      scores: axes().map((axis) => resolveRadarScore(axis, model.catalog)),
    })),
  )
  const accessibleDescription = createMemo(() =>
@ -62,6 +68,9 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
              <span>
                <strong>{model.name}</strong>
                <Show when={model.labName}>{(name) => <small>{name()}</small>}</Show>
+                <Show when={model.scores.some((score) => score.fallback)}>
+                  <small data-slot="compare-radar-coverage">Hollow points use fallbacks · hover for details</small>
+                </Show>
              </span>
            </li>
          )}
@ -89,10 +98,16 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
                  <polygon data-slot="compare-radar-area" points={radarSeriesPolygon(model.scores)} />
                  <For each={model.scores}>
                    {(score, index) => {
-                      const point = () => radarPoint(index(), axes().length, score ?? 0)
+                      const point = () => radarPoint(index(), axes().length, score.value)
                      return (
                        <>
-                          <circle data-slot="compare-radar-point" cx={point().x} cy={point().y} r="0.95" />
+                          <circle
+                            data-slot="compare-radar-point"
+                            data-fallback={score.fallback ? "true" : undefined}
+                            cx={point().x}
+                            cy={point().y}
+                            r="0.95"
+                          />
                          <circle
                            data-slot="compare-radar-point-hit"
                            cx={point().x}
@ -138,6 +153,13 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
          >
            <strong>{axes()[activeAxis() ?? 0]?.label}</strong>
            <p>{axes()[activeAxis() ?? 0]?.description}</p>
+            <For each={series()}>
+              {(model) => (
+                <p>
+                  {model.name}: {formatRadarScore(model.scores[activeAxis() ?? 0])}
+                </p>
+              )}
+            </For>
          </div>
        </Show>
      </div>
@ -166,7 +188,7 @@ export function ComparisonRadar(props: ComparisonRadarProps) {
  )
 }

-function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[] {
+export function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[] {
  const benchmarks = benchmarkScoreGroups(catalogModels)
  const toolUseBenchmarks = benchmarkScoreGroups(catalogModels, true)
  const costs = catalogModels.flatMap((model) => {
@ -180,9 +202,10 @@ function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[
  return [
    {
      label: "Reasoning",
-      description: "Ability to solve complex, multi-step problems. Based on reasoning benchmarks when available.",
-      score: (model) =>
-        benchmarkPercentile(model, benchmarks, reasoningBenchmarkPattern) ?? (model.reasoning ? 100 : 0),
+      capability: "reasoning",
+      description:
+        "Ability to solve complex, multi-step problems. Benchmarks take priority; reasoning support defaults to 50/100.",
+      score: (model) => benchmarkPercentile(model, benchmarks, reasoningBenchmarkPattern),
    },
    {
      label: "Coding",
@ -218,7 +241,8 @@ function buildRadarAxes(catalogModels: readonly ModelCatalogEntry[]): RadarAxis[
    },
    {
      label: "Tool use",
-      description: "Performance on agent benchmarks including Terminal-Bench, Tau3, and Claw-Eval.",
+      capability: "toolCall",
+      description: "Agent benchmark performance. Benchmarks take priority; tool calling support defaults to 50/100.",
      score: (model) =>
        benchmarkPercentile(model, toolUseBenchmarks, toolUseBenchmarkPattern, {
          aggregate: "average",
@ -324,9 +348,9 @@ function radarPolygonPoints(count: number, score: number) {
    .join(" ")
 }

-function radarSeriesPolygon(scores: (number | undefined)[]) {
+function radarSeriesPolygon(scores: RadarScore[]) {
  return scores
-    .map((score, index) => radarPoint(index, scores.length, score ?? 0))
+    .map((score, index) => radarPoint(index, scores.length, score.value))
    .map((point) => `${point.x},${point.y}`)
    .join(" ")
 }
@ -355,6 +379,17 @@ function roundRadarCoordinate(value: number) {
  return Math.round(value * 1000) / 1000
 }

-function formatRadarScore(score: number | undefined) {
-  return score === undefined ? "No data" : `${Math.round(score)}/100`
+function formatRadarScore(score: RadarScore) {
+  return `${Math.round(score.value)}/100${score.fallback ? ` — ${score.fallback}` : ""}`
+}
+
+export function resolveRadarScore(axis: RadarAxis, model: ModelCatalogEntry | null): RadarScore {
+  const score = model ? axis.score(model) : undefined
+  if (score !== undefined) return { value: score }
+  const supported = axis.capability ? model?.[axis.capability] : undefined
+  if (supported === undefined) return { value: 50, fallback: "No data; neutral placeholder" }
+  const capability = axis.capability === "toolCall" ? "Tool calling" : "Reasoning"
+  return supported
+    ? { value: 50, fallback: `${capability} supported; no comparable benchmark` }
+    : { value: 0, fallback: `${capability} not supported` }
 }
--- a/packages/stats/app/src/routes/index.css
+++ b/packages/stats/app/src/routes/index.css
@ -6291,8 +6291,7 @@ body {
  vector-effect: non-scaling-stroke;
 }

-[data-page="stats"] [data-slot="compare-radar-area"],
-[data-page="stats"] [data-slot="compare-radar-line"] {
+[data-page="stats"] [data-slot="compare-radar-area"] {
  stroke: currentColor;
  stroke-width: 1.5px;
  stroke-linejoin: round;
@ -6304,10 +6303,6 @@ body {
  fill-opacity: 0.09;
 }

-[data-page="stats"] [data-slot="compare-radar-line"] {
-  fill: none;
-}
-
 [data-page="stats"] [data-slot="compare-radar-point"] {
  fill: currentColor;
  stroke: currentColor;
@ -6315,6 +6310,10 @@ body {
  vector-effect: non-scaling-stroke;
 }

+[data-page="stats"] [data-slot="compare-radar-point"][data-fallback="true"] {
+  fill: var(--stats-bg);
+}
+
 [data-page="stats"] [data-slot="compare-radar-point-hit"] {
  fill: transparent;
  cursor: pointer;
--- a/packages/stats/app/src/routes/model-catalog.ts
+++ b/packages/stats/app/src/routes/model-catalog.ts
@ -25,8 +25,8 @@ export type ModelCatalogEntry = {
  limit?: { context?: number; output?: number }
  modalities: { input: string[]; output: string[] }
  openWeights: boolean
-  reasoning: boolean
-  toolCall: boolean
+  reasoning?: boolean
+  toolCall?: boolean
  attachment: boolean
  temperature: boolean
  cost?: ModelCatalogCost
@ -54,6 +54,7 @@ export type ModelCatalogLab = {

 export type ModelCatalog = {
  models: ModelCatalogEntry[]
+  aliases?: ModelCatalogEntry[]
  labs: ModelCatalogLab[]
 }

@ -80,7 +81,8 @@ export function findModelCatalogEntry(catalog: ModelCatalog, model: string, lab?
  return (
    catalog.models.find((entry) => entry.id.toLowerCase() === normalizedId) ??
    catalog.models.find((entry) => (lab ? entry.lab === catalogLabSlug(lab) : true) && entry.slug === leaf) ??
-    catalog.models.find((entry) => entry.slug === leaf)
+    catalog.models.find((entry) => entry.slug === leaf) ??
+    catalog.aliases?.find((entry) => (lab ? entry.lab === catalogLabSlug(lab) : true) && entry.slug === leaf)
  )
 }

@ -133,7 +135,7 @@ export function catalogSlug(value: string) {
    .replace(/-{2,}/g, "-")
 }

-function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayload?: unknown): ModelCatalog {
+export function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayload?: unknown): ModelCatalog {
  const costs = readCatalogCosts(pricingPayload)
  const labDescriptions = readCatalogLabDescriptions(payload, pricingPayload, labPayload)
  const models = readCatalogModels(payload)
@ -149,6 +151,25 @@ function buildModelCatalog(payload: unknown, pricingPayload?: unknown, labPayloa
    .toSorted((a, b) => a.lab.localeCompare(b.lab) || displayDateTime(b.releaseDate) - displayDateTime(a.releaseDate))
  return {
    models,
+    // Contributor is a serving tier of these Muse models, with its own pricing.
+    // Keep aliases out of the model population used to normalize benchmark scores.
+    aliases: ["meta/muse-spark-1.2", "meta/muse-spark-1.3"].flatMap((id) => {
+      const model = models.find((entry) => entry.id === id)
+      if (!model) return []
+      const alias = `${id}-contributor`
+      return [
+        {
+          ...model,
+          id: alias,
+          slug: `${model.slug}-contributor`,
+          name: `${model.name} Contributor`,
+          cost:
+            costs.get(catalogIdKey(alias)) ??
+            costs.get(`${model.lab}/${model.slug}-contributor`) ??
+            costs.get(`${model.slug}-contributor`),
+        },
+      ]
+    }),
    labs: Object.values(
      models.reduce<Record<string, ModelCatalogLab>>((result, model) => {
        result[model.lab] = {
@ -184,8 +205,8 @@ function readModelCatalogEntry(value: unknown): ModelCatalogEntry[] {
      limit: readCatalogLimit(value.limit),
      modalities: readCatalogModalities(value.modalities),
      openWeights: booleanValue(value.open_weights),
-      reasoning: booleanValue(value.reasoning),
-      toolCall: booleanValue(value.tool_call),
+      reasoning: typeof value.reasoning === "boolean" ? value.reasoning : undefined,
+      toolCall: typeof value.tool_call === "boolean" ? value.tool_call : undefined,
      attachment: booleanValue(value.attachment),
      temperature: booleanValue(value.temperature),
      cost: readCatalogCost(value.cost),
--- a/packages/stats/app/test/compare-radar.test.ts
+++ b/packages/stats/app/test/compare-radar.test.ts
@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test"
+import { buildRadarAxes, resolveRadarScore } from "../src/routes/compare-radar"
+import type { ModelCatalogEntry } from "../src/routes/model-catalog"
+
+const model: ModelCatalogEntry = {
+  id: "meta/muse-spark-1.3",
+  lab: "meta",
+  slug: "muse-spark-1-3",
+  name: "Muse Spark 1.3",
+  modalities: { input: ["text", "image"], output: ["text"] },
+  reasoning: true,
+  toolCall: true,
+  openWeights: false,
+  attachment: true,
+  temperature: true,
+  weights: [],
+  benchmarks: [],
+}
+
+function scores(entry: ModelCatalogEntry | null, catalog = [model]) {
+  return Object.fromEntries(buildRadarAxes(catalog).map((axis) => [axis.label, resolveRadarScore(axis, entry)]))
+}
+
+describe("radar capability fallbacks", () => {
+  test("supported capabilities have visible baselines without benchmarks", () => {
+    const result = scores(model)
+    expect(result["Tool use"]).toEqual({ value: 50, fallback: "Tool calling supported; no comparable benchmark" })
+    expect(result.Reasoning).toEqual({ value: 50, fallback: "Reasoning supported; no comparable benchmark" })
+    expect(result.Coding).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
+  })
+
+  test("explicitly unsupported capabilities remain zero", () => {
+    const result = scores({ ...model, reasoning: false, toolCall: false })
+    expect(result["Tool use"]).toEqual({ value: 0, fallback: "Tool calling not supported" })
+    expect(result.Reasoning).toEqual({ value: 0, fallback: "Reasoning not supported" })
+  })
+
+  test("unknown capabilities and unmatched models use neutral placeholders", () => {
+    const result = scores({ ...model, reasoning: undefined, toolCall: undefined })
+    expect(result["Tool use"]).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
+    expect(result.Reasoning).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
+    expect(Object.values(scores(null))).toEqual(Array(6).fill({ value: 50, fallback: "No data; neutral placeholder" }))
+  })
+
+  test("measured benchmark percentiles override fallbacks, including zero", () => {
+    const low = {
+      ...model,
+      benchmarks: [
+        { name: "Tau3", score: 20 },
+        { name: "GPQA", score: 40 },
+      ],
+    }
+    const high = {
+      ...model,
+      id: "other/model",
+      benchmarks: [
+        { name: "Tau3", score: 80 },
+        { name: "GPQA", score: 90 },
+      ],
+    }
+    expect(scores(low, [low, high])["Tool use"]).toEqual({ value: 0 })
+    expect(scores(low, [low, high]).Reasoning).toEqual({ value: 0 })
+    expect(scores(high, [low, high])["Tool use"]).toEqual({ value: 100 })
+    expect(scores(high, [low, high]).Reasoning).toEqual({ value: 100 })
+  })
+
+  test("a benchmark without comparison peers retains the capability baseline", () => {
+    const entry = { ...model, benchmarks: [{ name: "Tau3", score: 90 }] }
+    expect(scores(entry, [entry])["Tool use"]).toEqual({
+      value: 50,
+      fallback: "Tool calling supported; no comparable benchmark",
+    })
+  })
+})