74 lines
2.9 KiB
TypeScript
74 lines
2.9 KiB
TypeScript
import { describe, expect, test } from "bun:test"
|
|
import { buildRadarAxes, resolveRadarScore } from "../src/routes/compare-radar"
|
|
import type { ModelCatalogEntry } from "../src/routes/model-catalog"
|
|
|
|
const model: ModelCatalogEntry = {
|
|
id: "meta/muse-spark-1.3",
|
|
lab: "meta",
|
|
slug: "muse-spark-1-3",
|
|
name: "Muse Spark 1.3",
|
|
modalities: { input: ["text", "image"], output: ["text"] },
|
|
reasoning: true,
|
|
toolCall: true,
|
|
openWeights: false,
|
|
attachment: true,
|
|
temperature: true,
|
|
weights: [],
|
|
benchmarks: [],
|
|
}
|
|
|
|
function scores(entry: ModelCatalogEntry | null, catalog = [model]) {
|
|
return Object.fromEntries(buildRadarAxes(catalog).map((axis) => [axis.label, resolveRadarScore(axis, entry)]))
|
|
}
|
|
|
|
describe("radar capability fallbacks", () => {
|
|
test("supported capabilities have visible baselines without benchmarks", () => {
|
|
const result = scores(model)
|
|
expect(result["Tool use"]).toEqual({ value: 50, fallback: "Tool calling supported; no comparable benchmark" })
|
|
expect(result.Reasoning).toEqual({ value: 50, fallback: "Reasoning supported; no comparable benchmark" })
|
|
expect(result.Coding).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
|
|
})
|
|
|
|
test("explicitly unsupported capabilities remain zero", () => {
|
|
const result = scores({ ...model, reasoning: false, toolCall: false })
|
|
expect(result["Tool use"]).toEqual({ value: 0, fallback: "Tool calling not supported" })
|
|
expect(result.Reasoning).toEqual({ value: 0, fallback: "Reasoning not supported" })
|
|
})
|
|
|
|
test("unknown capabilities and unmatched models use neutral placeholders", () => {
|
|
const result = scores({ ...model, reasoning: undefined, toolCall: undefined })
|
|
expect(result["Tool use"]).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
|
|
expect(result.Reasoning).toEqual({ value: 50, fallback: "No data; neutral placeholder" })
|
|
expect(Object.values(scores(null))).toEqual(Array(6).fill({ value: 50, fallback: "No data; neutral placeholder" }))
|
|
})
|
|
|
|
test("measured benchmark percentiles override fallbacks, including zero", () => {
|
|
const low = {
|
|
...model,
|
|
benchmarks: [
|
|
{ name: "Tau3", score: 20 },
|
|
{ name: "GPQA", score: 40 },
|
|
],
|
|
}
|
|
const high = {
|
|
...model,
|
|
id: "other/model",
|
|
benchmarks: [
|
|
{ name: "Tau3", score: 80 },
|
|
{ name: "GPQA", score: 90 },
|
|
],
|
|
}
|
|
expect(scores(low, [low, high])["Tool use"]).toEqual({ value: 0 })
|
|
expect(scores(low, [low, high]).Reasoning).toEqual({ value: 0 })
|
|
expect(scores(high, [low, high])["Tool use"]).toEqual({ value: 100 })
|
|
expect(scores(high, [low, high]).Reasoning).toEqual({ value: 100 })
|
|
})
|
|
|
|
test("a benchmark without comparison peers retains the capability baseline", () => {
|
|
const entry = { ...model, benchmarks: [{ name: "Tau3", score: 90 }] }
|
|
expect(scores(entry, [entry])["Tool use"]).toEqual({
|
|
value: 50,
|
|
fallback: "Tool calling supported; no comparable benchmark",
|
|
})
|
|
})
|
|
})
|