diff --git a/packages/stats/core/src/domain/inference.test.ts b/packages/stats/core/src/domain/inference.test.ts index 20dbe86205..4f51599d71 100644 --- a/packages/stats/core/src/domain/inference.test.ts +++ b/packages/stats/core/src/domain/inference.test.ts @@ -36,6 +36,8 @@ describe("inference stat normalization", () => { expect(modelAuthor("nemotron-3-super-free")).toBe("nvidia") expect(modelAuthor("qwen3.7-max")).toBe("qwen") expect(modelAuthor("alpha-gpt-next")).toBeUndefined() + expect(modelAuthor("omen-alpha")).toBe("unknown") + expect(modelAuthor("OMEN-ALPHA-free:global")).toBe("unknown") }) test("uses provider.model to resolve opencode route providers", () => { @@ -49,6 +51,22 @@ describe("inference stat normalization", () => { expect(statProvider("unknown", "", "custom-provider")).toBe("custom-provider") }) + test("keeps stealth model usage without exposing the route provider", () => { + expect(statProvider("omen-alpha", "gpt-test-model", "test-provider")).toBe("unknown") + expect(statProvider("OMEN-ALPHA-free:global", "gpt-test-model", "test-provider")).toBe("unknown") + expect(statProvider("omen-alpha", "", "test-provider")).toBe("unknown") + + const row = { ...aggregate("omen-alpha", "test-provider"), provider_model: "gpt-test-model" } + expect(toModelAggregate(row)).toMatchObject([{ model: "omen-alpha", provider: "unknown", requests: 1 }]) + expect(toProviderAggregate(row)).toMatchObject([{ provider: "unknown", requests: 1 }]) + expect(toGeoAggregate({ ...row, country: "US" })).toMatchObject([ + { model: "omen-alpha", provider: "unknown", country: "US", requests: 1 }, + ]) + expect(toRetentionAggregate({ ...row, cohort_date: "2026-08-10", eligible_users: "12" })).toMatchObject([ + { model: "omen-alpha", provider: "unknown", eligibleUsers: 12 }, + ]) + }) + test("merges renamed models under their current name", () => { expect(statModel("deepseek-v4-flash-0731", "")).toBe("deepseek-v4-flash") expect(statModel("deepseek-v4-flash-0731-free", "")).toBe("deepseek-v4-flash") @@ -124,6 +142,10 @@ describe("inference stat normalization", () => { }) expect(queries).toHaveLength(8) + queries.forEach((query) => { + expect(query).toContain("WHERE lower(model) NOT IN ('alpha-gpt-next')") + expect(query).toContain("CASE\n WHEN lower(model) IN ('omen-alpha') THEN 'unknown'\n") + }) expect(queries[0]).toContain("'week' AS grain") expect(queries[0]).toContain("'2026-W33' AS period_key") expect(queries[2]).toContain("'2026-08-10' AS period_key") @@ -186,6 +208,8 @@ describe("inference stat normalization", () => { expect(queries).toHaveLength(1) expect(queries[0]?.cohortDates).toEqual(["2026-08-10", "2026-08-17"]) expect(queries[0]?.query).toContain("AND product = 'go'") + expect(queries[0]?.query).toContain("AND lower(model) NOT IN ('alpha-gpt-next')") + expect(queries[0]?.query).toContain("CASE\n WHEN lower(model) IN ('omen-alpha') THEN 'unknown'\n") expect(queries[0]?.query).toContain("COUNT(*) AS model_requests") expect(queries[0]?.query).toContain("SUM(model_requests) AS total_requests") expect(queries[0]?.query).toContain("MAX(model_requests) AS max_model_requests") diff --git a/packages/stats/core/src/domain/inference.ts b/packages/stats/core/src/domain/inference.ts index 3767b7ba4d..bdba90b11f 100644 --- a/packages/stats/core/src/domain/inference.ts +++ b/packages/stats/core/src/domain/inference.ts @@ -8,6 +8,7 @@ import { MODEL_AUTHOR_RULES, MODEL_NAME_ALIASES, RETIRED_STAT_PROVIDERS, + STEALTH_MODELS, statModel, statProvider, } from "./model-normalization" @@ -483,6 +484,7 @@ function freeTierSql(tier: string, model: string) { function statProviderSql(model: string, providerModel: string, provider: string) { return `CASE + WHEN lower(${model}) IN (${[...STEALTH_MODELS].map(sqlString).join(", ")}) THEN 'unknown' ${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${providerModel}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")} ${MODEL_AUTHOR_RULES.map((item) => ` WHEN strpos(lower(${model}), ${sqlString(item.match)}) > 0 THEN ${sqlString(item.author)}`).join("\n")} WHEN ${provider} <> '' AND lower(${provider}) NOT IN (${RETIRED_STAT_PROVIDERS.map(sqlString).join(", ")}) THEN ${provider} diff --git a/packages/stats/core/src/domain/model-normalization.ts b/packages/stats/core/src/domain/model-normalization.ts index 6f2edb117d..cbceca3486 100644 --- a/packages/stats/core/src/domain/model-normalization.ts +++ b/packages/stats/core/src/domain/model-normalization.ts @@ -14,6 +14,7 @@ export const MODEL_AUTHOR_RULES = [ { match: "qwen", author: "qwen" }, ] as const export const EXCLUDED_MODELS = new Set(["alpha-gpt-next"]) +export const STEALTH_MODELS = new Set(["omen-alpha"]) export const FREE_MODELS = new Set(["gpt-5-nano", "grok-code", "big-pickle"]) export const MODEL_NAME_ALIASES: Record = { "deepseek-v4-flash-0731": "deepseek-v4-flash", @@ -47,7 +48,10 @@ export function statProvider( providerModel: string | undefined, provider: string | undefined, ) { - const modelAuthorValue = modelAuthor(statModel(model, providerModel)) + const normalized = statModel(model, providerModel) + if (STEALTH_MODELS.has(normalized)) return "unknown" + + const modelAuthorValue = modelAuthor(normalized) if (!modelAuthorValue) return undefined const providerModelAuthor = modelAuthor(providerModel)