From ba182898daf486dc8c6482017e438c7e99c854ff Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Thu, 11 Jun 2026 22:14:39 +0200 Subject: [PATCH 1/2] feat(materialize): extract next-free loader core + versioned snapshot schema (steps 1-2) --- src/lib/materialize/load.ts | 840 ++++++++++++++++++++++++++++++++++ src/lib/materialize/schema.ts | 111 +++++ src/lib/spec.ts | 834 +-------------------------------- src/types/benchmark.ts | 12 + 4 files changed, 981 insertions(+), 816 deletions(-) create mode 100644 src/lib/materialize/load.ts create mode 100644 src/lib/materialize/schema.ts diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts new file mode 100644 index 00000000..182ae975 --- /dev/null +++ b/src/lib/materialize/load.ts @@ -0,0 +1,840 @@ +/** + * Next-free benchmark loader core. + * + * Everything here runs in BOTH contexts: the Next.js site (wrapped in + * unstable_cache layers by src/lib/spec.ts) and the standalone + * materialization worker on Railway (which imports this directly and + * must never pull next/* into its bundle). Do not import next/react + * here; persistence side effects are injected via hooks. + */ + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import yaml from "js-yaml"; +import type { + Benchmark, + CellRankEntry, + MetricPanel, + ProviderResult, +} from "@/types/benchmark"; +import { Prometheus } from "@/lib/prometheus"; +import { SpecSchema, type Spec } from "@/lib/spec-schema"; +import { renderBenchmarkText } from "@/lib/bench-template"; +import { liveResults as liveProviderResults } from "@/lib/provider-filters"; + +/** Overridable so the worker can run with a different cwd. */ +const SPECS_DIR = + process.env.OCB_SPECS_DIR ?? path.join(process.cwd(), "benchmarks"); + +/** Side-effect hooks injected by the caller (site: KV snapshot write; + * worker: store publish). Keeps this module free of persistence deps. */ +export type LoadHooks = { + /** Called with the rendered bench after a successful UNFILTERED live + * load of a live spec. Not called for filtered variants or drafts. */ + onRendered?: (b: Benchmark) => void; +}; + +export type BenchmarkFilters = { + chain?: string; + region?: string; + kind?: string; +}; + +export function filterSig(f: BenchmarkFilters): string { + // Stable ordering, ignore "all" / undefined which mean "no filter". + const parts: string[] = []; + for (const k of Object.keys(f).sort()) { + const v = (f as Record)[k]; + if (v && v !== "all") parts.push(`${k}=${v}`); + } + return parts.join("&"); +} +export function parseFilterSig(sig: string): BenchmarkFilters { + const out: BenchmarkFilters = {}; + if (!sig) return out; + for (const kv of sig.split("&")) { + const [k, v] = kv.split("="); + if (k && v && (k === "chain" || k === "region" || k === "kind")) { + out[k as "chain" | "region" | "kind"] = v; + } + } + return out; +} +export async function loadSpecsUncached(): Promise { + let files: string[] = []; + try { + files = (await fs.readdir(SPECS_DIR)).filter( + (f) => f.endsWith(".yml") || f.endsWith(".yaml") + ); + } catch { + return []; + } + const parsed = await Promise.all( + files.map(async (f) => { + const raw = await fs.readFile(path.join(SPECS_DIR, f), "utf8"); + const result = SpecSchema.safeParse(yaml.load(raw)); + if (!result.success) { + // CI catches this via `pnpm validate`. At runtime we skip and log. + console.warn(`[spec] skipping ${f}:`, result.error.message); + return null; + } + return result.data; + }) + ); + return parsed.filter((s): s is Spec => s !== null); +} + +export function buildEditorial( + spec: Spec, +): Omit { + return { + slug: spec.slug, + number: spec.number, + title: spec.title, + seoTitle: spec.seo_title, + seoDescription: spec.seo_description, + seoIntro: spec.seo_intro, + disclaimer: spec.disclaimer, + faq: spec.faq, + perChainExplainer: spec.per_chain_explainer, + subtitle: spec.subtitle, + category: spec.category, + status: spec.status, + editorialStatus: spec.status, + metric: spec.metric, + unit: spec.unit, + higherIsBetter: spec.higher_is_better, + abstract: spec.abstract, + methodology: spec.methodology, + findings: spec.findings, + source: spec.source, + dimensions: spec.dimensions, + ledgerColumns: spec.ledger_columns, + }; +} + +// Used by the per-bench cache aggregator when a single bench fully fails +// (cold start + Prom blackout, no previous cache to preserve). Renders a +// draft placeholder so the page still works. +export function draftPlaceholderForSpec(spec: Spec): Benchmark { + return draftBenchmark(spec, buildEditorial(spec)); +} + +export async function specToBenchmark( + spec: Spec, + options: BenchmarkFilters = {}, + hooks?: LoadHooks, +): Promise { + const editorial = buildEditorial(spec); + + const activeLabels = activeFilterLabels(options); + const isFiltered = Object.keys(activeLabels).length > 0; + const filteredSpec = isFiltered ? applyDimensionsToSpec(spec, activeLabels) : spec; + + const live = await tryLoadLive(filteredSpec, isFiltered); + if (live) { + // Mark live entries explicitly so a missing `availability` reads as + // "unknown" everywhere else in the code. + for (const r of live.results) r.availability = "live"; + + // Augment with spec-declared providers that didn't return data this + // cycle, but only on the *unfiltered* view. When the reader has + // applied a dimension filter (e.g. chain=bnb on rpc-capabilities) + // a no-data result almost always means the provider doesn't cover + // that dimension at all (rpc-capabilities ships ~15 providers but + // only 5 of them serve BNB; the other 10 are by-design absent on + // that tab). Surfacing those as "Currently unavailable" rows would + // pollute the leaderboard with 10 fake-offline entries and confuse + // the reader about which providers are actually broken vs which + // simply don't compete on this chain. + // + // On the unfiltered "All" tab we still augment because then a no-data + // result really does mean "harness lost this provider"; product + // pages also rely on the augmentation to stay reachable when the + // upstream is briefly down. + if (!isFiltered) { + const liveSlugs = new Set(live.results.map((r) => r.slug.toLowerCase())); + for (const p of spec.providers) { + if (liveSlugs.has(p.slug.toLowerCase())) continue; + live.results.push({ + name: p.name, + slug: p.slug, + tag: p.tag, + type: p.type, + layer: p.layer, + ms: { p50: 0, p90: 0, p99: 0, mean: 0 }, + successRate: 0, + secondary: p.secondary, + availability: "unavailable", + formula: p.formula, + }); + } + + // Companion-metric backstop. The augmentation above marks any + // provider whose headline `p50` query returned nothing as + // "unavailable". On benches like hl-frontends where the headline + // metric (effective fee bps) is legitimately empty for a builder + // that had no fees in the rolling window but still routed real + // volume, the same builder shows up live on the companion + // panels (volume, fills/min, last fill age, taker share). Marking + // it "unavailable" then is misleading: the reader switches to + // the Volume panel, sees the line, then looks at the table and + // reads "Currently unavailable" against a row that clearly has + // data. Reclassify those to "live" so the row renders as a + // first-class entry; the ledger still sorts it last because + // ms.p50=0, but it stops claiming the provider is down. + if (live.metricPanels && live.metricPanels.length > 0) { + for (const r of live.results) { + if (r.availability !== "unavailable") continue; + const slug = r.slug.toLowerCase(); + const hasPanelData = live.metricPanels.some((panel) => { + const v = panel.values?.[r.slug] ?? panel.values?.[slug]; + if (v != null && Number.isFinite(v)) return true; + const series = + panel.seriesByProvider?.[r.slug] ?? + panel.seriesByProvider?.[slug]; + return Array.isArray(series) && series.length > 0; + }); + if (hasPanelData) r.availability = "live"; + } + } + } + // Per-chain leaders/trailers: computed only on the unfiltered "All" + // view of benches that declare `dimensions.chain`. Fan out one extra + // tryLoadLive() per chain value (excluding "all") with the chain + // label injected via applyDimensionsToSpec, then pick the live + // leader + trailer for that chain. This powers the + // `{{best_name:chain:X}}` placeholders + chain-aware OG/badge + // surfaces. We deliberately don't augment unavailable providers + // here: for per-chain leader we only care which provider actually + // reported data on that chain. Failures are tolerated — a chain + // with no Prom data just doesn't show up in bestPerChain. + let bestPerChain: Record | undefined; + let worstPerChain: Record | undefined; + let providersPerChain: Record | undefined; + // Per-chain leaders/trailers/presence are computed ONLY on the + // unfiltered view. Earlier this also ran for filtered variants to + // populate {{best_name:chain:X}} in the variant's editorial copy, + // but that quadrupled Prom load per page (3 extra queries × 9 + // pre-fetched variants on benches like aggregator-head-lag). The + // filtered variants now inherit findings/faq/seoIntro from the + // aggregate via the page-level fetch in app/benchmarks/[slug]/page.tsx, + // so per-chain compute on filtered variants is no longer needed. + if (!isFiltered && spec.dimensions?.chain && spec.dimensions.chain.length > 0) { + const chainValues = spec.dimensions.chain + .map((c) => c.value) + .filter((v) => v !== "all"); + const perChainEntries = await Promise.all( + chainValues.map(async (chain) => { + const chainSpec = applyDimensionsToSpec(spec, { chain }); + const chainLive = await tryLoadLive(chainSpec, true); + if (!chainLive) { + return [chain, undefined, undefined, [] as string[]] as const; + } + for (const r of chainLive.results) r.availability = "live"; + const liveForChain = liveProviderResults(chainLive.results); + const slugs = liveForChain.map((r) => r.slug); + if (liveForChain.length === 0) { + return [chain, undefined, undefined, slugs] as const; + } + const sorted = [...liveForChain].sort((a, b) => + spec.higher_is_better ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50, + ); + return [ + chain, + sorted[0], + sorted[sorted.length - 1], + slugs, + ] as const; + }), + ); + const bests: Record = {}; + const worsts: Record = {}; + const providers: Record = {}; + for (const [chain, leader, trailer, slugs] of perChainEntries) { + if (leader) bests[chain] = leader; + if (trailer) worsts[chain] = trailer; + if (slugs.length > 0) providers[chain] = slugs; + } + if (Object.keys(bests).length > 0) bestPerChain = bests; + if (Object.keys(worsts).length > 0) worstPerChain = worsts; + if (Object.keys(providers).length > 0) providersPerChain = providers; + } + + // Exact per-cell rankings (chain × region) from the spec's single + // grouped matrix query. Failures are tolerated: badge/product + // surfaces fall back to the coarser bestPerChain path. + const cellRanks = !isFiltered ? await tryLoadCellRanks(spec) : undefined; + + // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders + // against the freshly loaded numbers so editorial text (findings, + // seo_intro, faq) never drifts from the displayed data. + const rendered = renderBenchmarkText({ + ...editorial, + ...live, + bestPerChain, + worstPerChain, + providersPerChain, + cellRanks, + }); + // Persistence is the caller's concern (site: KV snapshot write, + // worker: store publish). Only the unfiltered "All" view of a live + // spec triggers the hook; filtered variants are derived views. + if (!isFiltered && spec.status === "live") { + hooks?.onRendered?.(rendered); + } + return rendered; + } + return draftBenchmark(spec, editorial); +} + +function activeFilterLabels(opts: BenchmarkFilters): Record { + const out: Record = {}; + for (const [k, v] of Object.entries(opts)) { + if (v && v !== "all") out[k] = v; + } + return out; +} + +/** + * Run the spec's `rank_matrix_query` (one instant vector with a sample per + * (provider[, chain][, region])) and fold it into full per-cell rankings. + * + * Output keys are `|` with "all" standing in for an + * undeclared dimension. When BOTH dimensions are declared, marginal cells + * (`|all`, `all|`) are derived by averaging a provider's + * finest-cell values over the collapsed dimension — same semantics as the + * bench page's unscoped `avg(...)` headline queries. + * + * Samples whose provider label doesn't match a spec provider slug, or + * whose chain/region label isn't a declared dimension value, are dropped: + * the matrix is unfiltered PromQL, so stray series (retired providers, + * staging labels) must not leak into rankings. + */ +async function tryLoadCellRanks( + spec: Spec, +): Promise | undefined> { + if (!spec.rank_matrix_query) return undefined; + const url = spec.prometheus?.url ?? process.env.PROMETHEUS_URL; + if (!url) return undefined; + try { + const prom = new Prometheus(url); + const res = await prom.query(spec.rank_matrix_query); + if (res.resultType !== "vector") return undefined; + + const slugByLower = new Map( + spec.providers.map((p) => [p.slug.toLowerCase(), p.slug] as const), + ); + // Canonical dimension value by lowercase, so a harness emitting + // `chain="Base"` still maps onto the declared `base` value instead + // of silently dropping the cell. + const chainByLower = new Map( + (spec.dimensions?.chain ?? []) + .filter((c) => c.value !== "all") + .map((c) => [c.value.toLowerCase(), c.value] as const), + ); + const regionByLower = new Map( + (spec.dimensions?.region ?? []) + .filter((r) => r.value !== "all") + .map((r) => [r.value.toLowerCase(), r.value] as const), + ); + + // key → provider slug → samples (averaged if the grouping left + // residual label splits, e.g. multiple replicas per region). + const acc = new Map>(); + for (const sample of res.result) { + const slug = slugByLower.get((sample.metric.provider ?? "").toLowerCase()); + if (!slug) continue; + const chain = + chainByLower.size > 0 + ? chainByLower.get((sample.metric.chain ?? "").toLowerCase()) + : undefined; + const region = + regionByLower.size > 0 + ? regionByLower.get((sample.metric.region ?? "").toLowerCase()) + : undefined; + if (chainByLower.size > 0 && !chain) continue; + if (regionByLower.size > 0 && !region) continue; + const v = Number(sample.value[1]); + if (!Number.isFinite(v) || v <= 0) continue; + const key = `${chain ?? "all"}|${region ?? "all"}`; + const cell = acc.get(key) ?? new Map(); + const vals = cell.get(slug) ?? []; + vals.push(v); + cell.set(slug, vals); + acc.set(key, cell); + } + if (acc.size === 0) return undefined; + + const mean = (vals: number[]) => + vals.reduce((a, b) => a + b, 0) / vals.length; + const sortCell = (cell: Map): CellRankEntry[] => + [...cell.entries()] + .map(([slug, vals]) => ({ slug, p50: mean(vals) })) + .sort((a, b) => + spec.higher_is_better ? b.p50 - a.p50 : a.p50 - b.p50, + ); + + const out: Record = {}; + for (const [key, cell] of acc) out[key] = sortCell(cell); + + // Marginals, only when both dimensions exist in the finest cells. + // A provider only enters a marginal if it covers EVERY cell of the + // collapsed dimension that exists for that row/column. Without this, + // a provider measured only from its fastest region wins the + // `|all` average by omission (Simpson's bias), and the badge + // for "leads chain X" disagrees with the per-cell wins that earned it. + if (chainByLower.size > 0 && regionByLower.size > 0) { + const regionsOfChain = new Map>(); + const chainsOfRegion = new Map>(); + for (const key of acc.keys()) { + const [chain, region] = key.split("|"); + (regionsOfChain.get(chain) ?? regionsOfChain.set(chain, new Set()).get(chain)!).add(region); + (chainsOfRegion.get(region) ?? chainsOfRegion.set(region, new Set()).get(region)!).add(chain); + } + const marginalFor = ( + groups: Map>, + keyOf: (group: string, member: string) => string, + mKeyOf: (group: string) => string, + ) => { + for (const [group, members] of groups) { + const cell = new Map(); + // Providers present in every member cell of the group. + let eligible: Set | undefined; + for (const member of members) { + const slugs = new Set(acc.get(keyOf(group, member))?.keys() ?? []); + eligible = eligible + ? new Set([...eligible].filter((s) => slugs.has(s))) + : slugs; + } + for (const slug of eligible ?? []) { + const vals: number[] = []; + for (const member of members) { + const v = acc.get(keyOf(group, member))?.get(slug); + if (v) vals.push(mean(v)); + } + if (vals.length > 0) cell.set(slug, [mean(vals)]); + } + if (cell.size > 0) out[mKeyOf(group)] = sortCell(cell); + } + }; + marginalFor( + regionsOfChain, + (chain, region) => `${chain}|${region}`, + (chain) => `${chain}|all`, + ); + marginalFor( + chainsOfRegion, + (region, chain) => `${chain}|${region}`, + (region) => `all|${region}`, + ); + } + return out; + } catch (e) { + console.warn( + `cellRanks skip: ${spec.slug} matrix query failed: ${e instanceof Error ? e.message : String(e)}`, + ); + return undefined; + } +} + +/** Inject every active `