{ "schema": "nodus.benchmark-claim-guide/1", "title": "Choose the benchmark for the claim", "prepared_at": "2026-11-10", "status": "source_reviewed_no_benchmarks_or_training_executed", "disclosure": "Source-based guide; no benchmarks or training experiments were executed for this guide.", "scope": "Six selected public benchmark/toolkit rows and two explicitly original proposed study designs. Not exhaustive, not an endorsement or leaderboard.", "sources": [ { "id": "kernelbench", "title": "KernelBench task and evaluation", "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/README.md", "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1" }, { "id": "kernelbench_score", "title": "KernelBench fastp implementation", "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/src/kernelbench/score.py", "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1" }, { "id": "kernelbench_analysis", "title": "KernelBench analysis: missing results and baseline alignment", "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/scripts/benchmark_eval_analysis.py", "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1" }, { "id": "meta_tritonbench", "title": "Meta/PyTorch TritonBench operator toolkit", "url": "https://github.com/meta-pytorch/tritonbench/blob/6394394849c41c23699f757b4cc2712d35ad4d86/README.md", "revision": "6394394849c41c23699f757b4cc2712d35ad4d86" }, { "id": "thunlp_tritonbench", "title": "THUNLP TritonBench generation evaluation", "url": "https://github.com/thunlp/TritonBench/blob/603e28a5050e8c268f6883a69709d477a272d49a/README.md", "revision": "603e28a5050e8c268f6883a69709d477a272d49a" }, { "id": "sol", "title": "SOL-ExecBench task, runtime and evaluation", "url": "https://github.com/NVIDIA/SOL-ExecBench/blob/a9fa0804c793d438e70850c33fe34426e66d53dd/README.md", "revision": "a9fa0804c793d438e70850c33fe34426e66d53dd" }, { "id": "sol_score", "title": "SOL anchored score implementation", "url": "https://github.com/NVIDIA/SOL-ExecBench/blob/a9fa0804c793d438e70850c33fe34426e66d53dd/src/sol_execbench/sol_score.py", "revision": "a9fa0804c793d438e70850c33fe34426e66d53dd" }, { "id": "flashinfer", "title": "FlashInfer-Bench and workload trace dataset", "url": "https://github.com/flashinfer-ai/flashinfer-bench/blob/15618a67e8d7b7f49ca61b5ba3b9f26a876d0bee/README.md", "revision": "15618a67e8d7b7f49ca61b5ba3b9f26a876d0bee" }, { "id": "flashinfer_schema", "title": "FlashInfer Trace schema", "url": "https://bench.flashinfer.ai/docs/flashinfer-trace", "revision": null, "observed_date": "2026-11-10", "version_note": "Dated public documentation, not represented as pinned to the repository commit." }, { "id": "mlperf", "title": "MLPerf Inference rules: scenarios, quality and metrics", "url": "https://github.com/mlcommons/inference_policies/blob/d3eba2f21026d868ad65cdcad2bb81e4a17ce3d3/inference_rules.adoc", "revision": "d3eba2f21026d868ad65cdcad2bb81e4a17ce3d3" } ], "rows": [ { "id": "kernelbench", "name": "KernelBench", "kind": "public_benchmark", "claim": "Generated kernels", "task_input": "A declared cohort of PyTorch programs; model writes CUDA or another supported kernel language.", "baseline": "Reference PyTorch implementation measured on the declared target.", "correctness_gate": "Reference comparison on randomized inputs under the selected checker.", "metric": "fast_p: correct entries with reference_time/candidate_time > p, divided by n.", "denominator": "Declare all cohort tasks and baseline coverage. The analysis script skips missing baseline entries; its aligned n can shrink.", "failure_handling": "Missing evaluations are patched as failures, but a task without a baseline is excluded by the inspected analysis script.", "timing_scope": "Pin GPU, dtype, runtime, timing method and trial settings.", "supports": "Correct-and-fast fraction for that task cohort and evaluation budget.", "does_not_establish": "Training-data causality, serving throughput or cloud startup performance.", "sources": [ "kernelbench", "kernelbench_score", "kernelbench_analysis" ] }, { "id": "sol", "name": "SOL-ExecBench", "kind": "public_benchmark", "claim": "Kernel performance relative to a hardware model", "task_input": "Kernel definitions and concrete workload shapes, dtypes, inputs and tolerances.", "baseline": "Optimized scoring baseline plus analytical speed-of-light estimate.", "correctness_gate": "Numerical reference checks and implemented reward-hacking checks.", "metric": "Per-workload latency and anchored SOL score. When baseline runtime exceeds the SOL estimate, the baseline anchor is 0.5 and SOL anchor is 1; the inspected code uses a separate branch for a nonpositive gap.", "denominator": "Declare problem/workload coverage, failed evaluations and aggregation. Anchors alone do not define a campaign-wide success rate.", "failure_handling": "Retain failed and missing traces; do not compute a success-only score and imply complete coverage.", "timing_scope": "Official README describes B200 with locked SM clocks; reference timing is optional and disabled by default.", "supports": "Kernel results under the declared hardware, workloads and scoring assumptions.", "does_not_establish": "A universal speedup over every production kernel, or verified training-data benefit.", "sources": [ "sol", "sol_score" ] }, { "id": "thunlp", "name": "THUNLP TritonBench", "kind": "public_benchmark", "claim": "Model generation of Triton operators", "task_input": "TritonBench-G or TritonBench-T instruction channel and declared generated solutions.", "baseline": "Channel-specific reference and efficiency pipeline.", "correctness_gate": "Call and execution accuracy; performance evaluation follows successful execution.", "metric": "Call accuracy, execution accuracy and efficiency; G also includes code similarity.", "denominator": "Report the chosen channel and all generated tasks. Keep accuracy coverage beside efficiency of executable operators.", "failure_handling": "A speed figure on executable operators does not describe failed generations.", "timing_scope": "Pin the environment and channel-specific performance scripts.", "supports": "Generation accuracy and measured operator efficiency for the selected channel.", "does_not_establish": "Meta/PyTorch TritonBench results or full-service throughput.", "sources": [ "thunlp_tritonbench" ] }, { "id": "meta_triton", "name": "Meta/PyTorch TritonBench", "kind": "public_toolkit", "claim": "Operator implementation performance", "task_input": "A selected operator, its example inputs and implementations.", "baseline": "Explicit comparison implementation; identify each operator/configuration.", "correctness_gate": "State the selected implementation's correctness coverage separately; README-level performance measurements are not a universal correctness proof.", "metric": "Operator performance under the selected benchmark configuration.", "denominator": "List every selected input case and implementation; preserve unsupported/failed cases.", "failure_handling": "Report coverage rather than averaging only the implementations that ran.", "timing_scope": "Pin PyTorch/Triton and submodules; the inspected installer defaults to latest PyTorch nightly.", "supports": "Performance of those operator implementations and inputs.", "does_not_establish": "An LLM kernel-generation pass rate or model-training improvement.", "sources": [ "meta_tritonbench" ] }, { "id": "flashinfer", "name": "FlashInfer-Bench", "kind": "public_benchmark_and_workflow", "claim": "Kernel performance on recorded workload configurations", "task_input": "Trace definitions, concrete workloads and candidate solutions.", "baseline": "The definition's reference implementation, with explicit selected workload coverage.", "correctness_gate": "Trace-schema workflow verifies solutions against the definition and records evaluation.", "metric": "Correctness and performance in evaluation traces; declare the chosen summary.", "denominator": "Distinguish workload-only entries (no solution/evaluation) from completed evaluations.", "failure_handling": "Keep workload inventory, attempts, failures and completed traces separately.", "timing_scope": "Pin framework and dataset revisions, input material, configuration and hardware.", "supports": "Measured behavior on those traced kernel workloads.", "does_not_establish": "An end-to-end serving SLO or a causal data-quality effect.", "sources": [ "flashinfer", "flashinfer_schema" ] }, { "id": "mlperf", "name": "MLPerf Inference", "kind": "public_benchmark", "claim": "Inference-system performance", "task_input": "A specified model/task, quality target and LoadGen scenario.", "baseline": "Compare declared systems within the applicable task, scenario and division.", "correctness_gate": "Benchmark-specific model quality plus scenario latency and compliance requirements.", "metric": "Server uses supported Poisson arrival rate under latency constraints; Offline reports throughput for a bulk query.", "denominator": "The complete scenario run and required samples; do not substitute one operator or one prompt.", "failure_handling": "A failed quality/latency/compliance requirement cannot be hidden by a throughput average.", "timing_scope": "System boundary and preprocessing/postprocessing follow the pinned rules.", "supports": "Scenario-specific inference-system performance when all applicable rules are met.", "does_not_establish": "Kernel-repair-data uplift, provider cold start or universal cloud ranking.", "sources": [ "mlperf" ] }, { "id": "matched_data_utility", "name": "Matched data-utility study", "kind": "nodus_proposed_protocol_not_public_benchmark", "claim": "Training on data improves a model", "task_input": "Licensed training groups and an independent frozen held-out task bank.", "baseline": "Same base model and controlled training/evaluation budget, comparing specified data conditions.", "correctness_gate": "Independent held-out evaluation; source-group separation and complete attempts.", "metric": "Paired change in prespecified held-out correctness or correct-and-fast success; uncertainty across independent groups.", "denominator": "All frozen evaluation tasks and attempts, including invalid outputs. Two repairs of one task remain one source group.", "failure_handling": "Keep failed training/evaluation attempts and source exclusions; no post-hoc held-out selection.", "timing_scope": "Fix model, data exposure, token budget, optimizer, inference budget and grader version.", "supports": "A scoped causal estimate for the actual matched experiment, if executed and valid.", "does_not_establish": "Any current Nodus uplift; this row describes a proposed evaluation, not a result.", "sources": [], "editorial_inference": true }, { "id": "provider_lifecycle", "name": "GPU lifecycle and idle-capacity study", "kind": "nodus_proposed_protocol_not_public_benchmark", "claim": "Capacity turns into accepted work", "task_input": "Fixed arrival trace, workload, deadline and declared isolated/shared GPU target.", "baseline": "Matched target and offered work, with explicit state/cache/partition treatments.", "correctness_gate": "Authenticated correct output and complete lifecycle/attempt accounting.", "metric": "Request-to-accepted-work timing, accepted work by deadline, and settled cost per accepted unit when all costs are known.", "denominator": "Every admitted attempt, failure, timeout and unserved request; declare the admission boundary.", "failure_handling": "Unknown costs and unfinished work remain visible; shared-GPU treatments require separate qualified scope.", "timing_scope": "Declare provisioning, image/model load, restore, execution, cleanup and retained state.", "supports": "Results for that measured workload, supply, treatment and time window.", "does_not_establish": "A ranking of Hyperbolic, SF Compute, Prime Intellect or Nodus without matched measurements.", "sources": [], "editorial_inference": true } ], "example": { "label": "Synthetic arithmetic illustration; no GPU or model measurement", "p": 1, "tasks": [ { "id": "a", "correct": true, "reference_ms": 3, "candidate_ms": 2 }, { "id": "b", "correct": true, "reference_ms": 2, "candidate_ms": 2 }, { "id": "c", "correct": false, "reference_ms": 4, "candidate_ms": 1 }, { "id": "d", "correct": false, "reference_ms": 2, "candidate_ms": null, "status": "missing" } ], "expected": { "n": 4, "correct": 2, "correct_and_strictly_faster": 1, "fast_1": 0.25 }, "notes": [ "The equality case is excluded because the inspected score uses strict > p.", "The missing result remains in this declared four-task denominator; all four baselines are present.", "This arithmetic does not execute KernelBench or validate a generated kernel." ] }, "commercial_boundary": "Public comparison methods are inspectable. Private training corpus, protected graders and customer records are not part of this guide.", "source_notes": [ "Repository revisions are the reviewed pins, not a claim that they will remain latest.", "Metric and denominator cautions in this guide are editorial interpretations of the inspected definitions/code.", "No source project is asserted to endorse Nodus." ] }