{"openapi":"3.1.0","info":{"title":"m.recipes API","version":"3.0.0","description":"AI Inference Benchmark Platform API. Website chat and the shared inference queue have been retired. /api/v1/chat and /api/v1/infer (including all subpaths), /api/v1/node/infer/poll, /api/v1/node/infer/{id}/response, /api/v1/realtime/nodes, /api/v1/realtime/stream, /api/v1/admin/slots/online, /v1/chat/completions and /v1/models return 404. Older node clients must upgrade to stop chat polling. Benchmark jobs, /api/v1/models and independently hosted local model APIs are unchanged.","contact":{"name":"m.recipes","url":"https://m.recipes"}},"servers":[{"url":"https://m.recipes","description":"Production"}],"tags":[{"name":"Auth","description":"Authentication endpoints"},{"name":"Submissions","description":"Benchmark session management"},{"name":"Benchmarks","description":"Benchmark results"},{"name":"Leaderboard","description":"Performance rankings"},{"name":"Nodes","description":"Worker node management"},{"name":"Jobs","description":"Job queue management"},{"name":"Admin","description":"Administrative endpoints"}],"paths":{"/api/v1/cli/version":{"get":{"tags":["CLI"],"summary":"Published CLI version for an installation platform","description":"Independent of the server package, source-tree CLI version and pinned recipe executor. The client must identify its installation platform.","parameters":[{"name":"platform","in":"query","required":true,"schema":{"type":"string","enum":["linux-x86_64","linux-aarch64","darwin-x86_64","darwin-arm64","windows-x64"]}}],"responses":{"200":{"description":"Pinned published CLI version","content":{"application/json":{"schema":{"type":"object","required":["success","version"],"properties":{"success":{"type":"boolean","enum":[true]},"version":{"type":"string","pattern":"^v[0-9]+\\.[0-9]+\\.[0-9]+$"}}}}}},"400":{"description":"Installation platform is required"},"404":{"description":"No pinned release for the requested platform","content":{"application/json":{"schema":{"type":"object","required":["success","error"],"properties":{"success":{"type":"boolean","enum":[false]},"error":{"type":"string"}}}}}}}}},"/api/v1/coverage/quant-matrix":{"get":{"tags":["Configuration"],"summary":"Recorded GPU / model-family / quantization coverage","description":"tests counts distinct recorded reports, including diagnostic attempts; it is not a successful-load or quality count. best_tok_s is the highest eligible single-path text-generation rate; native non-token rates are excluded. Workloads, models and engines may differ: this is coverage, not a comparable performance ranking. GPU groups preserve name and VRAM; device count is available in individual reports. Served from a complete internal snapshot refreshed every 30 seconds off the HTTP thread. Refresh failures retain the last good snapshot for at most five minutes; no request falls back to scanning raw evidence.","responses":{"200":{"description":"matrix cells, catalog families/quants, GPU name/VRAM groups and total_cells","content":{"application/json":{"schema":{"type":"object","required":["matrix","families","gpus","total_cells"],"additionalProperties":true}}},"headers":{"X-Result-Cache-Generated-At":{"schema":{"type":"string","format":"date-time"}},"X-Result-Cache-Stale":{"description":"True when the snapshot is older than 60 seconds","schema":{"type":"string","enum":["true","false"]}}}},"500":{"description":"Coverage is unavailable; clients must show an error and not an empty matrix"},"503":{"description":"Result snapshot unavailable, expired, or exceeds complete-population bounds; Retry-After: 5. No partial ranking or on-demand scan."}}}},"/api/v1/admin/models":{"get":{"tags":["Admin"],"summary":"Authenticated read-only model catalog view","description":"Model registration, update, deletion, approval, import, discovery and VRAM-recalculation writes are retired (404). Agents update the tracked catalog and publish one immutable revision.","responses":{"200":{"description":"Canonical model rows and revision"},"403":{"description":"Platform admin required"}}}},"/api/v1/manifests/index":{"get":{"tags":["Configuration"],"summary":"Model, engine and task index from the authoritative catalog","responses":{"200":{"description":"Index and catalog revision"}}}},"/api/v1/manifests/tasks":{"get":{"tags":["Configuration"],"summary":"Canonical task descriptions","responses":{"200":{"description":"Tasks and catalog revision"}}}},"/api/v1/manifests/engines":{"get":{"tags":["Configuration"],"summary":"Canonical engine descriptions; not executable runtime setup policy","parameters":[{"name":"task","in":"query","schema":{"type":"string"}}],"responses":{"200":{"description":"Engines and catalog revision"}}}},"/api/v1/manifests/engines/{id}":{"get":{"tags":["Configuration"],"summary":"One canonical engine description","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Engine and catalog revision"},"404":{"description":"Unknown engine; no embedded fallback"}}}},"/api/v1/datasets":{"get":{"tags":["Configuration"],"summary":"Canonical dataset sources and performance prompt assets","description":"A prompt asset download_url and total_prompts match config/datasets. source_url and sample_count describe the publisher dataset when present; these are distinct from transformed performance prompts and the versioned quality evaluation protocol.","parameters":[{"name":"task","in":"query","schema":{"type":"string"}}],"responses":{"200":{"description":"Datasets and catalog revision"}}}},"/api/v1/config/datasets":{"get":{"tags":["Configuration"],"summary":"Performance prompt assets and versioned capability evaluation catalog","description":"quality_catalog is additive. ready means a Rust evaluator exists, not a completed GPU run or an official leaderboard score. Pending/access-required entries are not automatically scheduled. Repository creation dates are not publication dates. Optional release_reports lists selected evaluations from pinned official model reports; it is research evidence, not runnable coverage or reproduced publisher scores. upstream_checked_at identifies a revision check, not a new benchmark edition. Public access does not imply redistribution permission: public LiveCodeBench v6 remains pending_harness and uses an aggregate-only platform publication policy while its conflicting cc/MIT data declarations await clarification. Its release_v6 cumulative1055 and v6-only175 cohorts are distinct; exact source revisions and license evidence are supplied in the catalog. No code benchmark is scheduled until its controlled execution harness is accepted.","responses":{"200":{"description":"Dataset configuration","content":{"application/json":{"schema":{"type":"object","required":["success","version","datasets","quality_catalog"],"properties":{"success":{"type":"boolean"},"version":{"type":"string"},"datasets":{"type":"object","additionalProperties":true},"quality_catalog":{"type":"object","required":["schema","verified_at","datasets"],"properties":{"schema":{"type":"string"},"verified_at":{"type":"string"},"date_basis":{"type":"string"},"release_reviewed_at":{"type":"string","format":"date"},"review_note":{"type":"string"},"review_note_zh":{"type":"string"},"release_reports":{"type":"array","items":{"type":"object","required":["id","name","source_url","revision","benchmarks","protocol_note","protocol_note_zh"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"source_url":{"type":"string","format":"uri"},"revision":{"type":"string","pattern":"^[a-f0-9]{40}$"},"benchmarks":{"type":"array","items":{"type":"string"}},"protocol_note":{"type":"string"},"protocol_note_zh":{"type":"string"},"evidence_urls":{"type":"array","items":{"type":"string","format":"uri"}}},"additionalProperties":true}},"datasets":{"type":"array","items":{"type":"object","required":["id","name","category","status","source_url","revision","access","license"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"category":{"type":"string"},"status":{"type":"string","enum":["ready","pending_scorer","pending_harness","access_required"]},"source_url":{"type":"string","format":"uri"},"revision":{"type":"string"},"access":{"type":"string","enum":["public","gated"]},"license":{"type":"string"},"license_review":{"type":"object","description":"Publisher declaration and available license files verified at the pinned revision; asset rights are separate. Not a universal redistribution grant.","properties":{"checked_at":{"type":"string","format":"date"},"scope":{"type":"string"},"public_content":{"type":"string","enum":["aggregate_only","source_terms","asset_review_required"]},"note":{"type":"string"},"note_zh":{"type":"string"},"evidence":{"type":"array","items":{"type":"object","properties":{"url":{"type":"string","format":"uri"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"}},"additionalProperties":true}}},"additionalProperties":true},"repository_created_at":{"type":"string","nullable":true},"edition":{"type":"string"},"upstream_checked_at":{"type":"string","format":"date"},"upstream_last_modified":{"type":"string","format":"date-time"},"cli_dataset":{"type":"string","nullable":true},"note":{"type":"string"},"note_zh":{"type":"string"}},"additionalProperties":true}}}}},"additionalProperties":true}}}}}}},"/api/v1/benchmarks/{id}/profiling":{"get":{"tags":["Profiling"],"summary":"All captured profiling levels","description":"Full by default, including artifact-backed traces streamed in bounded chunks. Optional max_samples requests actual observed previews with total/returned counts; bottleneck summary remains full-trace. Captured matrix evidence is used when a separate profiling row is absent.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"max_samples","in":"query","schema":{"type":"integer","minimum":1,"maximum":1000}}],"responses":{"200":{"description":"Captured profiling evidence","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"},"profiling":{"type":"array","items":{"type":"object","additionalProperties":true,"properties":{"concurrency":{"type":"integer","minimum":1},"profiler_type":{"type":"string"},"sample_interval_ms":{"type":"integer"},"samples":{"type":"array","items":{"type":"object","additionalProperties":true}},"bottleneck":{"type":"object","additionalProperties":true},"sample_storage":{"type":"object","additionalProperties":true,"required":["schema","total_samples","report","preview"],"properties":{"schema":{"type":"string","enum":["wbench.profiling.v1"]},"total_samples":{"type":"integer","minimum":1,"maximum":5000000},"report":{"$ref":"#/components/schemas/BenchmarkArtifact"},"preview":{"$ref":"#/components/schemas/BenchmarkArtifact"}}},"sampling":{"type":"object","properties":{"method":{"type":"string","enum":["uniform-index-v1"]},"total_samples":{"type":"integer"},"returned_samples":{"type":"integer"}}}}}}}}}}},"400":{"description":"Invalid preview limit"},"404":{"description":"No captured trace for this level"},"409":{"description":"Missing/corrupt/inconsistent evidence; never substitute an empty trace"},"500":{"description":"Database operation failed"},"503":{"description":"Artifact storage unavailable"}}},"post":{"tags":["Profiling"],"summary":"Store captured profiling or validated artifact references","description":"Legacy samples arrays remain supported. Large clients send empty samples plus sample_storage wbench.profiling.v1 referencing complete JSON and a <=1000-point preview in the existing 4 MiB SHA256 artifact store. Do not mix inline samples and storage. Client limit: 2 GiB/five million samples; reference preview is checked before storing.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":true,"properties":{"concurrency":{"type":"integer","minimum":1},"profiler_type":{"type":"string"},"sample_interval_ms":{"type":"integer"},"samples":{"type":"array","items":{"type":"object","additionalProperties":true}},"bottleneck":{"type":"object","additionalProperties":true},"sample_storage":{"type":"object","additionalProperties":true,"required":["schema","total_samples","report","preview"],"properties":{"schema":{"type":"string","enum":["wbench.profiling.v1"]},"total_samples":{"type":"integer","minimum":1,"maximum":5000000},"report":{"$ref":"#/components/schemas/BenchmarkArtifact"},"preview":{"$ref":"#/components/schemas/BenchmarkArtifact"}}},"sampling":{"type":"object","properties":{"method":{"type":"string","enum":["uniform-index-v1"]},"total_samples":{"type":"integer"},"returned_samples":{"type":"integer"}}}}}}}},"responses":{"200":{"description":"Stored"},"400":{"description":"Invalid samples/concurrency"},"404":{"description":"No captured trace for this level"},"409":{"description":"Missing/corrupt/inconsistent evidence; never substitute an empty trace"},"500":{"description":"Database operation failed"},"503":{"description":"Artifact storage unavailable"}}}},"/api/v1/benchmarks/{id}/profiling/{concurrency}":{"get":{"tags":["Profiling"],"summary":"Captured trace for one level","description":"Full by default. max_samples requests at most 1000 actual observed points with explicit total/returned counts; no synthesized points. Use full-trace summary rather than preview means for bottleneck inference.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"concurrency","in":"path","required":true,"schema":{"type":"integer","minimum":1}},{"name":"max_samples","in":"query","schema":{"type":"integer","minimum":1,"maximum":1000}}],"responses":{"200":{"description":"Captured profiling evidence","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"},"profiling":{"type":"object","additionalProperties":true,"properties":{"concurrency":{"type":"integer","minimum":1},"profiler_type":{"type":"string"},"sample_interval_ms":{"type":"integer"},"samples":{"type":"array","items":{"type":"object","additionalProperties":true}},"bottleneck":{"type":"object","additionalProperties":true},"sample_storage":{"type":"object","additionalProperties":true,"required":["schema","total_samples","report","preview"],"properties":{"schema":{"type":"string","enum":["wbench.profiling.v1"]},"total_samples":{"type":"integer","minimum":1,"maximum":5000000},"report":{"$ref":"#/components/schemas/BenchmarkArtifact"},"preview":{"$ref":"#/components/schemas/BenchmarkArtifact"}}},"sampling":{"type":"object","properties":{"method":{"type":"string","enum":["uniform-index-v1"]},"total_samples":{"type":"integer"},"returned_samples":{"type":"integer"}}}}}}}}}},"400":{"description":"Invalid preview limit"},"404":{"description":"No captured trace for this level"},"409":{"description":"Missing/corrupt/inconsistent evidence; never substitute an empty trace"},"500":{"description":"Database operation failed"},"503":{"description":"Artifact storage unavailable"}}}},"/api/v1/benchmarks/{id}/profiling/{concurrency}/download":{"get":{"tags":["Profiling"],"summary":"Download every captured profiling sample","description":"Complete JSON, independent of chart preview. Artifact-backed originals stream SHA-verified 4 MiB chunks; missing later chunks fail the stream rather than producing valid partial JSON. X-Content-SHA256 identifies original bytes for client verification. Legacy inline traces download in full.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"concurrency","in":"path","required":true,"schema":{"type":"integer","minimum":1}}],"responses":{"200":{"description":"Complete original profiling JSON","headers":{"X-Content-SHA256":{"description":"Artifact-backed original SHA256; absent for legacy inline data","schema":{"type":"string"}}},"content":{"application/json":{"schema":{"type":"object","additionalProperties":true,"properties":{"concurrency":{"type":"integer","minimum":1},"profiler_type":{"type":"string"},"sample_interval_ms":{"type":"integer"},"samples":{"type":"array","items":{"type":"object","additionalProperties":true}},"bottleneck":{"type":"object","additionalProperties":true},"sample_storage":{"type":"object","additionalProperties":true,"required":["schema","total_samples","report","preview"],"properties":{"schema":{"type":"string","enum":["wbench.profiling.v1"]},"total_samples":{"type":"integer","minimum":1,"maximum":5000000},"report":{"$ref":"#/components/schemas/BenchmarkArtifact"},"preview":{"$ref":"#/components/schemas/BenchmarkArtifact"}}},"sampling":{"type":"object","properties":{"method":{"type":"string","enum":["uniform-index-v1"]},"total_samples":{"type":"integer"},"returned_samples":{"type":"integer"}}}}}}}},"404":{"description":"No captured trace for this level"},"409":{"description":"Missing/corrupt/inconsistent evidence; never substitute an empty trace"},"500":{"description":"Database operation failed"},"503":{"description":"Artifact storage unavailable"}}}},"/api/v1/hardware":{"get":{"tags":["Leaderboard"],"summary":"Compact hardware summary or one hardware detail","parameters":[{"name":"quant","in":"query","description":"Exact recorded format, case-insensitive; applied before aggregation.","schema":{"type":"string","maxLength":64}},{"name":"hardware","in":"query","description":"Hardware key from the summary. Omit for homepage summaries.","schema":{"type":"string","maxLength":180,"pattern":"^[a-z0-9-]+$"}},{"name":"case","in":"query","description":"Requires hardware. Select a measured case; comparisons include only its family.","schema":{"type":"string","pattern":"^sha256:[a-f0-9]{64}$"}},{"name":"If-None-Match","in":"header","schema":{"type":"string"},"description":"Revalidate this public view with its ETag."}],"description":"The default summary contains only hardware identity, record_count, a verified project pick, tied largest models, total_records and the five latest report links. It excludes full records, measurements, prices and comparison protocols. Set hardware to fetch that hardware detail, its measured cases and comparison measurements only for the selected case family. case selects a measured configuration; without it, the project pick's measured case is selected when available. The frontend consumes this single server selection owner. Case and family keys use sha256:<hex>. Snapshots and response views each have a 60-second server cache; browser max-age is 60 seconds; generated_at records when the snapshot was built. Complete population up to 10000 reports; returns 503 rather than partial ranking beyond that bound. Identity preserves GPU variants, memory capacity and device count. detail.records.model_type and inference_verified describe recorded task type and whether any successful inference request was observed; listing alone is not launch proof. detail.records.model_size is null for unresolved or ambiguous identifiers, otherwise {total_params_b,active_params_b,source:catalog,model_id}, a declared catalog count rather than measurement of checkpoint tensors. Optional main_params_b/auxiliary_params_b disclose main-network versus auxiliary embedding counts when upstream separates them. Pinned parameter_evidence supplies reviewed legacy operator aliases independently of executable/download model entries. Exact approved model IDs/provider IDs/names and recognized HF or ModelScope cache paths are resolved; name number guessing and VRAM fit estimates are not used. Largest-model views consider only inference_verified LLM/VL records and sort known total parameters; unknown counts or offload placement remain unknown. detail.records.model_recommendation is null unless successful LLM/VL inference resolves to an exact curated catalog model ID; otherwise it contains {family,basis:project-priority,priority}. Lower priority numbers are preferred editorial project choices, not measured quality ranks or evidence of model fit on other hardware. Model recommendations are maintained in the catalog database alongside executable catalog entries; performance and quality still require matching recorded configurations. Cases separate recorded model/engine/precision/method/workload/length/dataset/software settings, including full nested engine_provenance when supplied (observed runtime versions, source revision and driver evidence); cases.engine_provenance is optional and cases.software may contain observed runtime evidence. Unknown historical runtime versions are not inferred. Only error-free non-fallback measurements have averages; legacy records stay inspectable. The additive detail.unclassified_measurements array retains positive error-free non-fallback LLM observations whose method is missing, with record ID, concurrency, rate, unit and matching power; these are evidence only, never comparable cases or averages. Quality averages require identical recorded run fingerprints in addition to dataset/model/engine/precision; missing fingerprints remain individual records. Seller quotes are snapshots, not market averages or completed sales, and expire for default valuation after seven days. Served from a complete internal snapshot refreshed every 30 seconds off the HTTP thread. Refresh failures retain the last good snapshot for at most five minutes; no request falls back to scanning raw evidence.","responses":{"200":{"description":"Summary excludes detail data; detail contains one hardware and the selected comparison family.","content":{"application/json":{"schema":{"type":"object","required":["schema","view","generated_at","hardware","families","methodology"],"properties":{"schema":{"type":"string","enum":["wbench.hardware.v3"]},"view":{"type":"string","enum":["summary","detail"]},"generated_at":{"type":"string"},"methodology":{"type":"string"},"quantizations":{"type":"array","items":{"type":"string"}},"quant_filter":{"type":"string","nullable":true},"families":{"type":"array","items":{"type":"string"}},"total_records":{"type":"integer"},"latest":{"type":"array","maxItems":5,"items":{"type":"object","required":["r","hardware_key"],"additionalProperties":true}},"hardware":{"type":"array","items":{"type":"object","required":["key","name","label","vram_gb","gpu_count","record_count","pick","largest"],"properties":{"key":{"type":"string"},"name":{"type":"string"},"label":{"type":"string"},"vram_gb":{"type":"number"},"gpu_count":{"type":"integer"},"record_count":{"type":"integer"},"pick":{"type":"object","nullable":true,"additionalProperties":true,"description":"Selected record, own speed, first quality group, quality_count, family and variants."},"largest":{"type":"array","items":{"type":"object","additionalProperties":true},"description":"Tied largest verified catalog models with their own evidence; not a quality ranking."}}}},"detail":{"type":"object","required":["key","records","measurements","quality","prices"],"additionalProperties":true},"cases":{"type":"array","items":{"type":"object","additionalProperties":true}},"selected_case":{"type":"string","nullable":true},"comparisons":{"type":"array","items":{"type":"object","required":["key","measurements"],"additionalProperties":true}}}}}},"headers":{"X-Result-Cache-Generated-At":{"schema":{"type":"string","format":"date-time"}},"X-Result-Cache-Stale":{"description":"True when the snapshot is older than 60 seconds","schema":{"type":"string","enum":["true","false"]}}}},"304":{"description":"ETag matched; no response body"},"400":{"description":"Invalid filter or case key"},"404":{"description":"Hardware has no records in this scope"},"503":{"description":"Result snapshot unavailable, expired, or exceeds complete-population bounds; Retry-After: 5. No partial ranking or on-demand scan."}}}},"/api/v1/benchmarks/{id}/average":{"get":{"summary":"Mean quality score and paginated constituent runs for an exact configuration","description":"Groups by quality suite, model path, precision, engine, GPU name/memory/count, tensor parallel size, method, modality and run fingerprint. Mean covers all matching scored runs, not top-N. Errors and truncation remain in run scores; missing scores are counted separately. Legacy records without fingerprints return 409.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"offset","in":"query","schema":{"type":"integer","minimum":0,"default":0}}],"responses":{"200":{"description":"Mean and all contributing records through 50-row pagination","content":{"application/json":{"schema":{"type":"object","required":["reference_id","configuration","run_fingerprint","aggregation","records","scored_records","unscored_records","average","minimum","maximum","offset","limit","items"],"properties":{"reference_id":{"type":"string"},"dataset":{"type":"object","nullable":true,"additionalProperties":true,"description":"Dataset completeness and sample provenance; null for unknown historical provenance"},"configuration":{"type":"object","additionalProperties":true},"run_fingerprint":{"type":"string"},"aggregation":{"type":"string"},"records":{"type":"integer"},"scored_records":{"type":"integer"},"unscored_records":{"type":"integer"},"average":{"type":"number","nullable":true},"minimum":{"type":"number","nullable":true},"maximum":{"type":"number","nullable":true},"offset":{"type":"integer"},"limit":{"type":"integer"},"items":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"submitted_at":{"type":"string"},"score":{"type":"number","nullable":true},"request_errors":{"type":"integer","nullable":true},"truncated":{"type":"integer","nullable":true},"samples":{"type":"integer","nullable":true}}}}}}}}},"400":{"description":"Invalid offset"},"404":{"description":"Benchmark not found"},"409":{"description":"Missing comparable quality configuration"}}}},"/api/v1/coverage/check":{"post":{"tags":["Benchmarks"],"summary":"Check exact LLM configuration coverage without reserving work","description":"global_checkpoint is the public opt-in fill preflight: a model is covered only when its entire canonical matrix or native variant matches current required data fields, including tensor-issue-rate observations and text phase/context/search evidence. Missing any cell or field means rerun the whole model. Source session/device/job never limits fill lookup. With cloud_job_id, requires the active node token/device/nonce. resume_only restricts references to the interrupted shared job so default full runs never reuse unrelated historical scores. Matches hardware, model revision, engine, quantization and workload; failed/partial/incompatible evidence never suppresses work. Lookup failure must stop recovery, not start duplicate measurements. Original provenance remains with each report. Without cloud_job_id: opt-in fill-empty-first. Matches GPU name/rounded VRAM/count/TP, remote model ID, quantization, engine, workload, context, duration and every requested concurrency. Requires successful zero-error reference results and GPU samples. Historical missing fields do not cover; latest 100 matching records only. Does not compare engine versions or reserve concurrent work.","requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","anyOf":[{"required":["cloud_job_id"]},{"required":["global_checkpoint","hardware","autotest_identity"]},{"required":["gpu_name","gpu_vram_gb","gpu_count","tensor_parallel_size","model_path","quant","engine","context_name","input_tokens","output_tokens","test_duration_sec","dataset","model_type","bench_method","concurrency"]}],"properties":{"gpu_name":{"type":"string","minLength":1,"maxLength":512},"model_path":{"type":"string","minLength":1,"maxLength":512},"quant":{"type":"string","minLength":1,"maxLength":512},"engine":{"type":"string","minLength":1,"maxLength":512},"context_name":{"type":"string","minLength":1,"maxLength":512},"dataset":{"type":"string","minLength":1,"maxLength":512},"model_type":{"type":"string","minLength":1,"maxLength":512},"bench_method":{"type":"string","minLength":1,"maxLength":512},"gpu_vram_gb":{"type":"number","exclusiveMinimum":0},"gpu_count":{"type":"integer","minimum":1},"tensor_parallel_size":{"type":"integer","minimum":1},"input_tokens":{"type":"integer","minimum":0},"output_tokens":{"type":"integer","minimum":1},"test_duration_sec":{"type":"integer","minimum":0},"run_fingerprint":{"type":"string","pattern":"^[a-f0-9]{64}$","description":"Exact engine/config/workload fingerprint; required for quality coverage."},"kernel_options":{"type":"object","additionalProperties":true,"description":"Exact explicit engine options for an interrupted matrix cell."},"global_checkpoint":{"type":"boolean"},"hardware":{"type":"object","properties":{"gpu_name":{"type":"string"},"gpu_vram_gb":{"type":"number"},"gpu_count":{"type":"integer","enum":[1]}}},"resume_only":{"type":"boolean","description":"Only reference measurements from this interrupted shared job; otherwise global fill lookup."},"cloud_job_id":{"type":"string","format":"uuid"},"step":{"type":"object","additionalProperties":true},"model_revision":{"type":"string"},"autotest_identity":{"type":"object","additionalProperties":true},"quality_suite":{"type":"string","description":"Required for quality-v1; exact dataset/sample/scorer/generation suite."},"quality_item_count":{"type":"integer","minimum":1,"description":"Required for quality-v1; complete zero-error, untruncated scored item count."},"concurrency":{"type":"array","minItems":1,"maxItems":64,"items":{"type":"integer","minimum":1}}}}}}},"responses":{"200":{"description":"Exact matching evidence or missing coverage","content":{"application/json":{"schema":{"type":"object","required":["success","covered","benchmark_id"],"properties":{"success":{"type":"boolean"},"covered":{"type":"boolean"},"benchmark_id":{"type":["string","null"]},"checkpoint":{"type":"object","nullable":true,"additionalProperties":true}}}}}},"400":{"description":"Invalid configuration"},"401":{"description":"Cloud node authentication required"},"409":{"description":"Cloud lease lost"},"500":{"description":"Coverage unavailable; client must not interpret as covered"},"503":{"description":"Checkpoint evidence unavailable; do not repeat measurements"}}}},"/api/v1/models/next-task":{"get":{"tags":["Models"],"summary":"Next uncovered task by project priority and verified release","description":"Requires gpu_name and gpu_vram_gb. Filters by engine, task_type and memory fit. Orders project priority, verified family publication date, then family and weight format; unknown dates last. No GPU-native format is inferred when SM is absent. Completed cells (three records) do not block older unfinished cells.","responses":{"200":{"description":"Task and coverage summary, or null task when no eligible work remains"},"400":{"description":"Missing GPU parameters"}}}},"/api/v1/models/suggested":{"get":{"tags":["Models"],"summary":"Hardware-compatible models by project priority and verified release","description":"Requires gpu_name and gpu_vram_gb. Uses the shared project-priority and verified family publication policy before coverage tie-breaks; no repository timestamp proxy. Unknown dates last. Limit is applied after ranking (1–200, default 20). Memory, task and engine filters remain in effect.","responses":{"200":{"description":"Ranked model list and coverage"},"400":{"description":"Missing GPU parameters"}}}},"/api/v1/models":{"get":{"tags":["Models"],"summary":"Canonical model catalog with runtime-specific metadata","description":"Projection of /api/v1/catalog from the authoritative model catalog database, independent of historical SQL rows. Default LLM/VL selection preserves legacy autorun compatibility. Use type=embedding or type=all. type=all retains explicitly blocked entries with their reasons; default runnable views exclude them. source and revision identify the maintained catalog. Missing metadata stays unknown.","parameters":[{"name":"type","in":"query","schema":{"type":"string"}},{"name":"engine","in":"query","schema":{"type":"string"}}],"responses":{"200":{"description":"Approved catalog; embedding_config includes pinned revision, backend, dimensions, modalities, evaluation protocol, autotest variants/blocked reasons and unverified source recipes","content":{"application/json":{"schema":{"type":"object","properties":{"models":{"type":"array","items":{"type":"object","properties":{"id":{"type":"string"},"engine":{"type":"string"},"embedding_dim":{"type":"integer"},"embedding_config":{"type":"object","additionalProperties":true}}}},"count":{"type":"integer"}}}}}}}}},"/api/v1/recipes/embedding/{benchmarkId}":{"get":{"tags":["Benchmarks"],"summary":"Versioned executable embedding recipe","description":"Read-only and idempotent. Exact report, batch and stream selection; never chooses a changing latest/best configuration. v1 supports verified native Sentence Transformers CUDA graph measurements. Report shell text is never executed. Consumers tolerate unknown fields and reject unknown schema versions. ETag supports conditional reads. A recipe records a measured protocol; it does not promise identical performance on different hardware. Program is null until its immutable executor is published.","parameters":[{"name":"benchmarkId","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"batch_size","in":"query","required":true,"schema":{"type":"integer","minimum":1,"maximum":1024}},{"name":"request_concurrency","in":"query","schema":{"type":"integer","minimum":1,"maximum":32,"default":1}},{"name":"If-None-Match","in":"header","schema":{"type":"string"}}],"responses":{"200":{"description":"Pinned executable configuration with program checksum","headers":{"ETag":{"schema":{"type":"string"}}},"content":{"application/json":{"schema":{"type":"object","required":["schema_version","configuration","program","execution"],"properties":{"schema_version":{"type":"string","enum":["wbench.embedding.recipe.v1"]},"configuration":{"type":"object","required":["benchmark_id","model","model_revision","engine","precision","dimensions","max_sequence_length","batch_size","request_concurrency","workload_mode","graph_mode","reference_eligible","software","quality","duration_sec","iterations","gpu"],"properties":{"benchmark_id":{"type":"string","format":"uuid"},"model":{"type":"string"},"model_revision":{"type":"string","pattern":"^[a-f0-9]{40}$"},"engine":{"type":"string","enum":["sentence-transformers"]},"weight_quantization":{"type":"string","enum":["none","int8","nf4"],"description":"Weight representation, independent of compute precision"},"precision":{"type":"string","enum":["fp32","fp16","bf16"]},"dimensions":{"type":"integer","minimum":1,"maximum":32768},"max_sequence_length":{"type":"integer","minimum":1,"maximum":131072},"batch_size":{"type":"integer","minimum":1,"maximum":1024},"request_concurrency":{"type":"integer","enum":[1]},"workload_mode":{"type":"string","enum":["single"]},"graph_mode":{"type":"string","enum":["captured"]},"reference_eligible":{"type":"boolean","enum":[true]},"duration_sec":{"type":"number","minimum":0,"maximum":3600},"iterations":{"type":"integer","minimum":1,"maximum":10000},"software":{"type":"object","required":["torch","transformers","sentence-transformers"],"additionalProperties":{"type":"string"}},"quality":{"type":"array","minItems":1,"items":{"type":"object","required":["suite","score","dataset"],"properties":{"suite":{"type":"string"},"score":{"type":"number"},"dataset":{"type":"object","required":["id","revision","full_run","size","files"],"properties":{"id":{"type":"string"},"revision":{"type":"string"},"full_run":{"type":"boolean","enum":[true]},"size":{"type":"integer","minimum":1},"files":{"type":"array","minItems":1,"items":{"type":"object","required":["path","sha256"],"properties":{"path":{"type":"string"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"}}}}}}}}},"gpu":{"type":"string"},"report_url":{"type":"string"},"performance_scope":{"type":"string"}}},"program":{"type":"object","nullable":true,"required":["version","source_revision","platform","url","sha256"],"properties":{"supports_environment_setup":{"type":"boolean","description":"Executor verifies/reuses compatible environments, requests consent before isolated installation, and supports service start --reuse --wait."},"supports_embedding_service":{"type":"boolean"},"supports_managed_services":{"type":"boolean"},"supports_star_setup":{"type":"boolean"},"supported_weight_quantizations":{"type":"array","items":{"type":"string","enum":["none","int8","nf4"]}},"supports_pinned_model_launches":{"type":"boolean","description":"Published executor accepts immutable model revisions and explicit tool-call parsers for managed vLLM launches."},"max_embedding_batch_size":{"type":"integer","minimum":1,"maximum":1024,"description":"Largest embedding recipe batch supported by this immutable executor. Older manifests without this field support at most 128."},"version":{"type":"string"},"source_revision":{"type":"string","pattern":"^[a-f0-9]{40}$"},"platform":{"type":"string","enum":["linux-x86_64"]},"url":{"type":"string","format":"uri"},"gzip_url":{"type":"string","format":"uri"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"}}},"deployment":{"type":"object","description":"Start an authenticated native embedding service from the measured model/software configuration; live API throughput is not the fixed-input graph benchmark","properties":{"service_configuration":{"type":"object","description":"Typed background-service configuration; see /service-config.schema.json","required":["schema_version","id","host","port","runtime"],"properties":{"schema_version":{"type":"string","enum":["wbench.service.config.v1"]},"id":{"type":"string"},"host":{"type":"string"},"port":{"type":"integer","minimum":1,"maximum":65535},"runtime":{"type":"object","required":["engine","benchmark_id","batch_size"],"properties":{"engine":{"type":"string","enum":["embedding-recipe"]},"benchmark_id":{"type":"string"},"batch_size":{"type":"integer"}}}}},"available":{"type":"boolean"},"arguments":{"type":"array","items":{"type":"string"}},"protocol":{"type":"string","enum":["openai-embeddings"]},"authentication":{"type":"string","enum":["bearer-token-file"]},"default_host":{"type":"string"},"default_port":{"type":"integer"},"configuration_url":{"type":"string"},"reference_report_url":{"type":"string"},"graph_mode":{"type":"string","enum":["eager_fallback"]},"reference_eligible":{"type":"boolean","enum":[false]},"scope":{"type":"string"},"reason":{"type":"string","nullable":true}}},"execution":{"type":"object","required":["executor","arguments","automatic_setup"],"properties":{"executor":{"type":"string","enum":["mr","w-bench"]},"arguments":{"type":"array","items":{"type":"string"}},"automatic_setup":{"type":"boolean","enum":[true]}}}}}}}},"304":{"description":"Recipe unchanged"},"400":{"description":"Invalid selector","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EmbeddingRecipeError"}}}},"404":{"description":"Report or workload not found","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EmbeddingRecipeError"}}}},"422":{"description":"Unverified, incomplete or unsupported configuration","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EmbeddingRecipeError"}}}},"503":{"headers":{"Retry-After":{"schema":{"type":"string"},"description":"Seconds before a bounded retry"}},"description":"Temporary storage failure; retry at most twice","content":{"application/json":{"schema":{"$ref":"#/components/schemas/EmbeddingRecipeError"}}}}}}},"/api/v1/recipes/decision/{benchmarkId}":{"get":{"tags":["recipes"],"summary":"Measured native decision API and benchmark recipe","parameters":[{"name":"benchmarkId","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"batch_size","in":"query","required":true,"schema":{"type":"integer","minimum":1,"maximum":1024}}],"responses":{"200":{"description":"Immutable model/software configuration; deployment depends on the published executor. Native throughput is not HTTP throughput.","content":{"application/json":{"schema":{"type":"object","required":["schema_version","configuration","configuration_sha256","program","execution","deployment"],"properties":{"schema_version":{"type":"string","enum":["wbench.decision.recipe.v1"]},"configuration_sha256":{"type":"string"},"configuration":{"type":"object","required":["benchmark_id","task","model","model_revision","engine","precision","batch_size","python_version","software"],"properties":{"benchmark_id":{"type":"string"},"task":{"type":"string","enum":["decision"]},"model":{"type":"string"},"model_revision":{"type":"string"},"engine":{"type":"string"},"precision":{"type":"string"},"batch_size":{"type":"integer","minimum":1},"request_concurrency":{"type":"integer","minimum":1},"max_sequence_length":{"type":"integer","minimum":1},"python_version":{"type":"string"},"software":{"type":"object","additionalProperties":{"type":"string"}},"engine_provenance":{"type":"object"},"gpu":{"type":"string"},"gpu_vram_gb":{"type":"number"},"duration_sec":{"type":"number"},"iterations":{"type":"integer","minimum":1}}},"program":{"type":"object","nullable":true,"additionalProperties":true},"execution":{"type":"object","properties":{"available":{"type":"boolean"},"arguments":{"type":"array","items":{"type":"string"}},"automatic_setup":{"type":"boolean"},"reason":{"type":"string","nullable":true}}},"deployment":{"type":"object","properties":{"available":{"type":"boolean"},"protocol":{"type":"string"},"authentication":{"type":"string"},"arguments":{"type":"array","items":{"type":"string"}},"service_configuration":{"type":"object"},"configuration_url":{"type":"string"},"reference_report_url":{"type":"string"},"scope":{"type":"string"},"reason":{"type":"string","nullable":true}}}}}}}},"400":{"description":"Typed recipe error with error.code, error.message and error.retryable. Incomplete historical metadata may include an alternative for the same immutable model revision and GPU, with its own runtime, batch and measurements."},"404":{"description":"Typed recipe error with error.code, error.message and error.retryable. Incomplete historical metadata may include an alternative for the same immutable model revision and GPU, with its own runtime, batch and measurements."},"422":{"description":"Typed recipe error with error.code, error.message and error.retryable. Incomplete historical metadata may include an alternative for the same immutable model revision and GPU, with its own runtime, batch and measurements."},"503":{"description":"Typed recipe error with error.code, error.message and error.retryable. Incomplete historical metadata may include an alternative for the same immutable model revision and GPU, with its own runtime, batch and measurements."}}}},"/api/v1/recipes/llm/{benchmarkId}":{"get":{"tags":["Benchmarks"],"summary":"Pinned vLLM model API and optional benchmark recipe","description":"Read-only exact report and concurrency selection. Pins model revision, recorded package versions, startup settings, MoE requests and cache policy. Prefer runtime.startup typed context/max_num_seqs from suite-startup-settings; legacy fixed suites derive scheduling from their full measured concurrency list. Reject 1.1.55 recipe reruns without this evidence. Does not execute recorded shell commands. Missing or excluded measurements remain unavailable. program is null until a compatible immutable executor is published. Optional configuration.performance_task pins the generation task and the exact calibrated prompt SHA256, verified before warmup or measured requests; execution.available additionally requires the published executor to declare its exact task ID/SHA256 in supported_performance_tasks. Deployment only consumes runtime settings. Unknown response fields are tolerated; unknown schema versions are rejected. Configuration SHA256 binds recursively key-sorted compact JSON; ETag supports conditional reads.","parameters":[{"name":"benchmarkId","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"concurrency","in":"query","required":true,"schema":{"type":"integer","minimum":1,"maximum":256}}],"responses":{"200":{"description":"Measured settings, immutable executor and typed service configuration","content":{"application/json":{"schema":{"type":"object","required":["schema_version","configuration","configuration_sha256","program","deployment","execution","limitations"],"properties":{"schema_version":{"type":"string","enum":["wbench.llm.recipe.v1"]},"configuration_sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"configuration":{"type":"object","required":["benchmark_id","model","model_revision","quant","run_fingerprint","concurrency","input_tokens","output_tokens","iterations","cache_mode","runtime"],"properties":{"benchmark_id":{"type":"string","format":"uuid"},"model":{"type":"string"},"model_revision":{"type":"string","pattern":"^[a-f0-9]{40}$"},"quant":{"type":"string"},"gpu":{"type":"string"},"gpu_vram_gb":{"type":"number"},"run_fingerprint":{"type":"string","pattern":"^[a-f0-9]{64}$"},"concurrency":{"type":"integer"},"input_tokens":{"type":"integer"},"output_tokens":{"type":"integer"},"performance_task":{"type":"object","required":["id","sha256","prompt_sha256"],"properties":{"id":{"type":"string"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"prompt_sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"}}},"iterations":{"type":"integer"},"cache_mode":{"type":"string","enum":["isolated","prefix-reuse"]},"runtime":{"type":"object","required":["engine","model","model_revision","vllm_version","software","dtype","context","tensor_parallel_size","max_num_seqs","gpu_memory_utilization","timeout_sec"],"properties":{"engine":{"type":"string","enum":["vllm"]},"model":{"type":"string"},"model_revision":{"type":"string","pattern":"^[a-fA-F0-9]{40}$"},"vllm_version":{"type":"string","enum":["0.28.0","0.29.0"]},"software":{"type":"object","required":["vllm","torch","transformers","triton","tokenizers"],"additionalProperties":{"type":"string","pattern":"^[A-Za-z0-9][A-Za-z0-9.+_-]{0,79}$"}},"dtype":{"type":"string","enum":["auto"]},"context":{"type":"integer","minimum":1,"maximum":1048576},"tensor_parallel_size":{"type":"integer","enum":[1]},"max_num_seqs":{"type":"integer","minimum":1,"maximum":256},"gpu_memory_utilization":{"type":"number","minimum":0,"exclusiveMinimum":true,"maximum":1},"timeout_sec":{"type":"integer","enum":[1800]},"reasoning_parser":{"type":"string","pattern":"^[A-Za-z0-9_]{1,64}$"},"moe_backend":{"type":"string","enum":["auto","marlin","humming"]},"humming_input_dtype":{"type":"string","enum":["fp8","int8"],"description":"Recorded request; not per-layer dtype attestation"}}}}},"program":{"type":"object","nullable":true,"properties":{"version":{"type":"string"},"platform":{"type":"string"},"url":{"type":"string","format":"uri"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"supports_llm_recipes":{"type":"boolean"},"supported_recipe_vllm_versions":{"type":"array","items":{"type":"string"}},"supported_performance_tasks":{"type":"array","items":{"type":"object","required":["id","sha256"],"properties":{"id":{"type":"string"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"}}}}}},"deployment":{"type":"object","required":["available","arguments","service_configuration","reason"],"properties":{"available":{"type":"boolean"},"arguments":{"type":"array","items":{"type":"string"}},"service_configuration":{"type":"object","required":["schema_version","id","host","port","reference_report_id","runtime"],"properties":{"schema_version":{"type":"string","enum":["wbench.service.config.v1"]},"id":{"type":"string"},"host":{"type":"string","enum":["127.0.0.1"]},"port":{"type":"integer","enum":[8000]},"reference_report_id":{"type":"string","format":"uuid"},"runtime":{"type":"object","required":["engine","model","model_revision","vllm_version","software","dtype","context","tensor_parallel_size","max_num_seqs","gpu_memory_utilization","timeout_sec"],"properties":{"engine":{"type":"string","enum":["vllm"]},"model":{"type":"string"},"model_revision":{"type":"string","pattern":"^[a-fA-F0-9]{40}$"},"vllm_version":{"type":"string","enum":["0.28.0","0.29.0"]},"software":{"type":"object","required":["vllm","torch","transformers","triton","tokenizers"],"additionalProperties":{"type":"string","pattern":"^[A-Za-z0-9][A-Za-z0-9.+_-]{0,79}$"}},"dtype":{"type":"string","enum":["auto"]},"context":{"type":"integer","minimum":1,"maximum":1048576},"tensor_parallel_size":{"type":"integer","enum":[1]},"max_num_seqs":{"type":"integer","minimum":1,"maximum":256},"gpu_memory_utilization":{"type":"number","minimum":0,"exclusiveMinimum":true,"maximum":1},"timeout_sec":{"type":"integer","enum":[1800]},"reasoning_parser":{"type":"string","pattern":"^[A-Za-z0-9_]{1,64}$"},"moe_backend":{"type":"string","enum":["auto","marlin","humming"]},"humming_input_dtype":{"type":"string","enum":["fp8","int8"],"description":"Recorded request; not per-layer dtype attestation"}}}}},"reason":{"type":"string","nullable":true}}},"execution":{"type":"object","properties":{"available":{"type":"boolean"},"arguments":{"type":"array","items":{"type":"string"}},"automatic_setup":{"type":"boolean"},"reason":{"type":"string","nullable":true}}},"limitations":{"type":"array","items":{"type":"string"}}}}}}},"304":{"description":"Unchanged recipe"},"400":{"description":"Invalid report UUID or concurrency","content":{"application/json":{"schema":{"type":"object","required":["schema_version","error"],"properties":{"schema_version":{"type":"string","enum":["wbench.llm.recipe.v1"]},"error":{"type":"object","required":["code","message","retryable"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"404":{"description":"Report or measured concurrency not found","content":{"application/json":{"schema":{"type":"object","required":["schema_version","error"],"properties":{"schema_version":{"type":"string","enum":["wbench.llm.recipe.v1"]},"error":{"type":"object","required":["code","message","retryable"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"422":{"description":"Unsupported, incomplete or excluded measurement","content":{"application/json":{"schema":{"type":"object","required":["schema_version","error"],"properties":{"schema_version":{"type":"string","enum":["wbench.llm.recipe.v1"]},"error":{"type":"object","required":["code","message","retryable"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"503":{"description":"Temporary storage failure; retry at most twice","content":{"application/json":{"schema":{"type":"object","required":["schema_version","error"],"properties":{"schema_version":{"type":"string","enum":["wbench.llm.recipe.v1"]},"error":{"type":"object","required":["code","message","retryable"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}},"headers":{"Retry-After":{"schema":{"type":"integer"}}}}}}},"/api/v1/recipes/resolve":{"get":{"tags":["recipes"],"summary":"Resolve a model name or Hugging Face repository to an eligible measured recipe or a verified owned runtime / explicit default text-model launch","parameters":[{"name":"model","in":"query","required":true,"schema":{"type":"string","maxLength":256},"description":"Exact OWNER/MODEL or an unambiguous model name; quantized repositories stay distinct"},{"name":"gpu_name","in":"query","schema":{"type":"string","maxLength":256},"description":"Observed GPU name; throughput is never transferred between GPUs"}],"responses":{"200":{"description":"Task/revision-preserving resolution. Default model launches explicitly have measured=false.","content":{"application/json":{"schema":{"type":"object","required":["schema_version","kind","model","selection_basis"],"properties":{"schema_version":{"type":"string","enum":["wbench.model.launch.v1"]},"kind":{"type":"string","enum":["recipe","model"]},"model":{"type":"string"},"recipe_id":{"type":"string"},"reference_gpu":{"type":"string"},"hardware_match":{"type":"boolean"},"measured":{"type":"boolean"},"selection_basis":{"type":"string"},"runtime_build":{"type":"object","properties":{"url":{"type":"string","format":"uri"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"source_revision":{"type":"string"}}},"service_configuration":{"type":"object","required":["schema_version","id","host","port","runtime"],"properties":{"schema_version":{"type":"string","enum":["wbench.service.config.v1"]},"id":{"type":"string"},"host":{"type":"string","enum":["127.0.0.1"]},"port":{"type":"integer","enum":[8000]},"runtime":{"type":"object","required":["engine","model","model_revision","context","dtype","gpu_memory_utilization","tensor_parallel_size","timeout_sec"],"properties":{"engine":{"type":"string","enum":["vllm"]},"model":{"type":"string"},"model_revision":{"type":"string","pattern":"^[a-f0-9]{40}$"},"context":{"type":"integer","minimum":1,"maximum":8192},"dtype":{"type":"string","enum":["auto","bfloat16"]},"gpu_memory_utilization":{"type":"number","enum":[0.8,0.9]},"tensor_parallel_size":{"type":"integer","enum":[1]},"timeout_sec":{"type":"integer","enum":[1800]},"vllm_version":{"type":"string"},"task":{"type":"string","enum":["embed"]},"text_only":{"type":"boolean"},"max_num_seqs":{"type":"integer","minimum":1}}}}}}}}}},"400":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"},"403":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"},"404":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"},"409":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"},"422":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"},"502":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"},"503":{"description":"Typed model-resolution error with error.code/message/retryable and optional ambiguous candidates"}}}},"/api/v1/releases/{filename}":{"get":{"tags":["recipes"],"summary":"Download an immutable program or owned runtime build from m.recipes","parameters":[{"name":"filename","in":"path","required":true,"schema":{"type":"string","pattern":"^(?:mr-(?:(?:linux-(?:x86_64|aarch64)|darwin-(?:x86_64|arm64))-v\\d+\\.\\d+\\.\\d+|windows-x64-v\\d+\\.\\d+\\.\\d+\\.exe)|vllm-\\d+\\.\\d+\\.\\d+\\+mrecipes\\.\\d+(?:-cp\\d+-abi3-manylinux_2_28_x86_64\\.whl|\\.build\\.json))(?:\\.sha256)?$"}}],"responses":{"200":{"description":"Exact binary, build receipt or SHA256 sidecar; public immutable cache and ETag","content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}},"text/plain":{"schema":{"type":"string"}},"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"206":{"description":"Single byte range"},"304":{"description":"Matching ETag"},"404":{"description":"Release not published"},"416":{"description":"Invalid byte range"}}},"head":{"tags":["recipes"],"summary":"Inspect an immutable program download","parameters":[{"name":"filename","in":"path","required":true,"schema":{"type":"string","pattern":"^(?:mr-(?:(?:linux-(?:x86_64|aarch64)|darwin-(?:x86_64|arm64))-v\\d+\\.\\d+\\.\\d+|windows-x64-v\\d+\\.\\d+\\.\\d+\\.exe)|vllm-\\d+\\.\\d+\\.\\d+\\+mrecipes\\.\\d+(?:-cp\\d+-abi3-manylinux_2_28_x86_64\\.whl|\\.build\\.json))(?:\\.sha256)?$"}}],"responses":{"200":{"description":"Exact binary, build receipt or SHA256 sidecar; public immutable cache and ETag","content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}},"text/plain":{"schema":{"type":"string"}},"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"206":{"description":"Single byte range"},"304":{"description":"Matching ETag"},"404":{"description":"Release not published"},"416":{"description":"Invalid byte range"}}}},"/api/v1/recipes/embedding/{benchmarkId}/setup":{"post":{"tags":["Benchmarks"],"summary":"Create a one-time user-owned model API and Star setup","description":"Requires the signed-in browser session and an exact same-origin Origin header. Forwards only the encrypted session-owned W-ID access token to Star. Setup codes expire in ten minutes and can be consumed once by the local executor. Responses and copied scripts are private; codes must never be put in configuration URLs or logs. This POST is not retried automatically.","parameters":[{"name":"benchmarkId","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"batch_size","in":"query","required":true,"schema":{"type":"integer","minimum":1,"maximum":128}},{"name":"request_concurrency","in":"query","schema":{"type":"integer","enum":[1],"default":1}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","additionalProperties":false,"required":["host","port","star_tunnel"],"properties":{"host":{"type":"string","enum":["127.0.0.1","0.0.0.0"]},"port":{"type":"integer","minimum":1,"maximum":65535},"star_tunnel":{"type":"boolean","enum":[true]}}}}}},"responses":{"200":{"description":"Private one-time startup configuration","content":{"application/json":{"schema":{"type":"object","required":["schema_version","program","configuration","authorization"],"properties":{"membership":{"type":"object","properties":{"tier":{"type":"string","enum":["member"]},"source":{"type":"string","enum":["login-promotion"]},"persistent_tunnels":{"type":"boolean"},"expires_at":{"nullable":true,"type":"string","format":"date-time"}}},"schema_version":{"type":"string","enum":["wbench.service.setup.v1"]},"program":{"type":"object","required":["source_revision","url","sha256"],"properties":{"source_revision":{"type":"string","pattern":"^[a-f0-9]{40}$"},"url":{"type":"string","format":"uri"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"supports_star_setup":{"type":"boolean"}}},"configuration":{"type":"object","required":["schema_version","id","host","port","runtime","tunnel"],"properties":{"schema_version":{"type":"string","enum":["wbench.service.config.v1"]},"id":{"type":"string"},"host":{"type":"string"},"port":{"type":"integer"},"runtime":{"type":"object","properties":{"engine":{"type":"string","enum":["embedding-recipe"]},"benchmark_id":{"type":"string","format":"uuid"},"batch_size":{"type":"integer"}}},"tunnel":{"type":"object","properties":{"provider":{"type":"string","enum":["star"]},"control_url":{"type":"string","format":"uri"}}}}},"authorization":{"type":"object","required":["setup_code","expires_at"],"properties":{"setup_code":{"type":"string","minLength":32,"maxLength":256},"expires_at":{"type":"string","format":"date-time"}}}}}}}},"400":{"description":"Invalid configuration","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"401":{"description":"Login or W-ID renewal required","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"403":{"description":"Origin or user permission rejected","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"404":{"description":"Missing report or workload","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"422":{"description":"Unsupported measured configuration","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"502":{"description":"Invalid Star response","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}},"503":{"description":"Authorization or executor unavailable","content":{"application/json":{"schema":{"type":"object","properties":{"schema_version":{"type":"string"},"error":{"type":"object","required":["code","message"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"}}}}}}}}}}},"/api/v1/catalog":{"get":{"tags":["Configuration"],"summary":"Authoritative online model and native GPU capability catalog","description":"One revision binds model metadata and reviewed native tensor formats. CLI clients query this endpoint before automatic selection. Model and GPU specification endpoints are compatibility views of the same maintained sources. Unknown capabilities stay unknown. GPU formats do not attest checkpoint quantization or engine kernel support. Explicit offline snapshots must be labeled; clients must not silently mix online metadata with older bundled values.","parameters":[{"name":"section","in":"query","schema":{"type":"string","enum":["all","models","gpu","engines","datasets","tasks","model_releases"],"default":"all"}}],"responses":{"200":{"description":"Catalog snapshot; tolerate unknown fields, reject unknown schema versions","content":{"application/json":{"schema":{"type":"object","required":["schema","revision"],"properties":{"schema":{"type":"string","enum":["wbench.catalog.v1"]},"revision":{"type":"string","pattern":"^[a-f0-9]{64}$"},"dispatch_policy":{"type":"object","additionalProperties":true,"description":"Server-owned automatic test order: total model parameters ascending, unknown sizes last. A policy change updates catalog revision; workers refresh between model matrices."},"models":{"type":"array","items":{"type":"object","required":["id","engine","model_type","quant"],"properties":{"runtime_policy":{"type":"object","additionalProperties":true,"description":"Optional vllm_version and hardware_profiles: hardware key to a candidate attention_backend, kv_cache_dtype, max_num_batched_tokens map. Shared admission creates independent baseline and candidate tasks; these are experiments, not claims of speedup or quality."}},"additionalProperties":true}},"performance_workload":{"type":"object","additionalProperties":true,"description":"Default text workload, including the versioned generation task and corpus. Task changes invalidate queue/recovery identity; actual output and stop reasons remain observed evidence."},"throughput_search":{"type":"object","description":"Immutable executor search policy, also bound into the cloud workload. Native batch and request concurrency are separate axes; adapters may impose tighter bounds or declare unsupported.","required":["protocol","max_batch","max_concurrency","time_budget_seconds","repetitions","min_measurement_seconds","refine_top_k","capacity_budget_fraction"],"properties":{"protocol":{"type":"string","enum":["throughput-search-v1"]},"max_batch":{"type":"integer","minimum":1,"maximum":1024},"max_concurrency":{"type":"integer","minimum":1,"maximum":256},"time_budget_seconds":{"type":"number","minimum":1,"maximum":14400},"repetitions":{"type":"integer","minimum":2,"maximum":10},"min_measurement_seconds":{"type":"number","minimum":0.01,"maximum":60},"refine_top_k":{"type":"integer","minimum":1,"maximum":4},"capacity_budget_fraction":{"type":"number","minimum":0.01,"maximum":0.99}},"additionalProperties":true},"model_releases":{"type":"object","description":"Database-owned exact release lineage and aliases; repository creation is not publication date.","additionalProperties":true},"model_family_metadata":{"type":"object","description":"Reviewed base-family release evidence keyed by exact family ID; absent means unknown. Dates are not derivative checkpoint release dates.","additionalProperties":{"type":"object","required":["release_date","release_date_scope","source_url","verified_at"],"additionalProperties":true}},"model_recommendations":{"type":"array","description":"Editorial project priorities, lower first; not measured capability rankings. Exact model IDs identify each quantized checkpoint.","items":{"type":"object","required":["family","basis","priority","model_ids"],"additionalProperties":true}},"gpu_capabilities":{"type":"array","items":{"type":"object","required":["vendor","compute_capabilities","formats","source_urls","checked_at"],"additionalProperties":true}},"engines":{"type":"array","items":{"type":"object","required":["id","name"],"additionalProperties":true}},"tasks":{"type":"array","items":{"type":"object","required":["id","name"],"additionalProperties":true}},"datasets":{"type":"array","items":{"type":"object","required":["id","name","task"],"additionalProperties":true}},"dataset_config":{"type":"object","additionalProperties":true}},"additionalProperties":true}}}},"304":{"description":"Unchanged catalog revision"},"400":{"description":"Unknown section"}}}},"/api/v1/nodes/{device_code}/catalog-jobs":{"post":{"tags":["Nodes"],"summary":"Join the shared, server-ranked performance queue","security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]}],"description":"Authenticated node proposes exact locally compatible IDs. Server validates catalog, published Linux executor, registered GPU, adapters and scope, creates the catalog-performance slot and returns the canonical hardware key. Automatic selection requires a verified base-family publication date with UTC calendar age 0 through 99 days; older, future and unknown dates appear in excluded_models. Exact model_filter bypasses only the publication-age rule, preserving manual runs of old models. Age is rechecked on every atomic claim, including previously admitted queues. Orders positive finite total params_b ascending; unknown/zero/invalid sizes go last. Equal sizes use editorial priority, verified family release and native quantization within a family. Active model matrices retain their lease and immutable snapshot; refreshed queued work uses the current order. Admission jobs retain catalog/executor fences. A stable coverage_key ignores catalog-wide and CLI-version changes while retaining hardware, model revision, execution configuration and workload. execution_mode defaults to full: execute assigned models regardless of historical scores. fill skips a model only after checking the entire global matrix against current required fields; any missing cell or required field reruns the entire model. Current vLLM performance requests use fixed-token-count-v1 output control; early EOS and missing output-control evidence cannot cover a model. An active equivalent lease prevents concurrent duplicate claims across modes. Historical successful job receipts alone are not proof of current coverage. Claims are nonce-fenced and restricted to the admitted candidates; generic slot writes invalidate that admission. A configured hardware_profiles candidate creates separate baseline and hardware-profile jobs with distinct coverage keys; server-selected runtime_policy is fixed for each lease. One whole model performance matrix per job; at most two transient retries. No quality tests. Does not replace an active lease.","parameters":[{"name":"device_code","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["slot_id","catalog_revision","executor_version","eligible_models"],"properties":{"execution_mode":{"type":"string","enum":["full","fill"],"default":"full"},"slot_id":{"type":"string","enum":["catalog-performance"]},"catalog_revision":{"type":"string","pattern":"^[a-f0-9]{64}$"},"executor_version":{"type":"string"},"eligible_models":{"type":"array","minItems":1,"maxItems":1000,"uniqueItems":true,"items":{"type":"string","maxLength":512}},"scope":{"type":"string","enum":["all","llm","embedding","image","asr","tts","decision"]},"model_filter":{"type":"string","nullable":true,"description":"Exact explicit model selection bypasses automatic publication age; hardware, source and adapter checks still apply."}},"additionalProperties":true}}}},"responses":{"200":{"description":"Admitted slot, canonical hardware identity, exact workload and shared jobs","content":{"application/json":{"schema":{"type":"object","required":["success","hardware_key","catalog_revision","executor_version","eligible_models","workload","jobs","selection_owner"],"properties":{"excluded_models":{"type":"array","description":"Automatic candidates excluded by publication age or missing verified date; may be empty.","items":{"type":"object","required":["model_id","reason","release_date"],"properties":{"model_id":{"type":"string"},"reason":{"type":"string"},"release_date":{"type":"string","format":"date","nullable":true}},"additionalProperties":true}}},"additionalProperties":true}}}},"400":{"description":"Invalid proposal, model, adapter, hardware or scope"},"401":{"description":"Node authentication required"},"403":{"description":"Node not authorized"},"409":{"description":"Refresh catalog/executor/hardware or finish the active lease"},"413":{"description":"Proposal exceeds 256000 bytes, including streamed requests"}}}},"/api/v1/benchmarks/running":{"get":{"tags":["Leaderboard"],"summary":"Recently reported running benchmark sessions","description":"Public sessions and catalog.performance cloud queue status from the existing submissions/jobs database. Sessions are limited to updates within 24 hours; jobs to cloud batches created within 30 days. Neither running jobs nor queue completion counts are scores. Unknown model/format remains null. Expired leases have a120second delivery grace, then server recovery requeues them or archives exhausted attempts. Sessions without accepted heartbeats/activity for30minutes are failed with partial results retained. Expired leases are never running. No node credentials, nonces or private asset fields are exposed. Additive jobs and queues fields are consumed by the hardware leaderboard.","parameters":[{"name":"hardware","in":"query","schema":{"type":"string","maxLength":180,"pattern":"^[a-z0-9-]+$"}}],"responses":{"200":{"description":"Running sessions, newest first; unknown totals have no percentage.","content":{"application/json":{"schema":{"type":"object","required":["schema","generated_at","sessions"],"properties":{"schema":{"type":"string","enum":["wbench.running.v1"]},"generated_at":{"type":"string","format":"date-time"},"sessions":{"type":"array","maxItems":100,"items":{"type":"object","required":["id","status","hardware","model","quant","engine","started_at","updated_at","stale","uploaded_results"],"properties":{"id":{"type":"string"},"status":{"type":"string","enum":["running"]},"hardware":{"type":"object","additionalProperties":true},"model":{"type":"string","nullable":true},"quant":{"type":"string","nullable":true},"engine":{"type":"string","nullable":true},"context":{"type":"string","nullable":true},"started_at":{"type":"string"},"updated_at":{"type":"string"},"stale":{"type":"boolean"},"uploaded_results":{"type":"integer","minimum":0}}}},"jobs":{"type":"array","maxItems":100,"items":{"type":"object","additionalProperties":true,"description":"Active catalog jobs: id, model, hardware, status running|awaiting_reassignment, started_at (nullable; current lease start, excluding queue wait), updated_at, submission_id (nullable)."}},"queues":{"type":"array","items":{"type":"object","additionalProperties":true,"description":"Per hardware: hardware_key, queued, running, awaiting_reassignment, reported, succeeded, failed, exhausted and total. Reported/exhausted jobs are not necessarily successful measurements."}}}}}}},"400":{"description":"Invalid hardware key"},"503":{"description":"Live status unavailable or exceeds its complete-population bound"}}}},"/api/v1/gpu/specifications":{"get":{"tags":["Benchmarks"],"summary":"Downloadable GPU specification catalog with field-level provenance","description":"Pinned community snapshot plus reviewed vendor-field overlays. Separate desktop/mobile, capacity and PCIe/SXM identities. Missing values are null. memory_clock_mhz is physical clock as reported by the community; memory_effective_gbps is a separately sourced per-pin effective rate. Tensor TFLOPS are dense only. Official review applies only to official_fields, never to the entire row. Rental machines are dated user instance quotations; optional offers are dated public GPU-hour rate cards with their own currencies, tiers, source URLs and unverified availability. No automatic exchange conversion. Unresolved GPU variants have null specification_id. Observed virtual GPU does not infer a physical SKU. JSON includes field_sources; CSV is a flat projection. No measured inference performance is inferred. native_tensor_formats at the catalog root is the reviewed vendor/compute-capability format table shared with CLI automatic selection. Per-GPU native_tensor_formats derives from that row's compute_capability (provenance remains separate); null means unknown. The CLI uses the actually reported CUDA capability and prioritizes native NVFP4, then native FP8 among model/engine-compatible candidates; weight-only emulation never implies native support.","parameters":[{"name":"format","in":"query","schema":{"type":"string","enum":["json","csv"],"default":"json"}}],"responses":{"200":{"description":"Versioned catalog or CSV","content":{"application/json":{"schema":{"type":"object","required":["schema","updated_at","upstream","units","gpus","rental_quotes"],"properties":{"schema":{"type":"string","enum":["wbench.gpu.specifications.v1"]},"updated_at":{"type":"string","format":"date"},"upstream":{"type":"object","additionalProperties":true},"units":{"type":"object","additionalProperties":true},"native_tensor_formats":{"type":"array","items":{"type":"object","required":["vendor","compute_capabilities","formats","source_urls","checked_at"],"properties":{"vendor":{"type":"string"},"compute_capabilities":{"type":"array","items":{"type":"string"}},"formats":{"type":"array","items":{"type":"string","enum":["NVFP4","FP8","BF16","FP16"]}},"source_urls":{"type":"array","items":{"type":"string","format":"uri"}},"checked_at":{"type":"string","format":"date"}},"additionalProperties":true}},"rental_quotes":{"type":"object","additionalProperties":true},"gpus":{"type":"array","items":{"type":"object","required":["id","name","vendor","field_sources","official_fields"],"additionalProperties":true}}},"additionalProperties":true}},"text/csv":{"schema":{"type":"string"}}}},"400":{"description":"Unsupported download format"}}}},"/api/v1/gpu/rental-quotes":{"get":{"summary":"Dated instance quotations and public GPU rate cards","description":"Small companion to GPU specifications; not a live market feed. machines retains user-provided total-instance reserved-hour CNY quotations. Optional offers contains public per-GPU-hour prices with individual CNY/USD currency, cloud/member tier, source URL, quotation timestamp, billing_unit, price_basis and availability=not_checked. Consumers tolerate unknown fields; no exchange conversion, automatic instance purchase or performance prediction. specification_id is null when capacity/form-factor identity is unresolved. sources includes link-only providers whose prices are not redistributed.","responses":{"200":{"description":"Instance quotations plus optional attributed public offers","content":{"application/json":{"schema":{"type":"object","required":["currency","quoted_at","source","machines"],"properties":{"currency":{"type":"string"},"quoted_at":{"type":"string","format":"date"},"source":{"type":"string"},"machines":{"type":"array","items":{"type":"object","additionalProperties":true}},"offers_updated_at":{"type":"string","format":"date-time"},"offers":{"type":"array","items":{"type":"object","required":["id","provider","tier","currency","gpu_name","specification_id","nominal_vram_gb","hourly_cost","quoted_at","source_url","billing_unit","price_basis","availability"],"properties":{"currency":{"type":"string","enum":["CNY","USD"]},"specification_id":{"type":"string","nullable":true},"hourly_cost":{"type":"number","exclusiveMinimum":0},"billing_unit":{"type":"string","enum":["gpu_hour"]},"quoted_at":{"type":"string","format":"date-time"},"source_url":{"type":"string","format":"uri"}},"additionalProperties":true}},"sources":{"type":"array","items":{"type":"object","additionalProperties":true}}},"additionalProperties":true}}}}}}},"/api/v1/gpu/specifications/license":{"get":{"summary":"Pinned upstream GPU data license","responses":{"302":{"description":"Immutable upstream Apache-2.0 license URL"}}}},"/api/v1/embedding/configurations":{"get":{"tags":["Benchmarks"],"summary":"Embedding quality, throughput and reproducible configurations","description":"Separate single request streams from concurrent requests. Native batching remains single. Recommendations require complete matching quality, immutable model revision, captured graph and zero errors; grouped by model, revision, GPU, engine, compute precision, weight quantization, dimensions, max length, workload mode and suites. Diagnostic results retain exclusion reasons. Latest 500 reports only.","parameters":[{"name":"model","in":"query","schema":{"type":"string"}},{"name":"benchmark_id","in":"query","schema":{"type":"string"},"description":"Exact report identity for reproducible recipes, including reports beyond the latest 500"}],"responses":{"200":{"description":"Measured configurations, explicit eligibility and recommendations","content":{"application/json":{"schema":{"type":"object","properties":{"configurations":{"type":"array","items":{"type":"object","additionalProperties":true}},"limit":{"type":"integer"}}}}}}}}},"/api/v1/datasets/vl":{"get":{"summary":"Download the requested VL dataset without synthetic fallback","parameters":[{"name":"name","in":"query","schema":{"type":"string","default":"mmmu_lite"},"description":"Legacy mmmu_lite is a ScienceQA subset, not MMMU. ocr_bench selects OCR-Bench."}],"responses":{"200":{"description":"Nonempty dataset with name, version, source, total and items"},"503":{"description":"Requested dataset is missing, empty or invalid; no substitute score","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean","const":false},"error":{"type":"string"}}}}}}}}},"/api/v1/images/upload":{"post":{"tags":["Benchmarks"],"summary":"Idempotent account image upload","description":"One image ID per authenticated owner, benchmark, model name, prompt and file SHA256. The associated image report must belong to the same owner. Files and lookup rows must both persist before success. Retrying identical multipart data does not duplicate gallery entries.","parameters":[{"name":"X-API-Key","in":"header","required":true,"schema":{"type":"string"}},{"name":"Content-Length","in":"header","required":true,"schema":{"type":"integer","minimum":1,"maximum":33570816}}],"requestBody":{"required":true,"content":{"multipart/form-data":{"schema":{"type":"object","required":["image"],"properties":{"image":{"type":"string","format":"binary","description":"PNG, JPEG or WebP; 1 byte to 32 MiB."},"benchmark_id":{"type":"string"},"model_name":{"type":"string","maxLength":200},"prompt":{"type":"string","maxLength":1000}}}}}},"responses":{"200":{"description":"Stored image ID, URL and verified file SHA256"},"400":{"description":"Missing or unsupported image"},"401":{"description":"API key required"},"403":{"description":"Report owned by another user"},"404":{"description":"Image report not found"},"413":{"description":"Missing or excessive body length"},"503":{"description":"Image storage unavailable"}}}},"/api/v1/image-benchmarks":{"get":{"summary":"Image results; opt into verified official-reference comparisons with mode=formal","tags":["Benchmarks"],"parameters":[{"name":"mode","in":"query","schema":{"type":"string","enum":["formal","all"]},"description":"Default all preserves historical access. Formal excludes smoke tests and unverified legacy results."},{"name":"gpu","in":"query","schema":{"type":"string"}},{"name":"model","in":"query","schema":{"type":"string"}},{"name":"model_type","in":"query","schema":{"type":"string"}},{"name":"limit","in":"query","schema":{"type":"integer","minimum":1,"maximum":200}}],"responses":{"200":{"description":"Image benchmark rows"}}}},"/api/v1/image-stats":{"get":{"summary":"Image benchmark counts and aggregate throughput","tags":["Benchmarks"],"parameters":[{"name":"mode","in":"query","schema":{"type":"string","enum":["formal","all"]},"description":"Uses the same scope as the image results list; default all."}],"responses":{"200":{"description":"Counts and aggregates for the selected scope"}}}},"/api/v1/submissions/{id}/image-benchmark":{"put":{"summary":"Store an immutable image report and reuse its ID on identical retries","tags":["Benchmarks"],"description":"The report ID is the SHA256 of the canonical submission ID, benchmark and client_version. Identical retries reuse it; changed evidence gets a new report. The submission must exist and its recorded owner must match when present.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["benchmark"],"properties":{"client_version":{"type":"string"},"benchmark":{"type":"object","required":["model_path","resolution","num_inference_steps","guidance_scale","results"],"additionalProperties":true,"properties":{"model_path":{"type":"string","minLength":1},"resolution":{"type":"string","pattern":"^[0-9]+x[0-9]+$"},"num_inference_steps":{"type":"integer","minimum":1},"guidance_scale":{"type":"number","minimum":0},"results":{"type":"array","minItems":1,"maxItems":128,"items":{"type":"object","required":["batch_size","images_per_sec","latency_avg_sec","successful_images","failed_images"],"additionalProperties":true,"properties":{"batch_size":{"type":"integer","minimum":1},"images_per_sec":{"type":"number","minimum":0},"latency_avg_sec":{"type":"number","minimum":0},"successful_images":{"type":"integer","minimum":0},"failed_images":{"type":"integer","minimum":0},"metadata":{"type":"object","additionalProperties":true,"properties":{"generation":{"$ref":"#/components/schemas/ImageGeneration"}}}}}}}}}}}}},"responses":{"200":{"description":"Benchmark stored or identical existing report reused"},"400":{"description":"Invalid or incomplete image measurements"},"403":{"description":"Forbidden"},"404":{"description":"Submission not found"}}}},"/api/v1/image-benchmarks/{id}":{"get":{"summary":"Cached image report with one decoded result set and original output manifests","tags":["Benchmarks"],"description":"Returns results once, omitting results_json and private ownership fields. Supports public short caching and ETag revalidation.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Public image benchmark report","content":{"application/json":{"schema":{"type":"object","additionalProperties":true,"properties":{"results":{"type":"array","items":{"type":"object","additionalProperties":true,"properties":{"batch_size":{"type":"integer"},"images_per_sec":{"type":"number"},"metadata":{"type":"object","additionalProperties":true,"properties":{"resource_lifecycle":{"$ref":"#/components/schemas/ResourceLifecycle"},"generation":{"$ref":"#/components/schemas/ImageGeneration"},"resources":{"$ref":"#/components/schemas/ResourceSummary"},"generated_images":{"type":"array","items":{"$ref":"#/components/schemas/GeneratedImage"}}}}}}}}}}}},"404":{"description":"Report not found"}}}},"/api/v1/benchmark-media/{hash}":{"put":{"tags":["Benchmarks"],"summary":"Publish a verified WAV manifest after all chunks exist (maximum 128 MiB audio)","operationId":"putBenchmarkMedia","parameters":[{"name":"hash","in":"path","required":true,"schema":{"type":"string","pattern":"^[a-f0-9]{64}$"}},{"name":"X-Device-Code","in":"header","required":true,"schema":{"type":"string"}},{"name":"X-Node-Token","in":"header","schema":{"type":"string"}},{"name":"X-API-Key","in":"header","schema":{"type":"string"}},{"name":"Content-Length","in":"header","required":true,"schema":{"type":"integer","minimum":1,"maximum":16384}}],"requestBody":{"required":true,"content":{"application/octet-stream":{"schema":{"type":"string","format":"binary","description":"UTF-8 JSON with filename, content_type=audio/wav, sha256, bytes, chunks and chunk_sizes; path hash verifies these exact JSON bytes"}}}},"responses":{"200":{"description":"Manifest verified and published"},"400":{"description":"Invalid manifest or WAV"},"403":{"description":"Unauthorized"},"409":{"description":"Missing chunk or wrong size; not published"},"413":{"description":"Manifest too large"},"503":{"description":"Storage unavailable"}}},"get":{"tags":["Benchmarks"],"summary":"Stream immutable WAV audio with single byte ranges for seeking","operationId":"getBenchmarkMedia","parameters":[{"name":"hash","in":"path","required":true,"schema":{"type":"string"}},{"name":"Range","in":"header","schema":{"type":"string"}},{"name":"If-Range","in":"header","schema":{"type":"string"}}],"responses":{"200":{"description":"Full WAV","content":{"audio/wav":{"schema":{"type":"string","format":"binary"}}}},"206":{"description":"Requested WAV bytes","headers":{"Content-Range":{"schema":{"type":"string"}}}},"404":{"description":"Audio not found"},"416":{"description":"Unsatisfiable or multiple range"},"503":{"description":"Storage unavailable"}}},"head":{"tags":["Benchmarks"],"summary":"Check playable audio size and range support","operationId":"headBenchmarkMedia","parameters":[{"name":"hash","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"WAV exists; Content-Length and Accept-Ranges headers"},"404":{"description":"Audio not found"}}}},"/api/v1/image-benchmarks/{id}/images/{index}":{"get":{"tags":["Images"],"summary":"Read a prepared WebP preview or download the original PNG sample","description":"WebP previews are prepared on upload or startup, cached immutably, and never encoded during reads. PNG downloads retain the original SHA256-verified evidence.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"index","in":"path","required":true,"schema":{"type":"integer","minimum":0,"maximum":999}},{"name":"format","in":"query","schema":{"type":"string","enum":["webp","png"],"default":"webp"}}],"responses":{"200":{"description":"Image bytes","content":{"image/webp":{"schema":{"type":"string","format":"binary"}},"image/png":{"schema":{"type":"string","format":"binary"}}}},"304":{"description":"Cached preview remains valid"},"400":{"description":"Invalid selection"},"404":{"description":"Sample not found"},"503":{"description":"Preview or source evidence unavailable"}}},"head":{"tags":["Images"],"summary":"Read sample image headers","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"index","in":"path","required":true,"schema":{"type":"integer","minimum":0,"maximum":999}},{"name":"format","in":"query","schema":{"type":"string","enum":["webp","png"],"default":"webp"}}],"responses":{"200":{"description":"Image headers"},"400":{"description":"Invalid selection"},"404":{"description":"Sample not found"},"503":{"description":"Image unavailable"}}}},"/api/v1/benchmark-artifacts/{hash}":{"put":{"tags":["Benchmarks"],"summary":"Upload a lossless artifact chunk (maximum 4 MiB)","operationId":"putBenchmarkArtifact","parameters":[{"name":"hash","in":"path","required":true,"schema":{"type":"string","pattern":"^[a-f0-9]{64}$"}},{"name":"X-Device-Code","in":"header","required":true,"schema":{"type":"string"},"description":"Registered benchmark device"},{"name":"X-Node-Token","in":"header","schema":{"type":"string"},"description":"Anonymous device capability; owned devices use X-API-Key instead"},{"name":"X-API-Key","in":"header","schema":{"type":"string"},"description":"Owned device API key"},{"name":"Content-Length","in":"header","required":true,"schema":{"type":"integer","minimum":1,"maximum":4194304}}],"requestBody":{"required":true,"content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}}}},"responses":{"200":{"description":"Checksum verified and chunk stored"},"400":{"description":"Invalid hash or checksum"},"403":{"description":"Unregistered device"},"413":{"description":"Invalid chunk size"},"503":{"description":"Storage unavailable"}}},"get":{"tags":["Benchmarks"],"summary":"Download an artifact chunk; concatenate manifest chunks in order","operationId":"getBenchmarkArtifact","parameters":[{"name":"hash","in":"path","required":true,"schema":{"type":"string","pattern":"^[a-f0-9]{64}$"}}],"responses":{"200":{"description":"Immutable raw bytes","content":{"application/octet-stream":{"schema":{"type":"string","format":"binary"}}}},"404":{"description":"Chunk not found"}}},"head":{"tags":["Benchmarks"],"summary":"Check immutable chunk existence and size without downloading it","operationId":"headBenchmarkArtifact","parameters":[{"name":"hash","in":"path","required":true,"schema":{"type":"string","pattern":"^[a-f0-9]{64}$"}}],"responses":{"200":{"description":"Chunk exists","headers":{"Content-Length":{"schema":{"type":"integer","minimum":1,"maximum":4194304}}}},"404":{"description":"Chunk not found"}}}},"/api/v1/vl-benchmarks":{"get":{"tags":["Benchmarks"],"deprecated":true,"summary":"Legacy vision-language model-tag records; not evidence of a vision task or quality score","responses":{"200":{"description":"Historical records; response count cannot establish correctness","content":{"application/json":{"schema":{"type":"object","properties":{"data":{"type":"array","items":{"type":"object","additionalProperties":true,"properties":{"correct_count":{"type":"integer","nullable":true,"description":"Unknown without a scored dataset"},"accuracy_pct":{"type":"number","nullable":true,"description":"Unknown without a scored dataset"}}}}}}}}}}}},"/api/v1/submissions/{id}/heartbeat":{"post":{"tags":["Submissions"],"summary":"Keep a running benchmark session alive during loading and performance measurement","description":"Requires the same owner/device capability as item progress. Send every30seconds. Does not publish results, mark completion or revive terminal sessions. Sessions with no accepted heartbeat/activity for30minutes are failed with retained partial results and a stored timeout error.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Heartbeat accepted"},"401":{"description":"Authentication required"},"403":{"description":"Wrong owner/device capability"},"404":{"description":"Session not found"},"409":{"description":"Terminal session; cannot revive"}}}},"/api/v1/submissions/{id}/progress":{"post":{"tags":["Submissions"],"summary":"Report dataset item progress without publishing answers or scores","description":"Requires the session owner API key, or the matching anonymous device capability. Optional telemetry only: never finalizes a session or changes scores. Same-step dataset/total changes and observed progress regressions are rejected. Unknown fields are ignored and not stored. Body limit 4096 bytes.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"X-API-Key","in":"header","schema":{"type":"string"},"description":"Session owner API key. Required for owned sessions."},{"name":"X-Device-Code","in":"header","schema":{"type":"string"},"description":"Must match the session device for anonymous sessions."},{"name":"X-Node-Token","in":"header","schema":{"type":"string"},"description":"Existing private device capability; required together with X-Device-Code for anonymous sessions."}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["step","dataset","completed","total","phase"],"additionalProperties":true,"properties":{"step":{"type":"integer","minimum":1,"maximum":1000},"dataset":{"type":"string","enum":["aime2026","simpleqa-verified-self-judge","ifbench","hle-self-judge","mmlu-pro","gpqa-diamond","gsm8k","mmlu","cvbench","mmmu-pro"]},"completed":{"type":"integer","minimum":0,"maximum":100000},"total":{"type":"integer","minimum":1,"maximum":100000},"phase":{"type":"string","enum":["inference","judging","uploading"]}}}}}},"responses":{"200":{"description":"Progress accepted","content":{"application/json":{"schema":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean"}}}}}},"400":{"description":"Invalid progress or JSON"},"401":{"description":"Authentication required"},"403":{"description":"Wrong session owner or device capability"},"404":{"description":"Session not found"},"409":{"description":"Terminal session or conflicting progress"},"413":{"description":"Body too large"},"503":{"description":"Progress storage unavailable; benchmark evidence is unaffected"}}}},"/api/v1/submissions/{id}":{"put":{"tags":["Submissions"],"summary":"Upsert a session benchmark result","description":"Retries update only the same session, model/path, context, quantization, engine, method, dataset, quality suite and run fingerprint (including its absence). Different cache workloads or kernel configurations retain separate reports. Restricted quality reports must use the aggregate-only publication contract; full item evidence stays on the operator machine.","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["benchmark"],"properties":{"benchmark":{"$ref":"#/components/schemas/CreateBenchmark"},"client_version":{"type":"string"},"config_tracking_id":{"type":"string","nullable":true}},"additionalProperties":true}}}},"responses":{"200":{"description":"Benchmark upserted","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BenchmarkResponse"}}}},"400":{"description":"Invalid or non-aggregate restricted quality report"}}},"get":{"tags":["Submissions"],"summary":"Get session and benchmark summaries","operationId":"getSubmission","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"Session with task-aware summaries","content":{"application/json":{"schema":{"type":"object","additionalProperties":true,"properties":{"quality_progress":{"type":"object","required":["step","dataset","completed","total","phase","updated_at","stale"],"additionalProperties":true,"properties":{"step":{"type":"integer","minimum":1,"maximum":1000},"dataset":{"type":"string","enum":["aime2026","simpleqa-verified-self-judge","ifbench","hle-self-judge","mmlu-pro","gpqa-diamond","gsm8k","mmlu","cvbench","mmmu-pro"]},"completed":{"type":"integer","minimum":0,"maximum":100000},"total":{"type":"integer","minimum":1,"maximum":100000},"phase":{"type":"string","enum":["inference","judging","uploading"]},"updated_at":{"type":"string","format":"date-time"},"stale":{"type":"boolean","description":"Last accepted report is older than two minutes."}},"nullable":true,"description":"Reported dataset item progress, not a score or proof of completion. Null when unavailable or the session is terminal. No question, answer, item ID or model response is included."},"benchmarks":{"type":"array","items":{"type":"object","additionalProperties":true,"properties":{"id":{"type":"string"},"model_type":{"type":"string"},"bench_method":{"type":"string"},"engine":{"type":"string"},"best_items_per_sec":{"type":"number","nullable":true},"measurement_unit":{"type":"string"},"best_concurrency":{"type":"integer"},"measurement_summary":{"type":"object","description":"Independent per-report single-path and concurrent peaks; excluded observations are diagnostics, never reference scores.","properties":{"unit":{"type":"string"},"prefill":{"type":"object","nullable":true,"description":"Recorded input tokens divided by engine prefill phase duration; independent of whole-request throughput and concurrent aggregate. Prefers uncached observations. Missing trustworthy timing/cache counts remain null.","properties":{"throughput":{"type":"number","minimum":0,"description":"Zero is valid for fully cached input with no computed prompt work."},"level":{"type":"integer","nullable":true},"cached_tokens":{"type":"integer","minimum":0},"uncached_prefill":{"type":"boolean"},"source":{"type":"string","nullable":true}}},"best":{"type":"object","nullable":true,"properties":{"throughput":{"type":"number"},"level":{"type":"integer","nullable":true},"reference_eligible":{"type":"boolean"},"fallback_reasons":{"type":"array","items":{"type":"string"}}}},"single":{"type":"object","nullable":true,"properties":{"throughput":{"type":"number"},"level":{"type":"integer","nullable":true},"reference_eligible":{"type":"boolean"},"fallback_reasons":{"type":"array","items":{"type":"string"}}}},"concurrent":{"type":"object","nullable":true,"properties":{"throughput":{"type":"number"},"level":{"type":"integer","nullable":true},"reference_eligible":{"type":"boolean"},"fallback_reasons":{"type":"array","items":{"type":"string"}}}}}}}}}}}}}}}}},"/api/v1/submissions":{"post":{"tags":["Submissions"],"summary":"Create new submission session","operationId":"createSubmission","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateSubmission"}}}},"responses":{"200":{"description":"Submission created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/SubmissionResponse"}}}}}}},"/api/v1/compare":{"get":{"tags":["Benchmarks"],"summary":"Compare qualified reference scores from the current result pool","parameters":[{"name":"gpu","in":"query","schema":{"type":"string"}},{"name":"model","in":"query","schema":{"type":"string"}},{"name":"groupBy","in":"query","schema":{"type":"string","enum":["model","gpu"]}}],"responses":{"200":{"description":"Qualified scores grouped by model or GPU"},"503":{"description":"Result pool unavailable"}}}},"/api/v1/benchmarks/all":{"get":{"tags":["Benchmarks"],"summary":"Qualified benchmark series for comparisons","description":"Reads the current reference pool and returns only qualified measurement rows in results_json; full original evidence remains at the individual report. Persisted historical best fields are replaced by computed reference scores.","parameters":[{"name":"order","in":"query","schema":{"type":"string","enum":["recent","best"]}},{"name":"limit","in":"query","schema":{"type":"integer"}}],"responses":{"200":{"description":"Qualified comparison records","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/BenchmarkRecord"}}}}},"503":{"description":"Result pool unavailable"}}}},"/api/v1/benchmarks/{id}":{"get":{"tags":["Benchmarks"],"summary":"Original benchmark record and dataset quality evidence","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"profiling","in":"query","schema":{"type":"string","enum":["full","summary"],"default":"full"},"description":"summary omits inline profiling samples from both results and results_json and sets profiling_samples_omitted=true. Full captured traces remain in profiling APIs/downloads; stored evidence is unchanged."}],"responses":{"200":{"description":"Benchmark record; optional quality_json retains the decoded protocol described by the quality fields","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BenchmarkRecord"}}}},"400":{"description":"Invalid profiling view"},"404":{"description":"Benchmark not found"}}}},"/api/v1/benchmarks/{id}/responses":{"get":{"tags":["Benchmarks"],"summary":"Read legacy sampled responses for public datasets","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"limit","in":"query","schema":{"type":"integer","default":50}}],"responses":{"200":{"description":"Sampled responses","content":{"application/json":{"schema":{"type":"object","properties":{"success":{"type":"boolean"},"responses":{"type":"array","items":{"type":"object","additionalProperties":true}}}}}}},"403":{"description":"Restricted dataset responses stay on the operator machine"}}},"post":{"tags":["Benchmarks"],"summary":"Upload legacy sampled responses for public datasets","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["responses"],"properties":{"concurrency":{"type":"integer"},"responses":{"type":"array","maxItems":5000,"items":{"type":"object","additionalProperties":true}}},"additionalProperties":true}}}},"responses":{"200":{"description":"Responses saved"},"400":{"description":"Invalid response array or too many samples"},"403":{"description":"Restricted dataset responses must stay on the operator machine"}}}},"/api/v1/benchmarks/{id}/quality/items":{"get":{"tags":["Benchmarks"],"summary":"Paginated original quality items; filters apply to the entire report","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}},{"name":"page","in":"query","schema":{"type":"integer","minimum":1,"default":1}},{"name":"incorrect","in":"query","schema":{"type":"boolean","default":false}},{"name":"status","in":"query","description":"Exact response status; all includes every status","schema":{"type":"string","maxLength":128,"default":"all"}},{"name":"task","in":"query","description":"Exact subject name","schema":{"type":"string","maxLength":512}}],"responses":{"200":{"description":"Original item order, 12 items per page. total is the global filtered count; overall_total includes failures.","content":{"application/json":{"schema":{"type":"object","required":["items","total","overall_total","page","page_size","pages","status_counts"],"properties":{"items":{"type":"array","maxItems":12,"items":{"type":"object","additionalProperties":true}},"total":{"type":"integer"},"overall_total":{"type":"integer"},"page":{"type":"integer"},"page_size":{"type":"integer","enum":[12]},"pages":{"type":"integer"},"status_counts":{"type":"object","additionalProperties":{"type":"integer"}}}}}}},"400":{"description":"Invalid filters or out-of-range page"},"403":{"description":"Restricted dataset item evidence stays on the operator machine; public aggregate scores remain available"},"404":{"description":"Quality report not found"},"409":{"description":"Missing, corrupt or inconsistent quality evidence; no partial page is returned"},"503":{"description":"Artifact storage unavailable"}}}},"/api/v1/benchmarks/{id}/quality/download":{"get":{"tags":["Benchmarks"],"summary":"Download complete public quality evidence, or the score summary for a restricted dataset","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string"}}],"responses":{"200":{"description":"For restricted datasets this returns only quality-summary.json with aggregate counts/provenance; private item evidence is never returned. Otherwise the attachment streams checksum-verified chunks. X-Content-SHA256 is the original complete-file SHA256 for artifact-backed reports. A missing/corrupt later chunk terminates the stream; partial evidence is never returned as a valid complete report.","headers":{"X-Content-SHA256":{"schema":{"type":"string","pattern":"^[a-f0-9]{64}$"}}},"content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"404":{"description":"Quality report not found"},"409":{"description":"Evidence unavailable or corrupt before streaming"},"503":{"description":"Artifact storage unavailable"}}}},"/api/v1/benchmarks":{"post":{"tags":["Benchmarks"],"summary":"Upload benchmark results","operationId":"createBenchmark","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CreateBenchmark"}}}},"responses":{"200":{"description":"Benchmark created","content":{"application/json":{"schema":{"$ref":"#/components/schemas/BenchmarkResponse"}}}},"400":{"description":"Restricted quality report is not aggregate-only or has inconsistent counts"}}},"get":{"tags":["Benchmarks"],"summary":"Query benchmarks","operationId":"listBenchmarks","parameters":[{"name":"model","in":"query","schema":{"type":"string"}},{"name":"gpu","in":"query","schema":{"type":"string"}},{"name":"limit","in":"query","schema":{"type":"integer","default":20}},{"name":"offset","in":"query","schema":{"type":"integer","default":0}}],"responses":{"200":{"description":"Benchmark list","content":{"application/json":{"schema":{"$ref":"#/components/schemas/LeaderboardResponse"}}}}}}},"/api/v1/gpu/{name}/variants":{"get":{"tags":["Leaderboard"],"summary":"Reference performance by workload","parameters":[{"name":"workload","in":"query","description":"Peak single path or concurrent aggregate; known fallback measurements excluded.","schema":{"type":"string","enum":["all","single","concurrent"],"default":"single"}},{"name":"name","in":"path","required":true,"schema":{"type":"string"}},{"name":"model","in":"query","schema":{"type":"string"}},{"name":"limit","in":"query","schema":{"type":"string"}}],"responses":{"200":{"description":"Filtered reference results","headers":{"X-Result-Cache-Generated-At":{"schema":{"type":"string","format":"date-time"}},"X-Result-Cache-Stale":{"description":"True when the snapshot is older than 60 seconds","schema":{"type":"string","enum":["true","false"]}}}},"400":{"description":"Invalid workload or query"},"503":{"description":"Result snapshot unavailable, expired, or exceeds complete-population bounds; Retry-After: 5. No partial ranking or on-demand scan."}},"description":" Served from a complete internal snapshot refreshed every 30 seconds off the HTTP thread. Refresh failures retain the last good snapshot for at most five minutes; no request falls back to scanning raw evidence."}},"/api/v1/leaderboard":{"get":{"tags":["Leaderboard"],"summary":"Get performance leaderboard","operationId":"getLeaderboard","parameters":[{"name":"model_type","in":"query","description":"Legacy submitted model tag; language combines llm/text/vl/ocr in one family. This is not proof of tested input modality.","schema":{"type":"string"}},{"name":"workload","in":"query","description":"Select peak single-path or aggregate concurrent throughput. Known fallback execution is excluded; all preserves combined scope.","schema":{"type":"string","enum":["all","single","concurrent"],"default":"all"}},{"name":"model","in":"query","schema":{"type":"string"}},{"name":"gpu","in":"query","schema":{"type":"string"}},{"name":"engine","in":"query","schema":{"type":"string"}},{"name":"context","in":"query","schema":{"type":"string"}},{"name":"limit","in":"query","schema":{"type":"integer","default":50}}],"responses":{"200":{"description":"Leaderboard data","content":{"application/json":{"schema":{"$ref":"#/components/schemas/LeaderboardResponse"}}},"headers":{"X-Result-Cache-Generated-At":{"schema":{"type":"string","format":"date-time"}},"X-Result-Cache-Stale":{"description":"True when the snapshot is older than 60 seconds","schema":{"type":"string","enum":["true","false"]}}}},"503":{"description":"Result snapshot unavailable, expired, or exceeds complete-population bounds; Retry-After: 5. No partial ranking or on-demand scan."}},"description":" Served from a complete internal snapshot refreshed every 30 seconds off the HTTP thread. Refresh failures retain the last good snapshot for at most five minutes; no request falls back to scanning raw evidence."}},"/api/v1/nodes/register":{"post":{"tags":["Nodes"],"summary":"Register worker node","operationId":"registerNode","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NodeRegister"}}}},"responses":{"200":{"description":"Node registered"}}}},"/api/v1/nodes/{device_code}/heartbeat":{"post":{"tags":["Nodes"],"summary":"Node heartbeat","operationId":"nodeHeartbeat","parameters":[{"name":"device_code","in":"path","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/NodeHeartbeat"}}}},"responses":{"200":{"description":"Heartbeat accepted"}}}},"/api/v1/errors/report":{"post":{"tags":["Jobs"],"summary":"Persist a diagnostic chunk with idempotent report_id","description":"Anonymous formal executors remain supported. Optional UUID report_id deduplicates identical retries; conflicting content returns 409. Split full logs into chunks smaller than 128 KiB per request and retain unsent chunks for retry. Server removes common credentials and user home paths. /errors/vllm-startup accepts legacy failure_kind/failure_details/command names through the same handler and storage. Operator queries remain administrator-only.","requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"report_id":{"type":"string","format":"uuid"},"submission_id":{"type":"string","nullable":true},"model_name":{"type":"string"},"gpu_name":{"type":"string"},"client_version":{"type":"string"},"error_kind":{"type":"string"},"error_details":{"type":"string"},"last_log_lines":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"vllm_command":{"type":"string"},"failure_kind":{"type":"string"},"failure_details":{"type":"string"},"command":{"type":"string"}},"additionalProperties":true}}}},"responses":{"200":{"description":"Diagnostic stored; report_id returned"},"400":{"description":"Invalid JSON or UUID"},"409":{"description":"Different diagnostic already uses report_id"},"413":{"description":"Split diagnostic into smaller chunks"},"503":{"description":"Storage unavailable; retain and retry"}}}},"/api/v1/errors/vllm-startup":{"post":{"tags":["Jobs"],"summary":"Persist a diagnostic chunk with idempotent report_id","description":"Anonymous formal executors remain supported. Optional UUID report_id deduplicates identical retries; conflicting content returns 409. Split full logs into chunks smaller than 128 KiB per request and retain unsent chunks for retry. Server removes common credentials and user home paths. /errors/vllm-startup accepts legacy failure_kind/failure_details/command names through the same handler and storage. Operator queries remain administrator-only.","requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","properties":{"report_id":{"type":"string","format":"uuid"},"submission_id":{"type":"string","nullable":true},"model_name":{"type":"string"},"gpu_name":{"type":"string"},"client_version":{"type":"string"},"error_kind":{"type":"string"},"error_details":{"type":"string"},"last_log_lines":{"anyOf":[{"type":"string"},{"type":"array","items":{"type":"string"}}]},"vllm_command":{"type":"string"},"failure_kind":{"type":"string"},"failure_details":{"type":"string"},"command":{"type":"string"}},"additionalProperties":true}}}},"responses":{"200":{"description":"Diagnostic stored; report_id returned"},"400":{"description":"Invalid JSON or UUID"},"409":{"description":"Different diagnostic already uses report_id"},"413":{"description":"Split diagnostic into smaller chunks"},"503":{"description":"Storage unavailable; retain and retry"}}}},"/api/v1/nodes/{device_code}/slots":{"post":{"tags":["Jobs"],"security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]}],"summary":"Register authenticated node execution slots","parameters":[{"name":"device_code","in":"path","required":true,"schema":{"type":"string"}}],"description":"Existing llm/image slots are unchanged. Catalog performance workers use type=catalog, engine=autotest, model_id=*, fixed_mode=auto and server-admitted metadata.hardware_key/catalog_revision/executor_version/eligible_models from catalog-jobs. Generic slot writes clear admission; rejoin before polling. Only matching catalog.performance jobs can be leased. Each slot executes one whole model matrix with the formal CLI.","requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["slots"],"properties":{"slots":{"type":"array","items":{"type":"object","required":["slot_id","type","engine","model_id"],"properties":{"slot_id":{"type":"string"},"type":{"type":"string","enum":["llm","image","catalog"]},"engine":{"type":"string"},"model_id":{"type":"string"},"fixed_mode":{"type":"string","enum":["manual","auto"]},"metadata":{"type":"object","additionalProperties":true}},"additionalProperties":true}}}}}}},"responses":{"200":{"description":"upserted count and per-slot validation errors"},"401":{"description":"Node authentication required"},"403":{"description":"Node credential mismatch"}}}},"/api/v1/nodes/{device_code}/jobs":{"get":{"tags":["Jobs"],"security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]},{"SessionCookie":[]}],"summary":"Lease one matching job","operationId":"getNodeJobs","description":"Queued or expired work is assigned with a fresh job_nonce. Dynamic slots may use model_id=* with fixed_mode=auto. Catalog slots additionally match params.hardware_key, params.catalog_revision and params.executor_version against slot metadata and require the server-admitted eligible_models set. An active equivalent lease blocks concurrent duplicate catalog claims across catalog/executor versions; historical success alone never blocks full-mode work; only catalog.performance/v1 jobs can be claimed; one live job per catalog device. Renew every 15 seconds while preparing and running; node heartbeat does not renew a lease.","parameters":[{"name":"device_code","in":"path","required":true,"schema":{"type":"string"}},{"name":"slot_id","in":"query","required":true,"schema":{"type":"string"}},{"name":"lease_protocol","in":"query","required":true,"schema":{"type":"integer","const":1}},{"name":"longpoll","in":"query","schema":{"type":"string","enum":["0","1","false","true"]}}],"responses":{"200":{"description":"One lease or null","content":{"application/json":{"schema":{"type":"object","required":["success","job"],"properties":{"success":{"type":"boolean"},"job":{"anyOf":[{"$ref":"#/components/schemas/Job"},{"type":"null"}]}}}}}}}}},"/api/v1/jobs/{id}/lease":{"post":{"tags":["Jobs"],"security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]},{"SessionCookie":[]}],"summary":"Renew the current attempt for 60 seconds","operationId":"renewJobLease","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"X-Device-Code","in":"header","required":true,"schema":{"type":"string"}},{"name":"X-Job-Nonce","in":"header","required":true,"schema":{"type":"string","format":"uuid"}}],"description":"Requires node authentication and the current job_nonce. An expired or reassigned lease cannot be renewed. Execution must stop on loss of ownership or timeout_sec.","responses":{"200":{"description":"Lease renewed","content":{"application/json":{"schema":{"type":"object","required":["success","lease_seconds","lease_expires_at"],"properties":{"success":{"type":"boolean","const":true},"lease_seconds":{"type":"integer","const":60},"lease_expires_at":{"type":"string","format":"date-time"}}}}}},"409":{"description":"Expired or reassigned; stop execution"}}}},"/api/v1/jobs/{id}/result":{"post":{"tags":["Jobs"],"security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]},{"SessionCookie":[]}],"summary":"Submit an attempt result idempotently","operationId":"submitJobResult","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"X-Device-Code","in":"header","required":true,"schema":{"type":"string"}},{"name":"X-Job-Nonce","in":"header","required":true,"schema":{"type":"string","format":"uuid"}}],"description":"Persist results before sending. Retrying exactly the same JSON bytes with the same nonce returns the saved receipt, including failures requeued for another attempt. Late delivery is accepted only while that attempt has not been reassigned or revoked by timeout recovery. Different bytes or stale attempts return 409. Catalog resource interruptions requeue unfinished work and credit the lease allowance without consuming the bounded execution-failure retry budget. Uploaded scores are retained; historical terminal interruptions are preserved in audit events before recovery. Requires migration 0079.","requestBody":{"required":true,"content":{"application/json":{"schema":{"$ref":"#/components/schemas/JobResult"}}}},"responses":{"200":{"description":"Saved receipt (also returned on replay)","content":{"application/json":{"schema":{"type":"object","required":["success","requeued"],"properties":{"success":{"type":"boolean","const":true},"requeued":{"type":"boolean"},"error_code":{"type":["string","null"]},"attempts":{"type":"integer"},"max_attempts":{"type":"integer"}}}}}},"409":{"description":"Attempt superseded or different result already accepted; retain local evidence"}}}},"/api/v1/jobs/{id}/events":{"post":{"tags":["Jobs"],"security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]},{"SessionCookie":[]}],"summary":"Append events under the active lease","operationId":"postJobEvents","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"X-Device-Code","in":"header","required":true,"schema":{"type":"string"}},{"name":"X-Job-Nonce","in":"header","required":true,"schema":{"type":"string","format":"uuid"}}],"requestBody":{"required":true,"content":{"application/json":{"schema":{"type":"object","required":["events"],"properties":{"events":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":true}}}}}}},"responses":{"200":{"description":"Events stored; this does not renew the lease"},"409":{"description":"Lease expired or reassigned"}}}},"/api/v1/jobs/{id}/artifacts/image":{"post":{"tags":["Jobs"],"security":[{"ApiKeyAuth":[]},{"NodeTokenAuth":[]},{"SessionCookie":[]}],"summary":"Save a PNG artifact for the current attempt","operationId":"uploadJobImage","parameters":[{"name":"id","in":"path","required":true,"schema":{"type":"string","format":"uuid"}},{"name":"X-Device-Code","in":"header","required":true,"schema":{"type":"string"}},{"name":"X-Job-Nonce","in":"header","required":true,"schema":{"type":"string","format":"uuid"}}],"description":"Up to 32 MiB. Attempt nonce and content SHA256 determine the URL, so retries reuse the same artifact. Late upload is accepted only before reassignment or timeout revocation.","requestBody":{"required":true,"content":{"multipart/form-data":{"schema":{"type":"object","required":["image"],"properties":{"image":{"type":"string","format":"binary"}}}}}},"responses":{"200":{"description":"Artifact URL","content":{"application/json":{"schema":{"type":"object","required":["success","artifact_id","url"],"properties":{"success":{"type":"boolean"},"artifact_id":{"type":"string"},"url":{"type":"string","format":"uri"}}}}}},"409":{"description":"Attempt reassigned"}}}},"/api/v1/me":{"get":{"tags":["Auth"],"summary":"Get current user","operationId":"getMe","responses":{"200":{"description":"User info","content":{"application/json":{"schema":{"type":"object","properties":{"user":{"type":"object"},"is_platform_admin":{"type":"boolean"}}}}}}}}},"/health":{"get":{"tags":["Admin"],"summary":"Health check","operationId":"healthCheck","responses":{"200":{"description":"OK"}}}}},"components":{"schemas":{"EmbeddingRecipeError":{"type":"object","required":["schema_version","error"],"properties":{"schema_version":{"type":"string","enum":["wbench.embedding.recipe.v1"]},"error":{"type":"object","required":["code","message","retryable"],"properties":{"code":{"type":"string"},"message":{"type":"string"},"retryable":{"type":"boolean"},"details":{"type":"array","items":{"type":"string"}}}}}},"CreateSubmission":{"type":"object","required":["hardware"],"properties":{"hardware":{"$ref":"#/components/schemas/Hardware"},"engine":{"type":"string"},"cli_version":{"type":"string"}}},"SubmissionResponse":{"type":"object","properties":{"success":{"type":"boolean"},"submission_id":{"type":"string","format":"uuid"},"share_url":{"type":"string","format":"uri"}}},"Hardware":{"type":"object","properties":{"gpu_name":{"type":"string"},"gpu_vram_gb":{"type":"number"},"gpu_driver":{"type":"string"},"tensor_issue_rates":{"type":"object","additionalProperties":true,"description":"Optional measurement-time mrecipes.tensor-issue-rates.v1 NVIDIA RM readback. Current selectors are not proof of modification. Preserved in results[].metadata.tensor_issue_rates; unknown/absent never means stock. Linux 64-bit probe; identity, time and per-device failure reasons retained. Upload replay must not refresh this observation."},"cuda_version":{"type":"string"},"cpu_name":{"type":"string"},"cpu_cores_physical":{"type":"integer"},"cpu_cores_logical":{"type":"integer"},"ram_total_gb":{"type":"number"},"os_name":{"type":"string"},"os_version":{"type":"string"}}},"CreateBenchmark":{"type":"object","required":["submission_id","model_name","config","results"],"properties":{"quality_score":{"type":"number","nullable":true,"description":"Dataset accuracy; compare only the same quality_suite protocol and sample selection"},"quality_suite":{"type":"string","nullable":true,"description":"Versioned dataset/scorer identity including sample count, max output tokens and temperature"},"quality_json":{"type":"string","nullable":true,"description":"JSON-encoded report: suite,total,correct,accuracy,request_errors,truncated,by_task,items (id,prompt,expected,got,reasoning,correct,status,latency_ms (nullable for an interrupted request of unknown outcome),finish_reason,input_tokens,output_tokens,reasoning_tokens,response_payload_sha256); absent token counts are unknown, not zero; malformed HTTP200 items retain observed overall usage plus unambiguous single-choice reasoning and finish_reason from their original JSON payload in got, while remaining invalid_response and ungraded; response_payload_sha256 binds got to the retained malformed payload text and distinguishes it from a normal JSON-valued answer; resume verifies these fields against that payload; status distinguishes scored,invalid_answer,invalid_response,truncated,request_error. Text suites are versioned; v4 requires one response choice and finish_reason=stop, separates malformed HTTP200 responses from transport failures, and rejects unparsed reasoning. dataset (id,split,revision,sha256,full_size,sample_size,full_run,selection,sample_ids_sha256,scorer,prompt_protocol,source_url,license); generation (max_tokens,temperature,attempts_per_item,system_prompt,resume,engine,judge). Optional factuality for SimpleQA Verified records schema=wbench.simpleqa-verified.metrics.v1, correct/incorrect/not_attempted/evaluation_failures counts, total, complete_grading, primary=correct_over_total, correct_over_total, accuracy_given_attempted and f1. Execution failures are distinct from model abstentions; attempted accuracy and F1 are null if any item lacks a valid grade. Explicit simpleqa-verified-self-judge uses the question-only no-tools solver protocol, pinned version9 judge prompt, strict exact A/B/C parsing, temperature0 and max_tokens64; it is separate from publisher gpt-4.1 judging and the permissive notebook parser. Pending judge failure yields no complete score and recovery never regenerates a saved prediction. Optional HLE judge records mode=self-judge, model, protocol, upstream_revision, prompt_sha256, max_tokens, temperature, enable_thinking and telemetry_scope. Self-judge uses a separate suite and is not an independent-judge HLE score. dataset.image_items/text_items describe the actual evaluated population; full HLE includes 2500 items and 342 image items. Pending judge failures produce no complete score; private predictions are saved before judging, and resume repeats only unsuccessful judge requests with a total three-attempt budget. generation.resume.reused_predictions/new_judge_requests separate recovery from new solver requests. HLE GPU telemetry includes solver and judge calls; not a pure inference speed result. Optional generation.engine preserves managed engine/model provenance and runtime version evidence, also present in results metadata.engine_provenance and the configuration fingerprint; opaque external endpoints remain null. Runtime version evidence identifies installed distributions through the selected launch interpreter, not immutable image contents or all loaded libraries. generation.resume records retained items and new requests; GPU telemetry and throughput cover the current invocation only. Optional instruction_following reports IFBench prompt_strict/loose_accuracy and instruction_strict/loose_accuracy separately; quality_score is prompt-level loose accuracy, not an average of these metrics. Optional items.grading preserves instruction_ids and strict/loose Boolean arrays. dataset.grader binds official source revision, manifest SHA256, Python/dependency versions and pinned NLP resources. IFBench uses the original user-only prompt and the HF train split is the official 300-row test set. Large reports may omit items and supply item_storage {schema: wbench.quality.items.v1, manifest, report}; each artifact descriptor is {name,bytes,sha256,chunks} with ordered 4 MiB chunks (only the last may be smaller) available at /api/v1/benchmark-artifacts/{sha256}. The manifest contains {schema,total,pages:[{artifact,rows}]} with index rows [task-or-null,status-or-null,correct]. item_status_counts retains global failure counts. Use GET /api/v1/benchmarks/{id}/quality/items for globally filtered pagination and /quality/download for the exact complete original JSON. Restricted datasets use publication {mode: aggregate-only, item_evidence: retained-locally, private_report_sha256}; public reports contain aggregate counts/provenance only, no items or item_storage, and raw benchmark backups are likewise redacted. Full item evidence stays on the operator machine. GET /quality/items and legacy /responses return 403; /quality/download returns only quality-summary.json for these reports. Upload endpoints reject restricted reports containing item evidence or unknown aggregate payload fields with 400. Inline legacy reports remain supported. Unknown response fields tolerated. Subsets are not full test scores. Request failures and truncation count against accuracy."},"submission_id":{"type":"string","format":"uuid"},"model_name":{"type":"string"},"model_path":{"type":"string"},"quant":{"type":"string"},"engine":{"type":"string"},"config":{"$ref":"#/components/schemas/BenchmarkConfig"},"results":{"type":"array","items":{"$ref":"#/components/schemas/ConcurrencyResult"}}}},"BenchmarkResponse":{"type":"object","properties":{"success":{"type":"boolean"},"benchmark_id":{"type":"string","format":"uuid"}}},"BenchmarkRecord":{"type":"object","additionalProperties":true,"properties":{"id":{"type":"string"},"model_name":{"type":"string"},"model_path":{"type":"string"},"quant":{"type":"string"},"engine":{"type":"string"},"gpu_count":{"type":"integer","nullable":true,"description":"GPU count recorded with this benchmark. Null or absent means unknown; this value is not inferred from tensor parallelism."},"tensor_parallel_size":{"type":"integer","nullable":true,"description":"Tensor-parallel size recorded with this benchmark. Null or absent means unknown; it does not establish physical GPU count."},"bench_method":{"type":"string","nullable":true,"description":"Measurement protocol. v3 separates isolated and prefix-reuse cells; v2 isolates per-request caching; v1 uses historical cache policy. Compare matching workloads and protocols."},"profiling_samples_omitted":{"type":"boolean","description":"True only for explicit profiling=summary reads; full samples remain in profiling APIs/downloads."},"llm_recipes":{"type":"array","description":"Current recipe eligibility per measured concurrency, derived by the same validator as the recipe endpoint and the published executor pointer. Independent of performance ranking. Empty for other protocols. Revalidated on recipe access.","items":{"type":"object","required":["concurrency","available","reason"],"properties":{"concurrency":{"type":"integer"},"available":{"type":"boolean"},"reason":{"type":"string","nullable":true}}}},"quality_score":{"type":"number","nullable":true,"description":"Dataset accuracy; compare only the same quality_suite protocol and sample selection"},"quality_suite":{"type":"string","nullable":true,"description":"Versioned dataset/scorer identity including sample count, max output tokens and temperature"},"quality_json":{"type":"string","nullable":true,"description":"JSON-encoded report: suite,total,correct,accuracy,request_errors,truncated,by_task,items (id,prompt,expected,got,reasoning,correct,status,latency_ms (nullable for an interrupted request of unknown outcome),finish_reason,input_tokens,output_tokens,reasoning_tokens,response_payload_sha256); absent token counts are unknown, not zero; malformed HTTP200 items retain observed overall usage plus unambiguous single-choice reasoning and finish_reason from their original JSON payload in got, while remaining invalid_response and ungraded; response_payload_sha256 binds got to the retained malformed payload text and distinguishes it from a normal JSON-valued answer; resume verifies these fields against that payload; status distinguishes scored,invalid_answer,invalid_response,truncated,request_error. Text suites are versioned; v4 requires one response choice and finish_reason=stop, separates malformed HTTP200 responses from transport failures, and rejects unparsed reasoning. dataset (id,split,revision,sha256,full_size,sample_size,full_run,selection,sample_ids_sha256,scorer,prompt_protocol,source_url,license); generation (max_tokens,temperature,attempts_per_item,system_prompt,resume,engine,judge). Optional factuality for SimpleQA Verified records schema=wbench.simpleqa-verified.metrics.v1, correct/incorrect/not_attempted/evaluation_failures counts, total, complete_grading, primary=correct_over_total, correct_over_total, accuracy_given_attempted and f1. Execution failures are distinct from model abstentions; attempted accuracy and F1 are null if any item lacks a valid grade. Explicit simpleqa-verified-self-judge uses the question-only no-tools solver protocol, pinned version9 judge prompt, strict exact A/B/C parsing, temperature0 and max_tokens64; it is separate from publisher gpt-4.1 judging and the permissive notebook parser. Pending judge failure yields no complete score and recovery never regenerates a saved prediction. Optional HLE judge records mode=self-judge, model, protocol, upstream_revision, prompt_sha256, max_tokens, temperature, enable_thinking and telemetry_scope. Self-judge uses a separate suite and is not an independent-judge HLE score. dataset.image_items/text_items describe the actual evaluated population; full HLE includes 2500 items and 342 image items. Pending judge failures produce no complete score; private predictions are saved before judging, and resume repeats only unsuccessful judge requests with a total three-attempt budget. generation.resume.reused_predictions/new_judge_requests separate recovery from new solver requests. HLE GPU telemetry includes solver and judge calls; not a pure inference speed result. Optional generation.engine preserves managed engine/model provenance and runtime version evidence, also present in results metadata.engine_provenance and the configuration fingerprint; opaque external endpoints remain null. Runtime version evidence identifies installed distributions through the selected launch interpreter, not immutable image contents or all loaded libraries. generation.resume records retained items and new requests; GPU telemetry and throughput cover the current invocation only. Optional instruction_following reports IFBench prompt_strict/loose_accuracy and instruction_strict/loose_accuracy separately; quality_score is prompt-level loose accuracy, not an average of these metrics. Optional items.grading preserves instruction_ids and strict/loose Boolean arrays. dataset.grader binds official source revision, manifest SHA256, Python/dependency versions and pinned NLP resources. IFBench uses the original user-only prompt and the HF train split is the official 300-row test set. Large reports may omit items and supply item_storage {schema: wbench.quality.items.v1, manifest, report}; each artifact descriptor is {name,bytes,sha256,chunks} with ordered 4 MiB chunks (only the last may be smaller) available at /api/v1/benchmark-artifacts/{sha256}. The manifest contains {schema,total,pages:[{artifact,rows}]} with index rows [task-or-null,status-or-null,correct]. item_status_counts retains global failure counts. Use GET /api/v1/benchmarks/{id}/quality/items for globally filtered pagination and /quality/download for the exact complete original JSON. Restricted datasets use publication {mode: aggregate-only, item_evidence: retained-locally, private_report_sha256}; public reports contain aggregate counts/provenance only, no items or item_storage, and raw benchmark backups are likewise redacted. Full item evidence stays on the operator machine. GET /quality/items and legacy /responses return 403; /quality/download returns only quality-summary.json for these reports. Upload endpoints reject restricted reports containing item evidence or unknown aggregate payload fields with 400. Inline legacy reports remain supported. Unknown response fields tolerated. Subsets are not full test scores. Request failures and truncation count against accuracy."}}},"BenchmarkConfig":{"type":"object","properties":{"context_name":{"type":"string"},"input_tokens":{"type":"integer"},"output_tokens":{"type":"integer"},"test_duration_sec":{"type":"integer"}}},"ConcurrencyResult":{"type":"object","properties":{"profiling":{"type":"object","additionalProperties":true,"description":"Captured GPU time series and full-trace summary. Large uploads retain summary here and use metadata.profiling_storage for samples; an empty inline array alone does not imply missing capture."},"resource_lifecycle":{"$ref":"#/components/schemas/ResourceLifecycle"},"memory":{"$ref":"#/components/schemas/MemoryReport"},"request_timings":{"allOf":[{"$ref":"#/components/schemas/BenchmarkArtifact"}],"description":"All client request observations, microsecond timing units; content-bearing stream chunk intervals"},"raw_benchmark":{"allOf":[{"$ref":"#/components/schemas/BenchmarkArtifact"}],"description":"Complete original benchmark JSON with all responses and profiling, uploaded automatically and attached to the first tier"},"vllm_metrics":{"type":"object","properties":{"before":{"type":"array","items":{"$ref":"#/components/schemas/MetricsSnapshot"}},"after":{"type":"array","items":{"$ref":"#/components/schemas/MetricsSnapshot"}},"samples":{"type":"array","items":{"$ref":"#/components/schemas/MetricsSnapshot"},"description":"Full metric snapshots during the workload, sampled approximately once per second"},"mean_seconds":{"type":"object","additionalProperties":{"type":"number"},"description":"Interval histogram sum/count deltas; not client latency"}}},"operator_profile":{"type":"object","properties":{"status":{"type":"string","enum":["complete","failed"]},"error":{"type":"string","nullable":true},"requested_concurrency":{"type":"integer"},"captured_concurrency":{"type":"integer"},"output_tokens":{"type":"integer"},"artifacts":{"type":"array","items":{"$ref":"#/components/schemas/BenchmarkArtifact"}},"summary":{"type":"object","nullable":true,"additionalProperties":true,"description":"Top 20 operator groups; full summary and events retained in artifacts. Event durations may overlap."}}},"concurrency":{"type":"integer"},"requests_per_sec":{"type":"number","nullable":true},"items_per_sec":{"type":"number","description":"Native output throughput in measurement_unit; input token rate may also be reported"},"measurement_unit":{"type":"string","description":"vectors/sec, decisions/sec, images/sec or audio-sec/sec for native tasks"},"wall_seconds":{"type":"number"},"request_latencies_sec":{"type":"array","items":{"type":"number"},"description":"Individual observations; native task latency measures the entire batch or audio workload"},"metadata":{"type":"object","additionalProperties":true,"properties":{"context_efficiency":{"type":"object","additionalProperties":true,"required":["schema","cache_mode","cache_observation","cache_hit_ratio","requested_input_tokens","output_limit","actual_input_tokens","actual_output_tokens","gpu_memory_used_gib","kv_cache_dtype","attention_implementation","notes"],"properties":{"declared_kv_estimate":{"type":"object","additionalProperties":true,"nullable":true,"description":"Conditional full-attention BF16/FP16 KV estimate; excludes recurrent states and runtime overhead; not admission proof"},"concurrency_selection":{"type":"object","additionalProperties":true,"description":"Eligible/excluded concurrency with engine KV capacity evidence and conservative safety reserve"},"declared_model_kv_config":{"type":"object","additionalProperties":true,"description":"Checkpoint-declared KV shape; not verified runtime dtype or kernel"},"schema":{"type":"string","enum":["wbench.context-efficiency.v1"]},"cache_mode":{"type":"string","enum":["isolated","prefix-reuse"]},"cache_observation":{"type":"string","enum":["hit-observed","no-hit-observed","unknown"]},"cache_hit_ratio":{"type":"number","minimum":0,"maximum":1,"nullable":true},"requested_input_tokens":{"type":"integer","minimum":1},"output_limit":{"type":"integer","minimum":1},"actual_input_tokens":{"type":"integer","minimum":0},"actual_output_tokens":{"type":"integer","minimum":0},"gpu_memory_used_gib":{"type":"number","minimum":0,"nullable":true},"kv_cache_dtype":{"type":"string","nullable":true},"attention_implementation":{"type":"string","nullable":true},"notes":{"type":"string"}},"description":"Observed context/cache workload evidence; GPU allocation is not KV bytes per token; unknown implementations remain null"},"cache_policy":{"type":"string","description":"fixed-http-v2/v3: isolated requests use per-request-cache-salt or cache-prompt-disabled; prefix reuse uses shared-prefix-cache-salt or cache-prompt-enabled. Other engines remain unverified; actual counters determine hits."},"uncached_prefill":{"type":"boolean","description":"True only when measured engine cache counts establish no reused prompt tokens."},"phase_measurement":{"type":"object","properties":{"schema":{"type":"string","enum":["wbench.phase-timings.v1"]},"status":{"type":"string","enum":["valid","unavailable"]},"reason":{"type":"string"},"collection_attempts":{"type":"integer","minimum":1,"maximum":3},"metrics_collection_ms":{"type":"number","minimum":0}},"description":"Up to two bounded metrics-only retries preserve the original window and never rerun inference. Collection time is excluded from throughput."},"measurement_window":{"type":"object","properties":{"wall_seconds":{"type":"number","exclusiveMinimum":0},"successful_requests":{"type":"integer","minimum":0},"failed_requests":{"type":"integer","minimum":0},"input_tokens":{"type":"integer","minimum":0},"output_tokens":{"type":"integer","minimum":0},"input_tok_per_sec":{"type":"number","minimum":0},"output_tok_per_sec":{"type":"number","minimum":0},"scope":{"type":"string"}},"description":"Observed request-window denominator and actual completed token totals; excludes warmup, loading and final metrics collection."},"phase_timings":{"type":"object","nullable":true,"additionalProperties":true,"properties":{"prefill_tok_per_sec":{"type":"number","minimum":0},"decode_tok_per_sec":{"type":"number","minimum":0},"prompt_tokens":{"type":"number","minimum":0},"computed_prompt_tokens":{"type":"integer","minimum":0},"cached_tokens":{"type":"integer","minimum":0},"prefill_ms":{"type":"number","minimum":0},"decode_ms":{"type":"number","minimum":0},"aggregation":{"type":"string"},"source":{"type":"string"}},"description":"Computed tokens divided by summed engine request-phase seconds; concurrent clocks overlap, so these are not aggregate GPU rates. vLLM requires matching request/token counters and excludes first tokens from decode. Unknown remains null."},"profiling_storage":{"type":"object","additionalProperties":true,"description":"wbench.profiling.v1 complete profile and <=1000-point preview artifact descriptors. Original complete trace available through profiling download API.","properties":{"schema":{"type":"string","enum":["wbench.profiling.v1"]},"total_samples":{"type":"integer","minimum":1,"maximum":5000000},"report":{"$ref":"#/components/schemas/BenchmarkArtifact"},"preview":{"$ref":"#/components/schemas/BenchmarkArtifact"}}},"performance_qualification":{"type":"object","description":"Server-derived formal text performance eligibility and reasons. Requires the requested limit and every successful output to reach the declared minimum, complete counts/stop reasons, agreement with recorded measurement-window counters and eligible execution. Short/unknown reports retain their raw observations and remain usable as API configurations. Clients consume this projection; raw stored evidence is unchanged.","properties":{"schema":{"type":"string","enum":["wbench.text-performance.v1"]},"reference_eligible":{"type":"boolean"},"minimum_output_tokens":{"type":"integer"},"reasons":{"type":"array","items":{"type":"string"}},"evidence":{"type":"object","nullable":true,"properties":{"requests":{"type":"integer"},"minimum":{"type":"integer"},"maximum":{"type":"integer"},"input_tokens":{"type":"integer","nullable":true},"output_tokens":{"type":"integer","nullable":true},"complete":{"type":"boolean"},"finish_reasons":{"type":"object","additionalProperties":{"type":"integer"}}}}}},"performance_task":{"type":"object","required":["id","sha256","prompt_sha256"],"properties":{"id":{"type":"string"},"sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"},"prompt_sha256":{"type":"string","pattern":"^[a-f0-9]{64}$"}},"description":"Versioned synthetic generation task and exact calibrated request prompt. Full prompt/output/usage/finish reasons remain in request_timings artifacts. Does not attest output length or answer quality."},"engine_provenance":{"type":"object","additionalProperties":true,"description":"Managed engine/model identity used in the run fingerprint. Optional checkpoint_quantization separates recipe_label, declared quantized_weight_formats and mixed_quantization from observed runtime kernels","properties":{"runtime":{"type":"object","nullable":true,"additionalProperties":true,"description":"Installed distribution versions observed through the selected launch interpreter; not an immutable image or loaded-library attestation. Null/absent is unknown.","properties":{"schema":{"type":"string","enum":["wbench.python-runtime.v1"]},"scope":{"type":"string","enum":["launch-environment-installed-distributions"]},"python_version":{"type":"string"},"python_implementation":{"type":"string"},"packages":{"type":"object","additionalProperties":{"type":"string","nullable":true}}}}}},"vision":{"type":"object","additionalProperties":true,"description":"vision-forward-v1 matched PyTorch/TensorRT raw forward protocol; not task accuracy. Compilation and preprocessing excluded from measured tiers.","properties":{"task":{"type":"string","enum":["detection","segmentation","pose","classification"]},"backend":{"type":"string","enum":["pytorch","tensorrt"]},"precision":{"type":"string","enum":["fp16","fp32"]},"resolution":{"type":"array","items":{"type":"integer"}},"weights_sha256":{"type":"string"},"engine_build_seconds":{"type":"number"},"equivalence":{"type":"object","additionalProperties":true},"fixtures":{"type":"array","items":{"type":"object","additionalProperties":true}}}},"audio":{"type":"array","items":{"type":"object","additionalProperties":true,"required":["filename","media_id","status"],"properties":{"filename":{"type":"string"},"media_id":{"type":"string","pattern":"^[a-f0-9]{64}$"},"sha256":{"type":"string"},"bytes":{"type":"integer","minimum":1},"status":{"type":"string","enum":["local","ready"]},"text":{"type":"string"},"phase":{"type":"string","enum":["warmup","measured","profiling"]},"batch_size":{"type":"integer","minimum":1},"sample_rate":{"type":"number"},"duration_seconds":{"type":"number","minimum":0}}},"description":"All saved speech and input WAVs; GET benchmark-media/{media_id} plays ready audio; local audio has not been uploaded"},"transcript":{"description":"Complete original ASR output, including text and segment timestamps when available"},"transcript_runs":{"type":"array","items":{"type":"object","properties":{"phase":{"type":"string","enum":["warmup","profiling"]},"transcript":{"description":"Complete diagnostic recognition output"}}}},"execution":{"type":"object","properties":{"workload_mode":{"type":"string","enum":["single","concurrent"]},"graph_mode":{"type":"string","enum":["captured","unknown","eager_fallback"]},"graph_path":{"type":"string","enum":["token_graph","verify_windows","batch_windows","unknown"]},"reference_eligible":{"type":"boolean"},"shared_gpu":{"type":"boolean","description":"Other GPU workloads active; card-wide telemetry, diagnostic performance only"},"fallback_reasons":{"type":"array","items":{"type":"string"}}},"description":"Actual execution evidence; fallback measurements retained but excluded from reference scores. Untagged historical graph mode is unknown."},"input_tokens":{"type":"integer","minimum":0,"description":"Measured input token total including prompts and special tokens, excluding padding"},"resources":{"$ref":"#/components/schemas/ResourceSummary"},"resource_lifecycle":{"$ref":"#/components/schemas/ResourceLifecycle"},"generated_images":{"type":"array","items":{"$ref":"#/components/schemas/GeneratedImage"}},"embedding_quality":{"type":"array","description":"Pinned complete dataset task scores; signed correlations retained, not overall MTEB","items":{"type":"object","required":["suite","metric","score","metrics","dimensions","dataset"],"properties":{"suite":{"type":"string"},"metric":{"type":"string"},"score":{"type":"number"},"metrics":{"type":"object","additionalProperties":{"type":"number"}},"dimensions":{"type":"integer"},"dataset":{"type":"object","additionalProperties":true}}}},"model_revision":{"type":"string"},"dimensions":{"type":"integer"},"max_sequence_length":{"type":"integer"},"verified_recipe":{"type":"string"},"kernel_warnings":{"type":"array","items":{"type":"string"},"description":"Actual runtime reference-kernel warnings; wrapper module names alone are ambiguous"}},"description":"Task engine, hardware, and measured configuration"},"raw_artifacts":{"type":"array","items":{"type":"object","required":["kind","sha256","bytes","chunks"],"properties":{"kind":{"type":"string"},"sha256":{"type":"string"},"bytes":{"type":"integer"},"chunks":{"type":"array","items":{"type":"string"}},"profiling_status":{"type":"string","enum":["captured","unavailable"]}}},"description":"Lossless vectors, decisions, audio, timings and diagnostic CUDA traces; concatenate ordered chunks and verify SHA256"},"input_tok_per_sec":{"type":"number"},"output_tok_per_sec":{"type":"number"},"latency_avg_sec":{"type":"number"},"latency_p50_sec":{"type":"number"},"latency_p95_sec":{"type":"number"},"latency_p99_sec":{"type":"number"},"gpu_util_avg":{"type":"number","nullable":true},"power_draw_watts":{"type":"number","nullable":true},"gpu_mem_used_gb":{"type":"number","nullable":true},"power_limit_watts":{"type":"number","nullable":true}}},"LeaderboardResponse":{"type":"object","properties":{"data":{"type":"array","items":{"$ref":"#/components/schemas/LeaderboardEntry"}},"total":{"type":"integer"}}},"ImageGeneration":{"type":"object","additionalProperties":true,"description":"Client-observed reference configuration and completion evidence; parameter compliance does not prove visual quality.","properties":{"test_mode":{"type":"string","enum":["formal","smoke"]},"profile_version":{"type":"string"},"reference_url":{"type":"string","nullable":true},"requested_steps":{"type":"integer"},"completed_steps":{"type":"array","items":{"type":"integer","nullable":true},"description":"Completed denoising callbacks per successful request; Z-Image-Turbo requests 9; older pipelines complete 8 and newer schedules may complete 9. Missing callback evidence is null."},"steps_verified":{"type":"boolean"},"resolution":{"type":"string"},"guidance_scale":{"type":"number"},"scheduler":{"type":"string"},"scheduler_config":{"type":"object","additionalProperties":true},"pipeline":{"type":"string"},"extra_parameters":{"type":"object","additionalProperties":true}}},"GeneratedImage":{"type":"object","required":["artifact","prompt","mime_type"],"properties":{"artifact":{"$ref":"#/components/schemas/BenchmarkArtifact"},"prompt":{"type":"string","description":"Original prompt; empty if unavailable in a historical output"},"mime_type":{"type":"string","enum":["image/png"]}}},"ResourceLifecycle":{"type":"object","additionalProperties":true,"description":"Worker preparation through cleanup; upload excluded. Missing counters are unavailable, never zero.","properties":{"schema_version":{"type":"integer","enum":[1]},"status":{"type":"string","enum":["running","completed","failed","cancelled"]},"scope":{"type":"string"},"source":{"type":"string"},"process_id":{"type":"integer"},"sample_interval_sec":{"type":"number"},"sample_count":{"type":"integer"},"gpu_sample_count":{"type":"integer"},"missing_gpu_samples":{"type":"integer"},"errors":{"type":"array","items":{"type":"string"}},"timeline_file":{"type":"string"},"timeline_artifact":{"$ref":"#/components/schemas/BenchmarkArtifact"},"phases":{"type":"array","items":{"type":"object","properties":{"phase":{"type":"string"},"duration_sec":{"type":"number"},"sample_count":{"type":"integer"},"gpu_sample_count":{"type":"integer"},"vram_peak_percent":{"type":"number","nullable":true},"metrics":{"type":"object","additionalProperties":{"type":"object","properties":{"mean":{"type":"number"},"peak":{"type":"number"},"min":{"type":"number"},"sample_count":{"type":"integer"}}}}}}}}},"ResourceSummary":{"type":"object","additionalProperties":true,"description":"Additive per-level telemetry. Null/missing means unavailable. Window includes preprocessing/output handling separately from inference timing. Threshold analysis is heuristic.","properties":{"sample_count":{"type":"integer","nullable":true,"description":"sample count"},"gpu_sample_count":{"type":"integer","nullable":true,"description":"gpu sample count"},"duration_sec":{"type":"number","nullable":true,"description":"duration sec"},"vram_peak_gb":{"type":"number","nullable":true,"description":"Sampled device-wide peak GiB; includes other processes"},"vram_total_gb":{"type":"number","nullable":true,"description":"Visible device capacity in GiB"},"memory_busy_avg_percent":{"type":"number","nullable":true,"description":"NVML percent memory busy time; not bandwidth saturation"},"cpu_busy_cores_avg":{"type":"number","nullable":true,"description":"Process CPU seconds / telemetry wall seconds; all threads, 1 equals one busy core"},"cpu_capacity_cores":{"type":"number","nullable":true,"description":"Available affinity limited by detected container quota"},"rss_peak_gb":{"type":"number","nullable":true,"description":"Sampled process resident peak GiB"},"ram_available_min_gb":{"type":"number","nullable":true,"description":"Sampled minimum available RAM in detected host/cgroup budget, GiB"},"ram_total_gb":{"type":"number","nullable":true,"description":"Detected effective RAM budget, GiB"},"read_mib_per_sec":{"type":"number","nullable":true,"description":"Physical process read MiB/s; excludes cache hits"},"write_mib_per_sec":{"type":"number","nullable":true,"description":"Physical process write MiB/s"},"temperature_peak_c":{"type":"number","nullable":true,"description":"temperature peak c"},"clock_sm_avg_mhz":{"type":"number","nullable":true,"description":"clock sm avg mhz"},"clock_sm_min_mhz":{"type":"number","nullable":true,"description":"clock sm min mhz"},"power_avg_watts":{"type":"number","nullable":true,"description":"power avg watts"},"power_limit_watts":{"type":"number","nullable":true,"description":"power limit watts"},"torch_allocated_peak_gb":{"type":"number","nullable":true,"description":"Torch allocator peak allocated GiB"},"torch_reserved_peak_gb":{"type":"number","nullable":true,"description":"Torch allocator peak reserved GiB, not bandwidth demand"}}},"MemoryReport":{"type":"object","properties":{"total_gb":{"type":"number","description":"Effective process memory budget in GiB"},"available_gb":{"type":"number","description":"Available within the budget at test start"},"free_gb":{"type":"number","description":"Uncharged cgroup headroom, or OS free memory"},"reclaimable_gb":{"type":"number","description":"Inactive file cache credited in the available estimate"},"host_total_gb":{"type":"number","description":"OS-visible RAM; may describe host rather than guest"},"cgroup_limit_gb":{"type":"number","description":"Finite container memory limit if smaller than host"},"is_container":{"type":"boolean"}}},"BenchmarkArtifact":{"type":"object","required":["name","bytes","sha256","chunks"],"properties":{"name":{"type":"string"},"bytes":{"type":"integer"},"sha256":{"type":"string"},"chunks":{"type":"array","items":{"type":"string","pattern":"^[a-f0-9]{64}$"},"description":"Ordered chunk hashes; concatenate and verify full-file SHA256"}}},"MetricsSnapshot":{"type":"object","properties":{"port":{"type":"integer"},"collected_at":{"type":"string","format":"date-time"},"artifact":{"allOf":[{"$ref":"#/components/schemas/BenchmarkArtifact"}],"nullable":true},"error":{"type":"string","nullable":true},"histogram_totals":{"type":"object","additionalProperties":{"type":"number"}}}},"LeaderboardEntry":{"type":"object","properties":{"id":{"type":"string"},"model_name":{"type":"string"},"gpu_name":{"type":"string"},"engine":{"type":"string"},"best_output_tokens_per_sec":{"type":"number"},"power_draw_watts":{"type":"number"}}},"NodeRegister":{"type":"object","required":["device_code","gpu_name","gpu_vram_gb"],"properties":{"device_code":{"type":"string"},"gpu_name":{"type":"string"},"gpu_vram_gb":{"type":"number"},"capabilities":{"type":"array","items":{"type":"string"}}}},"NodeHeartbeat":{"type":"object","properties":{"gpu_utilization":{"type":"number"},"memory_used_gb":{"type":"number"},"current_model":{"type":"string"},"status":{"type":"string","enum":["idle","busy","error"]}}},"Job":{"type":"object","required":["id","suite_key","suite_version","type","model_id","params","timeout_sec","job_nonce","lease_expires_at","lease_protocol"],"properties":{"id":{"type":"string","format":"uuid"},"model_id":{"type":"string"},"suite_key":{"type":"string"},"suite_version":{"type":"string"},"type":{"type":"string"},"engine":{"type":["string","null"]},"params":{"type":"object","additionalProperties":true},"timeout_sec":{"type":"integer","minimum":1},"required_trust_tier":{"type":"string"},"job_nonce":{"type":"string","format":"uuid","description":"Fresh on every lease attempt; return as X-Job-Nonce."},"lease_expires_at":{"type":"string","format":"date-time"},"lease_protocol":{"type":"integer","const":1},"result_url":{"type":"string"},"events_url":{"type":"string"}}},"JobResult":{"type":"object","required":["success"],"properties":{"success":{"type":"boolean"},"retryable":{"type":"boolean","description":"Catalog failures requeue only when explicitly true for a transient error; legacy non-catalog jobs retain their existing default."},"error_code":{"type":["string","null"]},"error_message":{"type":["string","null"]},"metrics":{"type":"object","additionalProperties":true},"evidence":{"type":"object","additionalProperties":true},"artifacts":{"type":"array","items":{"type":"string"}}}},"Error":{"type":"object","properties":{"success":{"type":"boolean","const":false},"error":{"type":"string"},"request_id":{"type":"string"}}}},"securitySchemes":{"NodeTokenAuth":{"type":"apiKey","in":"header","name":"X-Node-Token","description":"Anonymous node credential; owned nodes require their owner authentication."},"ApiKeyAuth":{"type":"apiKey","in":"header","name":"X-API-Key","description":"API key for authenticated requests"},"SessionCookie":{"type":"apiKey","in":"cookie","name":"gpullmbench_session","description":"Session cookie from web login"}}}}