Grade 12-model backlog + fix native-API TTFT capture
Grading (12 new entries, 18→30 total in benchmark_history.json): - KAT-Coder v2.5 Dev XL: lfu 49 / tts 52 / webhook 72 / rust 36 / automation 60 - Qwen3 Coder 30B: lfu 45 / tts 44 / webhook 62 / rust 54 / automation 58 - Qwen 3.6 35B-A3B uncensored (automation): 46 - Gemma 4 26B-A4B (data): 86 [tests pass] All "coder" models scored Critical Bugs across prompts — plausible-looking async code with fatal bugs (broken LFU eviction, in-flight cancel no-op, submit() raising instead of backpressuring, un-awaited async read-through). grade_run.py: switch from OpenAI-compat /v1/chat/completions (empty stats) to native /api/v1/chat — returns full stats incl. time_to_first_token_seconds. Verified on Gemma-26B (53.5 t/s, ttft 1.13). Two native-API gotchas handled: input (string) not messages; max_output_tokens not max_tokens (that 400s). New capture: outputs/gemma-4-26b-a4b-data.py (native-API run). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Aygea
+38
-26
@@ -25,6 +25,13 @@ WHAT IT DOES
|
||||
5. Optionally --append writes a placeholder entry to data/benchmark_history.json
|
||||
with total_score:null, verdict:"pending", so the audit is the only manual bit.
|
||||
|
||||
STATS COME FROM THE NATIVE API, NOT OPENAI-COMPAT.
|
||||
LM Studio's OpenAI-compatible endpoint (/v1/chat/completions) returns an EMPTY
|
||||
stats object — use the native /api/v1/chat endpoint instead, which returns a
|
||||
full stats block including time_to_first_token_seconds. Request shape differs
|
||||
from OpenAI: pass {"model":..., "input": "<prompt text>", ...} (input is a
|
||||
string or array of input items, NOT messages).
|
||||
|
||||
NOTE: This only COLLECTS facts (runnability + metrics + file). The actual
|
||||
5-pillar audit is still done by the grader (me/you) — that part is subjective
|
||||
and shouldn't be faked.
|
||||
@@ -154,37 +161,41 @@ def main():
|
||||
sys.exit(f"prompt file not found: {ppath}")
|
||||
prompt_text = open(ppath).read()
|
||||
|
||||
print(f"→ POST {args.lmstudio}/v1/chat/completions model={args.model} prompt={args.prompt}")
|
||||
# NATIVE endpoint /api/v1/chat (not OpenAI-compat). The OpenAI endpoint
|
||||
# /v1/chat/completions returns an EMPTY stats object; the native endpoint
|
||||
# returns full stats including time_to_first_token_seconds. Request shape:
|
||||
# {"model":..., "input": "<prompt>", ...} — input is a string, NOT messages.
|
||||
# If stream:true is ever needed, parse the SSE `chat.end` event's result.stats
|
||||
# (identical schema to the non-streaming stats block below).
|
||||
print(f"→ POST {args.lmstudio}/api/v1/chat model={args.model} prompt={args.prompt}")
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"messages": [{"role": "user", "content": prompt_text}],
|
||||
"input": prompt_text,
|
||||
"temperature": 0.2,
|
||||
"max_tokens": args.max_tokens,
|
||||
"max_output_tokens": args.max_tokens, # native key (NOT max_tokens — that 400s)
|
||||
"stream": False,
|
||||
}
|
||||
try:
|
||||
resp, wall = api(args.lmstudio, "/v1/chat/completions", payload)
|
||||
resp, wall = api(args.lmstudio, "/api/v1/chat", payload)
|
||||
except urllib.error.URLError as e:
|
||||
sys.exit(f"LM Studio not reachable at {args.lmstudio} — is the server started and on 0.0.0.0? ({e})")
|
||||
|
||||
content = resp["choices"][0]["message"]["content"]
|
||||
usage = resp.get("usage", {})
|
||||
# LM Studio reports timing under a stats/timings object — key names vary by
|
||||
# version, so self-discover: walk known containers and pick the first numeric hit.
|
||||
def _find(obj, keys):
|
||||
for k in keys:
|
||||
v = obj.get(k)
|
||||
if isinstance(v, (int, float)):
|
||||
return v
|
||||
return None
|
||||
# native response: { model_instance_id, output:[ {type:"message",content:...}, ... ],
|
||||
# stats:{ input_tokens, total_output_tokens, reasoning_output_tokens,
|
||||
# tokens_per_second, time_to_first_token_seconds, model_load_time_seconds },
|
||||
# response_id }
|
||||
out_items = resp.get("output", [])
|
||||
content = ""
|
||||
for it in out_items:
|
||||
if it.get("type") == "message":
|
||||
content = it.get("content", "")
|
||||
break
|
||||
stats = resp.get("stats") or {}
|
||||
timings = resp.get("timings") or {}
|
||||
# try stats then timings for each metric
|
||||
tok_sec = _find(stats, ["tokens_per_second", "tokensPerSecond", "predicted_per_second", "predicted_tokens_per_second"]) \
|
||||
or _find(timings, ["predicted_per_second", "tokens_per_second", "predicted_n"])
|
||||
ttft = _find(stats, ["time_to_first_token", "timeToFirstToken", "ttft", "first_token_time"]) \
|
||||
or _find(timings, ["time_to_first_token", "prompt_n", "prompt_per_second", "first_token"])
|
||||
comp_tokens = usage.get("completion_tokens") or usage.get("completion_tokens_details", {}).get("reasoning_tokens", 0)
|
||||
tok_sec = stats.get("tokens_per_second")
|
||||
ttft = stats.get("time_to_first_token_seconds")
|
||||
comp_tokens = stats.get("total_output_tokens")
|
||||
input_tokens = stats.get("input_tokens")
|
||||
total_tokens = (input_tokens or 0) + (comp_tokens or 0) if (input_tokens or comp_tokens) else None
|
||||
# fallback: derive tok/sec from wall time if the server didn't report it
|
||||
if tok_sec is None and comp_tokens and wall:
|
||||
tok_sec = round(comp_tokens / wall, 2)
|
||||
@@ -192,7 +203,8 @@ def main():
|
||||
if not os.environ.get("GRADE_RUN_NO_SCHEMA_DUMP"):
|
||||
try:
|
||||
schema_path = os.path.join(HERE, "outputs", ".last_response_schema.json")
|
||||
slim = {k: v for k, v in resp.items() if k not in ("choices", "id", "object", "model", "created")}
|
||||
slim = {k: v for k, v in resp.items() if k != "output"}
|
||||
slim["_output_types"] = [it.get("type") for it in out_items]
|
||||
json.dump(slim, open(schema_path, "w"), indent=2)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -222,10 +234,10 @@ def main():
|
||||
open(out_path, "w").write(text)
|
||||
print(f"✓ saved {len(text)} bytes -> {os.path.relpath(out_path, HERE)}")
|
||||
|
||||
print("\n=== METRICS (from API) ===")
|
||||
print("\n=== METRICS (from native /api/v1/chat stats) ===")
|
||||
print(f" tok/sec : {tok_sec}")
|
||||
print(f" total tokens : {usage.get('total_tokens')}")
|
||||
print(f" TTFT : {ttft}")
|
||||
print(f" total tokens : {total_tokens} (in={input_tokens} out={comp_tokens})")
|
||||
print(f" TTFT (sec) : {ttft}")
|
||||
|
||||
print("\n=== DRAFT JSON ENTRY (fill breakdown + audit by hand) ===")
|
||||
entry = {
|
||||
@@ -236,7 +248,7 @@ def main():
|
||||
"quant": "TODO",
|
||||
"format": "mlx",
|
||||
"tok_sec": tok_sec,
|
||||
"total_tokens": usage.get("total_tokens"),
|
||||
"total_tokens": total_tokens,
|
||||
"ttft_sec": ttft,
|
||||
"filename": f"outputs/{args.name}.{ext}",
|
||||
"tests_pass": None, # grader runs the file
|
||||
|
||||
Reference in New Issue
Block a user