admin/modelTesting
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture kat-coder + qwen3-coder-30b outputs (5 prompts each, API-run, NOT graded yet)

Browse Source 
Via tools/grade_run.py against LM Studio (load -> all prompts -> unload -> verify).
kat-coder:      lfu/tts/webhook/automation/rust  (lfu has a real SyntaxError: 'self._ bump_freq')
qwen3-coder-30b: lfu/tts/webhook/automation/rust  (all 4 py parse clean; terse ~2k tok outputs)
Grading deferred to a later session (capture-only mode).

Script hardening:
  - --resident / --unload-all commands (read loaded_instances, POST unload, verify)
  - MEMORY GUARD: aborts if a different model is resident (never stack 30GB models)
  - TTFT self-discovery dump (LM Studio returns empty stats non-streaming;
    fix next session: switch to /api/v1/chat stream:true -> chat.end result.stats)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
admin
2026-07-28 19:22:41 -07:00
co-authored by Claude
parent b9f45a7c46
commit 0c79a997f5
12 changed files with 3910 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/grade_run.py
+99 -11
View File
@@ -42,10 +42,12 @@ PROMPTS = {
"rust": ("prompts/rust_service.txt", "rs"),
}
def api(base, path, payload=None, timeout=600):
def api(base, path, payload=None, timeout=600, method=None):
url = base.rstrip("/") + path
data = json.dumps(payload).encode() if payload else None
req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}, method="POST" if payload else "GET")
if method is None:
method = "POST" if payload is not None else "GET"
data = json.dumps(payload).encode() if payload is not None else None
req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}, method=method)
t0 = time.monotonic()
with urllib.request.urlopen(req, timeout=timeout) as r:
body = json.loads(r.read())
@@ -58,6 +60,47 @@ def list_models(base):
except Exception as e:
return []
# ---- memory-safe load/unload (NEVER have 2 big models resident at once) ----
def resident_instances(base):
"""Return list of (model_key, instance_id) for models actually in RAM.
Uses /api/v1/models -> loaded_instances (the ONLY reliable resident list;
/v1/models lists available models, not resident ones). Read-only, loads nothing."""
try:
body, _ = api(base, "/api/v1/models", None)
except urllib.error.HTTPError as e:
# endpoint may need GET not POST; fall back
body, _ = api(base, "/api/v1/models")
ms = body.get("data") or body.get("models") or []
out = []
for m in ms:
for inst in m.get("loaded_instances", []):
out.append((m.get("key") or m.get("id"), inst.get("id")))
return out
def unload(base, instance_id):
"""POST /api/v1/models/unload {instance_id}. Returns True on success."""
try:
body, _ = api(base, "/api/v1/models/unload", {"instance_id": instance_id}, timeout=60)
return body.get("instance_id") == instance_id or isinstance(body, dict)
except Exception as e:
print(f" unload FAILED for {instance_id}: {e}", file=sys.stderr)
return False
def unload_all(base, except_id=None):
"""Unload every resident instance except optionally one. Verify after."""
resid = resident_instances(base)
did = []
for key, iid in resid:
if except_id and (iid == except_id or key == except_id):
continue
if unload(base, iid):
did.append(iid)
else:
print(f" WARNING: could not unload {iid}", file=sys.stderr)
# verify
resid2 = resident_instances(base)
return did, resid2
def main():
ap = argparse.ArgumentParser(description="Run a prompt via LM Studio, save output + metrics.")
ap.add_argument("--lmstudio", default="http://10.0.0.31:1234", help="LM Studio base URL")
@@ -67,19 +110,44 @@ def main():
ap.add_argument("--list", action="store_true", help="Just list loaded models and exit")
ap.add_argument("--append", action="store_true", help="Append a draft (pending) entry to data/benchmark_history.json")
ap.add_argument("--max-tokens", type=int, default=8192, help="Max completion tokens")
ap.add_argument("--unload-all", action="store_true", help="Unload ALL resident models, verify, and exit (no run)")
ap.add_argument("--resident", action="store_true", help="Show what's actually resident in RAM and exit (no run)")
args = ap.parse_args()
# --list works standalone; otherwise model/prompt/name are required
if not args.list and not (args.model and args.prompt and args.name):
ap.error("--model, --prompt, and --name are required (unless --list)")
# standalone info/safety commands first
if args.unload_all:
print("=== unloading all resident models ===")
before = resident_instances(args.lmstudio)
print(" before:", [i for _, i in before] or "(none)")
did, after = unload_all(args.lmstudio)
print(" unloaded:", did or "(nothing to unload)")
print(" after (resident):", [i for _, i in after] or "(none — clean)")
return
if args.resident:
r = resident_instances(args.lmstudio)
print("Resident in RAM right now:")
for key, iid in r: print(f" {key} (instance {iid})")
if not r: print(" (none)")
return
if args.list:
ms = list_models(args.lmstudio)
print("Models loaded in LM Studio:")
print("Models available in LM Studio:")
for m in ms: print(" -", m)
if not ms: print(" (none / server not reachable)")
return
# --list/resident/unload-all work standalone; otherwise model/prompt/name required
if not (args.model and args.prompt and args.name):
ap.error("--model, --prompt, and --name are required (or use --list/--resident/--unload-all)")
# ---- MEMORY GUARD: refuse to stack a second big model ----
resid = resident_instances(args.lmstudio)
resident_ids = [iid for _, iid in resid]
if resid and args.model not in resident_ids and not any(args.model in (iid or "") or args.model == k for k, iid in resid):
print(f"!! ABORT: a DIFFERENT model is resident ({resid}) but you asked to run '{args.model}'.", file=sys.stderr)
print(f"!! Unload it first: python3 tools/grade_run.py --unload-all (or load '{args.model}' and unload others)", file=sys.stderr)
sys.exit(1)
prompt_file, ext = PROMPTS[args.prompt]
ppath = os.path.join(HERE, prompt_file)
if not os.path.exists(ppath):
@@ -101,13 +169,33 @@ def main():
content = resp["choices"][0]["message"]["content"]
usage = resp.get("usage", {})
# LM Studio / OpenAI-style usage: prompt_tokens, completion_tokens, total_tokens
# LM Studio reports timing under a stats/timings object — key names vary by
# version, so self-discover: walk known containers and pick the first numeric hit.
def _find(obj, keys):
for k in keys:
v = obj.get(k)
if isinstance(v, (int, float)):
return v
return None
stats = resp.get("stats") or {}
timings = resp.get("timings") or {}
# try stats then timings for each metric
tok_sec = _find(stats, ["tokens_per_second", "tokensPerSecond", "predicted_per_second", "predicted_tokens_per_second"]) \
or _find(timings, ["predicted_per_second", "tokens_per_second", "predicted_n"])
ttft = _find(stats, ["time_to_first_token", "timeToFirstToken", "ttft", "first_token_time"]) \
or _find(timings, ["time_to_first_token", "prompt_n", "prompt_per_second", "first_token"])
comp_tokens = usage.get("completion_tokens") or usage.get("completion_tokens_details", {}).get("reasoning_tokens", 0)
ttft = resp.get("stats", {}).get("time_to_first_token") or resp.get("timings", {}).get("prompt_n")
tok_sec = resp.get("stats", {}).get("tokens_per_second") or resp.get("timings", {}).get("predicted_per_second")
# fallback: derive tok/sec from wall time if the server didn't report it
if tok_sec is None and comp_tokens and wall:
tok_sec = round(comp_tokens / wall, 2)
# one-time schema dump so we can harden the extraction (write next to output)
if not os.environ.get("GRADE_RUN_NO_SCHEMA_DUMP"):
try:
schema_path = os.path.join(HERE, "outputs", ".last_response_schema.json")
slim = {k: v for k, v in resp.items() if k not in ("choices", "id", "object", "model", "created")}
json.dump(slim, open(schema_path, "w"), indent=2)
except Exception:
pass
# save the raw output — extract the code block if the model wrapped it in
# markdown fences or surrounded it with prose. Handles: (a) complete fenced