adminandClaude
e3e5fded6d
Runnability-first split: TRUSTED/VERIFY (runs clean) vs AVOID (crashes). Only 5/10 local models produce runnable code. Co-Authored-By: Claude <noreply@anthropic.com>
654 lines
36 KiB
Python
654 lines
36 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generator: builds dashboard.html + pages/<slug>..html from data/benchmark_history.json.
|
|
Re-run after each grading batch to regenerate everything.
|
|
Cyberpunk-terminal aesthetic. Pure stdlib + Chart.js via CDN.
|
|
"""
|
|
import json, html, os, sys, re as _re
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
DATA = os.path.join(HERE, "data", "benchmark_history.json")
|
|
OUT_DASH = os.path.join(HERE, "dashboard.html")
|
|
PAGES_DIR = os.path.join(HERE, "pages")
|
|
|
|
PILLARS = ["complexity", "concurrency", "isolation", "memory_edge_cases", "test_integrity"]
|
|
PILLAR_LABELS = {
|
|
"complexity": "Complexity (O(1))",
|
|
"concurrency": "Concurrency / Races",
|
|
"isolation": "Tx Isolation",
|
|
"memory_edge_cases": "Memory & Edges",
|
|
"test_integrity": "Test Integrity",
|
|
}
|
|
|
|
# ---- cyberpunk palette ----
|
|
NEON_CYAN = "#00ffc8"
|
|
NEON_MAG = "#ff2bd6"
|
|
NEON_LIME = "#b6ff3a"
|
|
NEON_AMBER= "#ffb000"
|
|
NEON_RED = "#ff3b5c"
|
|
BG = "#0a0a0f"
|
|
PANEL = "#101018"
|
|
INK = "#d7e0e6" # body text
|
|
INK_DIM = "#7a8590" # muted
|
|
|
|
def verdict_meta(v):
|
|
return {
|
|
"Production Ready": (NEON_LIME, "PROD"),
|
|
"Minor Logic Flaws": (NEON_AMBER, "FLAWS"),
|
|
"Critical Bugs": (NEON_RED, "CRIT"),
|
|
"Broken / Unusable": (NEON_RED, "DEAD"),
|
|
}.get(v, (INK_DIM, "???"))
|
|
|
|
def esc(s):
|
|
return html.escape(str(s)) if s is not None else ""
|
|
|
|
def speed_str(m):
|
|
if m.get("tok_sec") is None:
|
|
return "N/A"
|
|
return f"{m['tok_sec']:.1f}"
|
|
|
|
# series colors for charts (cycles through neon accents)
|
|
SERIES = [NEON_CYAN, NEON_MAG, NEON_LIME, NEON_AMBER, "#5b8cff", "#c084fc", "#34d399", "#f472b6"]
|
|
|
|
# shared <head> CSS — embedded so each page is standalone
|
|
def shared_css():
|
|
return """
|
|
:root{
|
|
--bg:#0a0a0f; --panel:#101018; --panel2:#14141f; --ink:#d7e0e6; --dim:#7a8590;
|
|
--cyan:#00ffc8; --mag:#ff2bd6; --lime:#b6ff3a; --amber:#ffb000; --red:#ff3b5c; --blue:#5b8cff;
|
|
--grid: rgba(0,255,200,0.04);
|
|
}
|
|
*{box-sizing:border-box}
|
|
html,body{margin:0;padding:0}
|
|
body{
|
|
background:var(--bg); color:var(--ink);
|
|
font-family:'Fira Sans',system-ui,-apple-system,sans-serif;
|
|
line-height:1.6; min-height:100vh;
|
|
/* scanline + grid texture overlay */
|
|
background-image:
|
|
repeating-linear-gradient(0deg, rgba(0,255,200,0.025) 0px, rgba(0,255,200,0.025) 1px, transparent 1px, transparent 3px),
|
|
linear-gradient(var(--grid) 1px, transparent 1px),
|
|
linear-gradient(90deg, var(--grid) 1px, transparent 1px),
|
|
radial-gradient(1200px 600px at 70% -10%, rgba(91,139,255,0.10), transparent 60%),
|
|
radial-gradient(900px 500px at 10% 110%, rgba(255,43,214,0.08), transparent 60%);
|
|
background-size: 3px 3px, 44px 44px, 44px 44px, 100% 100%, 100% 100%;
|
|
background-attachment: fixed;
|
|
}
|
|
mono,.mono,h1,h2,h3,.hud,th,.code,.badge,.stat-num,.lead-model,.verdict-chip{font-family:'Fira Code',ui-monospace,'SF Mono',Menlo,monospace}
|
|
a{color:var(--cyan);text-decoration:none}
|
|
a:hover{text-shadow:0 0 8px var(--cyan)}
|
|
.wrap{max-width:1180px;margin:0 auto;padding:28px 20px 80px}
|
|
header.hud-bar{
|
|
border:1px solid rgba(0,255,200,0.25);
|
|
background:linear-gradient(180deg,rgba(0,255,200,0.06),rgba(0,0,0,0));
|
|
box-shadow:0 0 24px rgba(0,255,200,0.08), inset 0 0 30px rgba(0,0,0,0.5);
|
|
border-radius:6px; padding:18px 22px; margin-bottom:24px;
|
|
position:relative; overflow:hidden;
|
|
}
|
|
header.hud-bar::before{
|
|
content:""; position:absolute; inset:0;
|
|
background:repeating-linear-gradient(90deg, transparent 0 7px, rgba(0,255,200,0.03) 7px 8px);
|
|
pointer-events:none;
|
|
}
|
|
.kicker{color:var(--cyan); letter-spacing:.32em; font-size:.72rem; text-transform:uppercase; text-shadow:0 0 10px rgba(0,255,200,0.5)}
|
|
h1{font-size:1.5rem; margin:.3em 0 0; letter-spacing:.02em; text-shadow:0 0 16px rgba(0,255,200,0.35)}
|
|
.subtitle{color:var(--dim); font-size:.9rem; margin-top:.35em}
|
|
.stats{display:grid; grid-template-columns:repeat(4,1fr); gap:12px; margin:18px 0 26px}
|
|
.stat{background:var(--panel); border:1px solid rgba(255,255,255,0.06); border-radius:6px; padding:14px 16px; position:relative; overflow:hidden}
|
|
.stat::after{content:"";position:absolute;left:0;top:0;bottom:0;width:3px;background:var(--cyan);box-shadow:0 0 12px var(--cyan)}
|
|
.stat .lbl{color:var(--dim);font-size:.68rem;letter-spacing:.18em;text-transform:uppercase}
|
|
.stat .num{font-size:1.6rem;margin-top:4px;color:var(--ink)}
|
|
/* hero top row: squished stat tiles beside the radar */
|
|
.toprow{display:grid;grid-template-columns:1.35fr 1fr;gap:18px;align-items:stretch;margin:18px 0 22px}
|
|
.stats-squish{display:grid;grid-template-columns:repeat(2,1fr);gap:12px;align-content:start}
|
|
.stats-squish .stat{padding:16px 18px}
|
|
.stats-squish .stat .num{font-size:1.9rem}
|
|
.radar-panel{display:flex;flex-direction:column}
|
|
.radar-panel .chart-box{flex:1;min-height:220px}
|
|
/* compact top-performers list */
|
|
.toplist{display:flex;flex-direction:column;gap:6px}
|
|
a.tr{display:grid;grid-template-columns:28px 1fr auto auto auto;align-items:center;gap:10px;padding:8px 12px;background:rgba(0,0,0,0.2);border:1px solid rgba(255,255,255,0.05);border-radius:5px;transition:all .16s;cursor:pointer;text-decoration:none;font-size:.84rem}
|
|
a.tr:hover{border-color:rgba(0,255,200,0.4);background:rgba(0,255,200,0.05)}
|
|
.tr-r{color:var(--dim)}
|
|
.tr-n{color:var(--ink)}
|
|
.tr-q{color:var(--dim);font-size:.74rem}
|
|
.tr-s{color:var(--mag);font-size:.78rem}
|
|
.tr-sc{font-size:1rem;font-weight:600;text-align:right;min-width:30px}
|
|
@media(max-width:900px){.toprow{grid-template-columns:1fr}.grid2{grid-template-columns:1fr}.stats,.stats-squish{grid-template-columns:repeat(2,1fr)}}
|
|
.grid2{display:grid;grid-template-columns:1.1fr .9fr;gap:18px;margin-bottom:26px}
|
|
.panel{background:var(--panel);border:1px solid rgba(255,255,255,0.07);border-radius:8px;padding:18px}
|
|
.panel h2{font-size:.95rem;letter-spacing:.12em;text-transform:uppercase;color:var(--cyan);margin:0 0 14px;text-shadow:0 0 10px rgba(0,255,200,0.35)}
|
|
.chart-box{position:relative;height:340px}
|
|
@media(max-width:900px){.grid2{grid-template-columns:1fr}}
|
|
/* leaderboard */
|
|
table{width:100%;border-collapse:collapse;font-size:.86rem}
|
|
thead th{text-align:left;color:var(--dim);font-size:.66rem;letter-spacing:.16em;text-transform:uppercase;border-bottom:1px solid rgba(0,255,200,0.2);padding:8px 10px}
|
|
tbody td{padding:11px 10px;border-bottom:1px solid rgba(255,255,255,0.05);vertical-align:middle}
|
|
tbody tr{transition:background .15s, box-shadow .15s}
|
|
tbody tr:hover{background:rgba(0,255,200,0.05);box-shadow:inset 0 0 0 1px rgba(0,255,200,0.25)}
|
|
.rank{color:var(--dim);width:34px}
|
|
.rank.top{color:var(--lime);text-shadow:0 0 8px var(--lime)}
|
|
.model-name{color:var(--ink)}
|
|
.quant{color:var(--dim);font-size:.78rem}
|
|
.badge{display:inline-block;padding:2px 9px;border-radius:3px;font-size:.68rem;letter-spacing:.1em;border:1px solid currentColor}
|
|
.btn{display:inline-block;padding:5px 12px;border:1px solid var(--cyan);color:var(--cyan);border-radius:4px;font-size:.74rem;letter-spacing:.08em;cursor:pointer;transition:all .18s}
|
|
.btn:hover{background:rgba(0,255,200,0.12);box-shadow:0 0 14px rgba(0,255,200,0.4)}
|
|
.bar-cell{display:flex;align-items:center;gap:10px}
|
|
.score-bar{flex:1;height:8px;background:rgba(255,255,255,0.06);border-radius:2px;overflow:hidden;min-width:70px}
|
|
.score-bar > i{display:block;height:100%;border-radius:2px;transition:width .4s, box-shadow .2s}
|
|
tr:hover .score-bar > i{box-shadow:0 0 12px currentColor}
|
|
.caveat{color:var(--amber);font-size:.72rem}
|
|
.cloud-tag{color:var(--blue);font-size:.7rem;border:1px solid rgba(91,139,255,.4);padding:1px 6px;border-radius:3px;margin-left:6px}
|
|
.quant-cell{color:var(--ink);font-size:.82rem;white-space:nowrap}
|
|
.fmt-chip{display:inline-block;font-size:.66rem;letter-spacing:.08em;padding:2px 7px;border-radius:3px;border:1px solid currentColor;font-family:'Fira Code',monospace}
|
|
.fmt-mlx{color:var(--cyan);background:rgba(0,255,200,0.08)}
|
|
.fmt-gguf{color:var(--mag);background:rgba(255,43,214,0.08)}
|
|
.fmt-cloud{color:var(--blue);background:rgba(91,139,255,0.08)}
|
|
/* format/quant showdown cards */
|
|
.fcard{background:var(--panel2);border:1px solid rgba(255,255,255,0.06);border-radius:6px;padding:12px 14px;margin-bottom:10px}
|
|
.fcard-h{display:flex;justify-content:space-between;align-items:baseline;gap:10px;flex-wrap:wrap;margin-bottom:9px}
|
|
.fcard-n{color:var(--ink);font-weight:600;font-family:'Fira Code',monospace;font-size:.92rem}
|
|
.fcard-meta{color:var(--dim);font-size:.72rem}
|
|
.fcard-v{display:flex;flex-wrap:wrap;gap:8px}
|
|
a.fv{display:grid;grid-template-columns:auto auto auto auto;align-items:center;gap:10px;padding:7px 11px;background:rgba(0,0,0,0.25);border:1px solid rgba(0,255,200,0.15);border-radius:5px;transition:all .18s;cursor:pointer;text-decoration:none}
|
|
a.fv:hover{border-color:var(--cyan);box-shadow:0 0 12px rgba(0,255,200,0.3);background:rgba(0,255,200,0.06)}
|
|
.fv-q{color:var(--ink);font-size:.82rem;min-width:64px}
|
|
.fv-f{font-size:.7rem;letter-spacing:.08em;font-family:'Fira Code',monospace}
|
|
.fv-s{color:var(--dim);font-size:.78rem}
|
|
.fv-sc{font-size:1rem;font-weight:600;min-width:28px;text-align:right}
|
|
/* key findings */
|
|
.findings{display:grid;grid-template-columns:repeat(4,1fr);gap:12px;margin-bottom:16px}
|
|
.fnd{background:var(--panel2);border:1px solid rgba(255,255,255,0.06);border-radius:6px;padding:14px 16px;position:relative;overflow:hidden}
|
|
.fnd::after{content:"";position:absolute;left:0;top:0;bottom:0;width:3px;background:var(--mag);box-shadow:0 0 12px var(--mag)}
|
|
.fnd-n{font-size:1.7rem;color:var(--cyan);text-shadow:0 0 10px rgba(0,255,200,0.3)}
|
|
.fnd-l{color:var(--ink);font-size:.72rem;letter-spacing:.12em;text-transform:uppercase;margin-top:4px;font-family:'Fira Code',monospace}
|
|
.fnd-s{color:var(--dim);font-size:.72rem;margin-top:6px;line-height:1.45}
|
|
ul.findings-notes{list-style:none;padding:0;margin:0}
|
|
ul.findings-notes li{padding:8px 0 8px 20px;border-bottom:1px solid rgba(255,255,255,0.05);position:relative;font-size:.84rem;color:var(--ink);line-height:1.5}
|
|
ul.findings-notes li:last-child{border-bottom:none}
|
|
ul.findings-notes li::before{content:"▸";position:absolute;left:0;color:var(--mag)}
|
|
ul.findings-notes code{background:rgba(0,255,200,0.1);color:var(--cyan);padding:1px 5px;border-radius:3px;font-family:'Fira Code',monospace;font-size:.8rem}
|
|
@media(max-width:900px){.findings{grid-template-columns:repeat(2,1fr)}}
|
|
/* cloud-offload strategy */
|
|
.offload-panel{border:1px solid rgba(0,255,200,0.18)}
|
|
.offload-intro{color:var(--ink);font-size:.86rem;line-height:1.55;margin-bottom:16px;padding:12px 14px;background:rgba(0,0,0,0.25);border-left:3px solid var(--cyan);border-radius:0 5px 5px 0}
|
|
.tiers{display:grid;grid-template-columns:1fr 1fr;gap:14px}
|
|
.tier-h{font-family:'Fira Code',monospace;font-size:.74rem;letter-spacing:.08em;text-transform:uppercase;margin-bottom:8px;padding-bottom:6px;border-bottom:1px solid currentColor}
|
|
.tier-list{display:flex;flex-direction:column;gap:6px}
|
|
.tier-list .empty{color:var(--dim);font-size:.8rem;padding:8px}
|
|
a.tr{grid-template-columns:1fr auto auto auto auto !important}
|
|
.tier-badge{font-size:.6rem;letter-spacing:.08em;padding:1px 6px;border:1px solid currentColor;border-radius:3px;font-family:'Fira Code',monospace}
|
|
@media(max-width:760px){.tiers{grid-template-columns:1fr}}
|
|
footer{color:var(--dim);font-size:.74rem;margin-top:40px;border-top:1px solid rgba(255,255,255,0.06);padding-top:14px;text-align:center}
|
|
@media (prefers-reduced-motion: reduce){*{animation:none!important;transition:none!important}}
|
|
"""
|
|
|
|
def head_html(title):
|
|
return f"""<!doctype html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>{esc(title)}</title>
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
|
<link href="https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;600;700&family=Fira+Sans:wght@300;400;500;600;700&display=swap" rel="stylesheet">
|
|
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.1/dist/chart.umd.min.js"></script>
|
|
<style>{shared_css()}</style>
|
|
</head>
|
|
<body><div class="wrap">
|
|
"""
|
|
|
|
FOOT = """</div>
|
|
<script>if(window.matchMedia&&window.matchMedia('(prefers-reduced-motion: reduce)').matches&&window.Chart){Chart.defaults.animation=false;Chart.defaults.animations.colors=false;Chart.defaults.animations.numbers=false;}</script>
|
|
</body></html>"""
|
|
|
|
def render_dashboard(data):
|
|
models = sorted(data["models"], key=lambda m: -m["total_score"])
|
|
n = len(models)
|
|
avg = sum(m["total_score"] for m in models) / n if n else 0
|
|
top = models[0] if models else None
|
|
prod = sum(1 for m in models if m["verdict"] == "Production Ready")
|
|
|
|
# chart datasets (exclude cloud model from speed bars — has no tok/sec)
|
|
local = [m for m in models if m.get("tok_sec") is not None]
|
|
local_sorted = sorted(local, key=lambda m: -m["tok_sec"])
|
|
|
|
radar_models = [m for m in models if m.get("tok_sec") is not None or True][:3] # top 3 by score already
|
|
|
|
rows = []
|
|
for i, m in enumerate(models, 1):
|
|
col, chip = verdict_meta(m["verdict"])
|
|
caveat = ""
|
|
if m.get("speed_caveat"):
|
|
caveat = '<div class="caveat">⚠ speed suspect</div>'
|
|
fmt = (m.get("format") or "").lower()
|
|
if fmt == "cloud":
|
|
fmt_chip = '<span class="fmt-chip fmt-cloud">CLOUD</span>'
|
|
elif fmt == "gguf":
|
|
fmt_chip = '<span class="fmt-chip fmt-gguf">GGUF</span>'
|
|
elif fmt == "mlx":
|
|
fmt_chip = '<span class="fmt-chip fmt-mlx">MLX</span>'
|
|
else:
|
|
fmt_chip = f'<span class="fmt-chip">{esc(m.get("format") or "—")}</span>'
|
|
bar_color = col
|
|
rows.append(f"""<tr>
|
|
<td class="rank {'top' if i<=3 else ''}">#{i}</td>
|
|
<td><div class="model-name">{esc(m['model_name'])}</div>{caveat}</td>
|
|
<td><span class="quant-cell mono">{esc(m['quant'])}</span></td>
|
|
<td>{fmt_chip}</td>
|
|
<td class="mono">{speed_str(m)} <span style="color:var(--dim);font-size:.72rem">t/s</span></td>
|
|
<td><div class="bar-cell"><span class="mono" style="width:34px;color:{col}">{m['total_score']}</span>
|
|
<span class="score-bar"><i style="width:{m['total_score']}%;background:{bar_color};color:{bar_color}"></i></span></div></td>
|
|
<td><span class="badge verdict-chip" style="color:{col}">{chip}</span></td>
|
|
<td style="color:var(--dim);font-size:.8rem">{esc(m['best_for'])[:60]}…</td>
|
|
<td><a class="btn" href="pages/{esc(m['id'])}.html">DECODE ▸</a></td>
|
|
</tr>""")
|
|
|
|
# JSON for charts
|
|
def short_label(m):
|
|
# family + quant so duplicates (same model, different quant) are distinguishable
|
|
fam = m["model_name"].split("(")[0].strip()
|
|
fam = _re.sub(r"(?i)\s+(uncensored|heretic|aggressive|hauhaucs).*$", "", fam)
|
|
q = (m.get("quant") or "").strip()
|
|
name = f"{fam} · {q}" if q else fam
|
|
return name[:28]
|
|
bar_labels = json.dumps([short_label(m) for m in local_sorted])
|
|
bar_speed = json.dumps([m["tok_sec"] for m in local_sorted])
|
|
bar_score = json.dumps([m["total_score"] for m in local_sorted])
|
|
|
|
radar_labels = json.dumps([PILLAR_LABELS[p] for p in PILLARS])
|
|
radar_sets = []
|
|
for idx, m in enumerate(radar_models):
|
|
col = SERIES[idx % len(SERIES)]
|
|
radar_sets.append({
|
|
"label": m["model_name"][:24],
|
|
"data": [m["breakdown"][p] for p in PILLARS],
|
|
"borderColor": col,
|
|
"backgroundColor": col + "22",
|
|
})
|
|
radar_json = json.dumps(radar_sets)
|
|
|
|
stats_tiles = f"""
|
|
<div class="stat"><div class="lbl">Models Tested</div><div class="num mono">{n}</div></div>
|
|
<div class="stat"><div class="lbl">Top Score</div><div class="num mono" style="color:var(--lime)">{top['total_score'] if top else '—'}</div></div>
|
|
<div class="stat"><div class="lbl">Average</div><div class="num mono" style="color:var(--amber)">{avg:.1f}</div></div>
|
|
<div class="stat"><div class="lbl">Prod-Ready</div><div class="num mono" style="color:var(--cyan)">{prod}/{n}</div></div>
|
|
"""
|
|
stats = f'<div class="stats">{stats_tiles}</div>'
|
|
top_name = esc(top["model_name"]) if top else "—"
|
|
|
|
# compact top-5 list for the right-of-radar panel
|
|
trows = []
|
|
for i, m in enumerate(models[:5], 1):
|
|
col, _ = verdict_meta(m["verdict"])
|
|
speed = f"{m['tok_sec']:.1f}" if m.get("tok_sec") is not None else "—"
|
|
trows.append(
|
|
f'<a class="tr" href="pages/{esc(m["id"])}.html">'
|
|
f'<span class="tr-r mono">#{i}</span>'
|
|
f'<span class="tr-n">{esc(m["model_name"][:20])}</span>'
|
|
f'<span class="tr-q mono">{esc(m["quant"])}</span>'
|
|
f'<span class="tr-s mono">{speed}</span>'
|
|
f'<span class="tr-sc mono" style="color:{col}">{m["total_score"]}</span></a>'
|
|
)
|
|
toplist = ''.join(trows)
|
|
|
|
# ---- Format / Quant showdown: group local models by base family ----
|
|
def family_of(m):
|
|
# strip parentheticals, uncensored/merge tags, quant words, format, params
|
|
name = m["model_name"].split("(")[0].strip()
|
|
name = _re.sub(r"(?i)\b(uncensored|heretic|aggressive|hauhaucs|coder|composer|fable5|v\d+\.\d+|merge)\b", "", name)
|
|
name = _re.sub(r"\b\d+(\.\d+)?[bB](-[aA]\d+[bB])?\b", "", name) # sizes: 35B, 26B-A4B
|
|
name = _re.sub(r"\s+", " ", name).strip(" -")
|
|
# collapse known families
|
|
for fam in ["Qwen 3.6", "Qwen3", "Gemma 4", "KAT-Coder", "DeepSeek"]:
|
|
if _re.sub(r"\s+", "", name).lower().startswith(_re.sub(r"\s+", "", fam).lower()):
|
|
return fam
|
|
return name or m["model_name"]
|
|
|
|
families = {}
|
|
for m in models:
|
|
if m.get("format") == "cloud":
|
|
continue
|
|
f = family_of(m)
|
|
families.setdefault(f, []).append(m)
|
|
# only show families with >=2 variants (the interesting comparisons)
|
|
multi = {f: ms for f, ms in families.items() if len(ms) >= 2}
|
|
|
|
if multi:
|
|
fcards = []
|
|
for fam, ms in sorted(multi.items(), key=lambda kv: -max(x["total_score"] for x in kv[1])):
|
|
ms_sorted = sorted(ms, key=lambda x: -x["total_score"])
|
|
best = ms_sorted[0]
|
|
spread = max(x["total_score"] for x in ms) - min(x["total_score"] for x in ms)
|
|
variants = []
|
|
for x in ms_sorted:
|
|
fmt = (x.get("format") or "").upper()
|
|
fcol = "var(--cyan)" if x.get("format") == "mlx" else ("var(--mag)" if x.get("format") == "gguf" else "var(--dim)")
|
|
col, _ = verdict_meta(x["verdict"])
|
|
speed = f"{x['tok_sec']:.0f} t/s" if x.get("tok_sec") is not None else "—"
|
|
variants.append(
|
|
f'<a class="fv" href="pages/{esc(x["id"])}.html">'
|
|
f'<span class="fv-q mono">{esc(x["quant"])}</span>'
|
|
f'<span class="fv-f" style="color:{fcol}">{fmt}</span>'
|
|
f'<span class="fv-s mono">{speed}</span>'
|
|
f'<span class="fv-sc mono" style="color:{col}">{x["total_score"]}</span>'
|
|
f'</a>'
|
|
)
|
|
fcards.append(f"""
|
|
<div class="fcard">
|
|
<div class="fcard-h"><span class="fcard-n">{esc(fam)}</span>
|
|
<span class="fcard-meta">{len(ms)} variants · score spread <b class="mono">{spread}</b> · best <b class="mono" style="color:var(--lime)">{best["total_score"]}</b> ({esc(best["quant"])})</span></div>
|
|
<div class="fcard-v">{''.join(variants)}</div>
|
|
</div>""")
|
|
family_panel = f"""
|
|
<div class="panel" style="margin-bottom:26px">
|
|
<h2>▮ FORMAT & QUANT SHOWDOWN — same family, different quants/formats</h2>
|
|
<div style="color:var(--dim);font-size:.76rem;margin-bottom:12px">Families with 2+ variants. Click a row for the full audit. Compare how quant depth and MLX-vs-GGUF change the score.</div>
|
|
{''.join(fcards)}
|
|
</div>"""
|
|
else:
|
|
family_panel = ""
|
|
|
|
# ---- Key findings: real stats computed from the data ----
|
|
import ast as _ast, os as _os
|
|
def _scan_file(m):
|
|
fn = m.get("filename", "")
|
|
path = _os.path.join(HERE, fn) if not _os.path.isabs(fn) else fn
|
|
try:
|
|
txt = open(path).read()
|
|
parses = True
|
|
try: _ast.parse(txt)
|
|
except Exception: parses = False
|
|
return {
|
|
"slots": txt.count("__slots__") > 0,
|
|
"monotonic": txt.count("monotonic") > 0,
|
|
"linear": any(p in txt for p in ("sorted(", ".sort(", "heapq", "min(", "max(")),
|
|
"parses": parses,
|
|
}
|
|
except Exception:
|
|
return {"slots": False, "monotonic": False, "linear": False, "parses": None}
|
|
|
|
scans = {m["id"]: _scan_file(m) for m in local}
|
|
n_run = sum(1 for m in local if m.get("tests_pass"))
|
|
n_crit = sum(1 for m in local if m["verdict"] == "Critical Bugs")
|
|
n_slots = sum(1 for m in local if scans[m["id"]]["slots"])
|
|
n_mono = sum(1 for m in local if scans[m["id"]]["monotonic"])
|
|
pillar_avg = {p: round(sum(m["breakdown"][p] for m in local)/len(local), 1) for p in PILLARS}
|
|
weakest = min(PILLARS, key=lambda p: pillar_avg[p])
|
|
|
|
def _fcard(num, label, sub):
|
|
return (f'<div class="fnd"><div class="fnd-n mono">{num}</div>'
|
|
f'<div class="fnd-l">{label}</div>'
|
|
f'<div class="fnd-s">{sub}</div></div>')
|
|
|
|
findings_tiles = "".join([
|
|
_fcard(f"{n_run}/{len(local)}", "RUN THEIR OWN TESTS",
|
|
"Half of local models crash before completing — runnability is the real filter."),
|
|
_fcard(f"{n_crit}/{len(local)}", "CRITICAL BUGS",
|
|
"Cache corruption, evict-crashes, or fatal KeyErrors — not safe for systems work."),
|
|
_fcard(f"{n_slots}/{len(local)}", "DECLARE __slots__",
|
|
"Rubric explicitly required it for memory efficiency; nearly all models miss it."),
|
|
_fcard(f"{n_mono}/{len(local)}", "USE time.monotonic()",
|
|
"The rest use the system clock — NTP jumps corrupt TTL eviction."),
|
|
])
|
|
findings_panel = f"""
|
|
<div class="panel" style="margin-bottom:26px">
|
|
<h2>▮ KEY FINDINGS — patterns across {len(local)} local models</h2>
|
|
<div class="findings">{findings_tiles}</div>
|
|
<ul class="findings-notes">
|
|
<li><b>Quant depth dominates quality.</b> Same model, different quant: Qwen 3.6 35B-A3B scores <span class="mono" style="color:var(--lime)">82</span> at 6-bit but <span class="mono" style="color:var(--red)">57</span> at 4-bit — a ~25-point drop. Aggressive quants cost real logic on systems code.</li>
|
|
<li><b>Speed ≠ quality.</b> The 4-bit Qwen is the <i>fastest</i> (83 t/s) yet scores 57; the 6-bit is slower (69 t/s) but scores 82. Pick quants for correctness first, throughput second.</li>
|
|
<li><b>Concurrency is the killer pillar</b> (avg <span class="mono">{pillar_avg['concurrency']}/20</span>). Local models most often break on async correctness — lock type mismatches, races, and non-reentrant-lock deadlocks.</li>
|
|
<li><b>The weakest pillar overall is {PILLAR_LABELS[weakest]}</b> (avg <span class="mono">{pillar_avg[weakest]}/20</span>). Test suites that ship with crashing code validate nothing.</li>
|
|
<li><b>Only the cloud baseline (DeepSeek, 91) cleared Production-Ready.</b> Best local scores cap at 82 — strong scaffolding, but every submission needs a human pass on <code>__slots__</code>, monotonic clocks, and lock granularity.</li>
|
|
</ul>
|
|
</div>"""
|
|
|
|
# ---- Cloud-offload strategy panel (the actual goal: a backup for cloud credits) ----
|
|
# A model is a useful offload ONLY if its code runs. Rank runnability-first.
|
|
ranked = sorted(local, key=lambda m: (-int(m.get("tests_pass", False)), -m["total_score"]))
|
|
runs = [m for m in local if m.get("tests_pass")]
|
|
crashers = [m for m in local if not m.get("tests_pass")]
|
|
fast_clean = [m for m in local if m.get("tests_pass") and m.get("tok_sec") and not m.get("speed_caveat")]
|
|
fast_clean.sort(key=lambda m: -m["tok_sec"])
|
|
|
|
def _tier_row(m, badge, color):
|
|
t = f"{m['tok_sec']:.0f} t/s" if m.get("tok_sec") else "—"
|
|
return (f'<a class="tr" href="pages/{esc(m["id"])}.html">'
|
|
f'<span class="tr-n">{esc(m["model_name"][:24])}</span>'
|
|
f'<span class="tr-q mono">{esc(m["quant"])}</span>'
|
|
f'<span class="tr-s mono">{t}</span>'
|
|
f'<span class="tr-sc mono" style="color:{color}">{m["total_score"]}</span>'
|
|
f'<span class="tier-badge" style="color:{color};border-color:{color}">{badge}</span></a>')
|
|
|
|
tier_rows = []
|
|
if runs:
|
|
# top runner(s) = recommended offload
|
|
for m in runs:
|
|
tier_rows.append(_tier_row(m, "TRUSTED" if m["total_score"] >= 78 else "VERIFY", verdict_meta(m["verdict"])[0]))
|
|
crash_rows = [_tier_row(m, "AVOID", "var(--red)") for m in crashers]
|
|
|
|
offload_panel = f"""
|
|
<div class="panel offload-panel" style="margin-bottom:26px">
|
|
<h2>▮ CLOUD-OFFLOAD STRATEGY — what to actually run when rationing cloud credits</h2>
|
|
<div class="offload-intro">
|
|
Goal: a <b>trustworthy local backup</b> for when your cloud-LLM plan runs low. A model only earns offload
|
|
duty if its code <b>runs</b> — a high score that crashes wastes your time <i>and</i> still costs a cloud call afterward.
|
|
Only <b class="mono" style="color:var(--lime)">{len(runs)}/{len(local)}</b> local models produced runnable code;
|
|
<b class="mono" style="color:var(--red)">{len(crashers)}/{len(local)}</b> crashed.
|
|
</div>
|
|
<div class="tiers">
|
|
<div class="tier"><div class="tier-h" style="color:var(--lime)">✓ RUNS — offload candidates (verify output, then ship)</div>
|
|
<div class="tier-list">{''.join(tier_rows) if tier_rows else '<div class="empty">none</div>'}</div></div>
|
|
<div class="tier"><div class="tier-h" style="color:var(--red)">✗ CRASHES — do NOT offload (wasted cycle + cloud call anyway)</div>
|
|
<div class="tier-list">{''.join(crash_rows) if crash_rows else '<div class="empty">none</div>'}</div></div>
|
|
</div>
|
|
<ul class="findings-notes" style="margin-top:14px">
|
|
<li><b>Rule of thumb:</b> route a task to local only if the top runner above handles it; if the local output doesn't run on first try, <b>stop and use the cloud model</b> — debugging bad local code usually costs more than the API call you tried to save.</li>
|
|
<li><b>Best pure-offload picks</b> (runs clean + fast + ≥good): {", ".join(f'<a href="pages/{esc(m["id"])}.html" style="color:var(--lime)">{esc(m["quant"])}</a>' for m in fast_clean[:3]) or "none yet"}.</li>
|
|
<li><b>Don't trust the score alone.</b> The 4-bit Qwen is the fastest local model (83 t/s) but crashes — speed is meaningless if the code doesn't run.</li>
|
|
</ul>
|
|
</div>"""
|
|
|
|
body = f"""
|
|
{head_html("LLM Benchmark Suite")}
|
|
<header class="hud-bar">
|
|
<div class="kicker">▚ LOCAL LLM BENCHMARK SUITE // LFU CACHE & ACID AUDIT</div>
|
|
<h1>// BENCHMARK_RESULTS <span style="color:var(--mag)">.json</span></h1>
|
|
<div class="subtitle">{n} models graded on a strict 5-pillar / 100-pt rubric · O(1) LFU + ACID transactions · M3 Max · LM Studio
|
|
· <span style="color:var(--cyan)">TOP: {top_name}</span></div>
|
|
</header>
|
|
<div class="toprow">
|
|
<div class="stats stats-squish">{stats_tiles}</div>
|
|
<div class="panel radar-panel">
|
|
<h2>▮ 5-PILLAR RADAR — TOP 3</h2>
|
|
<div class="chart-box" style="height:230px"><canvas id="radar"></canvas></div>
|
|
</div>
|
|
</div>
|
|
<div class="panel" style="margin-bottom:26px">
|
|
<h2>▮ SCORE vs THROUGHPUT (tok/sec)</h2>
|
|
<div class="chart-box" style="height:440px"><canvas id="bar"></canvas></div>
|
|
<div style="color:var(--dim);font-size:.72rem;margin-top:8px">Local models only — cloud baseline (DeepSeek) excluded from the speed axis. Bars flagged ⚠ have suspected GPU-offload / inference issues (not representative of the model).</div>
|
|
</div>
|
|
{findings_panel}
|
|
{offload_panel}
|
|
<div class="panel" style="margin-bottom:26px">
|
|
<h2>▮ LEADERBOARD</h2>
|
|
<div style="overflow-x:auto">
|
|
<table>
|
|
<thead><tr><th>#</th><th>Model</th><th>Quant</th><th>Format</th><th>Speed</th><th>Score</th><th>Verdict</th><th>Best For</th><th></th></tr></thead>
|
|
<tbody>{''.join(rows)}</tbody>
|
|
</table></div>
|
|
</div>
|
|
{family_panel}
|
|
<footer>Generated from <span class="mono">data/benchmark_history.json</span> · re-run <span class="mono">generate_dashboard.py</span> to refresh · cyberpunk-terminal UI</footer>
|
|
<script>
|
|
const NEON={{cyan:'#00ffc8',mag:'#ff2bd6',lime:'#b6ff3a',amber:'#ffb000',red:'#ff3b5c',blue:'#5b8cff'}};
|
|
new Chart(document.getElementById('bar'),{{
|
|
type:'bar',
|
|
data:{{labels:{bar_labels},
|
|
datasets:[
|
|
{{label:'Score /100',data:{bar_score},backgroundColor:'rgba(0,255,200,0.85)',borderColor:NEON.cyan,borderRadius:3,yAxisID:'y'}},
|
|
{{label:'tok/sec',data:{bar_speed},backgroundColor:'rgba(255,43,214,0.7)',borderColor:NEON.mag,borderRadius:3,yAxisID:'y1'}}
|
|
]}},
|
|
options:{{maintainAspectRatio:false,responsive:true,
|
|
plugins:{{legend:{{labels:{{color:'#d7e0e6',font:{{family:'Fira Code'}}}}}}}},
|
|
scales:{{
|
|
x:{{ticks:{{color:'#7a8590',font:{{family:'Fira Code',size:10}},maxRotation:50,minRotation:35,autoSkip:false}},grid:{{color:'rgba(255,255,255,0.05)'}}}},
|
|
y:{{position:'left',max:100,title:{{display:true,text:'Score',color:'#00ffc8'}},ticks:{{color:'#7a8590'}},grid:{{color:'rgba(255,255,255,0.05)'}}}},
|
|
y1:{{position:'right',title:{{display:true,text:'tok/sec',color:'#ff2bd6'}},grid:{{drawOnChartArea:false}},ticks:{{color:'#7a8590'}}}}
|
|
}}}}
|
|
}});
|
|
new Chart(document.getElementById('radar'),{{
|
|
type:'radar',
|
|
data:{{labels:{radar_labels},datasets:{radar_json}}},
|
|
options:{{maintainAspectRatio:false,responsive:true,
|
|
plugins:{{legend:{{labels:{{color:'#d7e0e6',font:{{family:'Fira Code',size:11}}}}}}}},
|
|
scales:{{r:{{min:0,max:20,
|
|
angleLines:{{color:'rgba(0,255,200,0.15)'}},
|
|
grid:{{color:'rgba(0,255,200,0.12)'}},
|
|
pointLabels:{{color:'#d7e0e6',font:{{family:'Fira Code',size:10}}}},
|
|
ticks:{{color:'#7a8590',backdropColor:'transparent',stepSize:5}}
|
|
}}}}
|
|
}}
|
|
}});
|
|
</script>
|
|
{FOOT}"""
|
|
return body
|
|
|
|
def render_detail(m, data):
|
|
col, chip = verdict_meta(m["verdict"])
|
|
# derive "what went right" from high pillars, "wrong" from low + critical_bugs
|
|
bd = m["breakdown"]
|
|
ranked = sorted(PILLARS, key=lambda p: -bd[p])
|
|
rights = [f"{PILLAR_LABELS[p]} ({bd[p]}/20)" for p in ranked if bd[p] >= 16]
|
|
wrongs_pillars = [f"{PILLAR_LABELS[p]} ({bd[p]}/20)" for p in ranked if bd[p] <= 13]
|
|
|
|
pillar_bars = ""
|
|
for p in PILLARS:
|
|
v = bd[p]
|
|
c = NEON_LIME if v >= 17 else (NEON_AMBER if v >= 13 else NEON_RED)
|
|
pillar_bars += f"""
|
|
<div class="prow">
|
|
<div class="plabel">{PILLAR_LABELS[p]}</div>
|
|
<div class="pbar"><i style="width:{v/20*100}%;background:{c};color:{c}"></i></div>
|
|
<div class="pval mono" style="color:{c}">{v}<span style="color:var(--dim)">/20</span></div>
|
|
</div>"""
|
|
|
|
bugs_html = "".join(f"<li>{esc(b)}</li>" for b in m.get("critical_bugs", []))
|
|
patch = m.get("patch_code", "")
|
|
metrics_block = ""
|
|
if m.get("tok_sec") is not None:
|
|
metrics_block = f"""
|
|
<div class="mini"><span class="mlbl">tok/sec</span><span class="mval mono" style="color:var(--mag)">{m['tok_sec']:.2f}</span></div>
|
|
<div class="mini"><span class="mlbl">tokens</span><span class="mval mono">{m.get('total_tokens') or '—'}</span></div>
|
|
<div class="mini"><span class="mlbl">TTFT</span><span class="mval mono">{m.get('ttft_sec'):.2f}s</span> </div>"""
|
|
else:
|
|
metrics_block = '<div class="mini" style="grid-column:1/-1"><span class="mlbl">runtime</span><span class="mval mono" style="color:var(--blue)">CLOUD — not measured</span></div>'
|
|
|
|
caveat_html = f'<div class="caveat-box">⚠ SPEED CAVEAT: {esc(m["speed_caveat"])}</div>' if m.get("speed_caveat") else ""
|
|
tests_html = '<span class="ok">PASS</span>' if m.get("tests_pass") else '<span class="bad">CRASH</span>'
|
|
cloud_tag = '<span class="cloud-tag">CLOUD BASELINE</span>' if m.get("format") == "cloud" else ""
|
|
|
|
rights_html = "".join(f"<li>{esc(r)}</li>" for r in rights) or "<li class='dim'>No pillar reached 16+ — no standout strengths.</li>"
|
|
wrongs_html = "".join(f"<li>{esc(w)}</li>" for w in wrongs_pillars) or "<li class='dim'>No pillar fell below 14 — solid across the board.</li>"
|
|
|
|
extra_css = """
|
|
.mini-grid{display:grid;grid-template-columns:repeat(3,1fr);gap:10px;margin:14px 0}
|
|
.mini{background:var(--panel2);border:1px solid rgba(255,255,255,0.06);border-radius:5px;padding:10px 12px}
|
|
.mlbl{display:block;color:var(--dim);font-size:.62rem;letter-spacing:.14em;text-transform:uppercase}
|
|
.mval{display:block;font-size:1.15rem;margin-top:3px}
|
|
.caveat-box{background:rgba(255,176,0,0.08);border:1px solid rgba(255,176,0,0.4);color:var(--amber);padding:10px 14px;border-radius:5px;font-size:.82rem;margin:12px 0}
|
|
.pillar-list{margin-top:8px}
|
|
.prow{display:grid;grid-template-columns:170px 1fr 56px;align-items:center;gap:12px;margin:9px 0}
|
|
.plabel{color:var(--ink);font-size:.84rem}
|
|
.pbar{height:9px;background:rgba(255,255,255,0.06);border-radius:2px;overflow:hidden}
|
|
.pbar > i{display:block;height:100%;border-radius:2px}
|
|
.pval{text-align:right;font-size:.92rem}
|
|
.ok{color:var(--lime);border:1px solid var(--lime);padding:1px 8px;border-radius:3px;font-size:.7rem}
|
|
.bad{color:var(--red);border:1px solid var(--red);padding:1px 8px;border-radius:3px;font-size:.7rem}
|
|
.section{background:var(--panel);border:1px solid rgba(255,255,255,0.07);border-radius:8px;padding:18px;margin-top:18px}
|
|
.section h2{font-size:.9rem;letter-spacing:.12em;text-transform:uppercase;color:var(--cyan);margin:0 0 12px;text-shadow:0 0 10px rgba(0,255,200,0.35)}
|
|
ul.clean{list-style:none;padding:0;margin:0}
|
|
ul.clean li{padding:7px 0 7px 18px;border-bottom:1px solid rgba(255,255,255,0.05);position:relative;font-size:.88rem}
|
|
ul.clean li::before{content:"▸";position:absolute;left:0;color:var(--cyan)}
|
|
ul.clean li.dim{color:var(--dim)}
|
|
ul.clean li.dim::before{color:var(--dim)}
|
|
.bugs li::before{content:"✗";color:var(--red)}
|
|
.rights li::before{content:"✓";color:var(--lime)}
|
|
pre.code{background:#06060a;border:1px solid rgba(0,255,200,0.2);border-radius:6px;padding:14px;overflow-x:auto;color:#cfe;font-size:.78rem;line-height:1.5;box-shadow:inset 0 0 30px rgba(0,0,0,0.6)}
|
|
pre.code::before{content:"// patch.py";display:block;color:var(--dim);font-size:.68rem;margin-bottom:8px}
|
|
.back{display:inline-block;margin-bottom:18px;font-size:.8rem}
|
|
.verdict-chip-lg{display:inline-block;padding:4px 14px;border-radius:3px;border:1px solid currentColor;font-size:.78rem;letter-spacing:.1em}
|
|
.best-box{background:linear-gradient(135deg,rgba(0,255,200,0.08),rgba(91,139,255,0.05));border:1px solid rgba(0,255,200,0.25);border-radius:8px;padding:16px 18px;font-size:.95rem;line-height:1.6}
|
|
@media(max-width:640px){.prow{grid-template-columns:1fr auto}.mini-grid{grid-template-columns:repeat(2,1fr)}}
|
|
"""
|
|
head = head_html(m["model_name"])
|
|
# inject extra css before </style>
|
|
head = head.replace("</style>", extra_css + "</style>")
|
|
|
|
return f"""
|
|
{head}
|
|
<a class="back" href="../dashboard.html">◂ BACK TO LEADERBOARD</a>
|
|
<header class="hud-bar">
|
|
<div class="kicker">▚ MODEL AUDIT // {esc(m['quant'])}</div>
|
|
<h1>// {esc(m['model_name'])} {cloud_tag}</h1>
|
|
<div style="margin-top:8px;display:flex;gap:14px;align-items:center;flex-wrap:wrap">
|
|
<span class="verdict-chip-lg" style="color:{col}">{chip} — {esc(m['verdict'])}</span>
|
|
<span class="mono" style="font-size:1.5rem;color:{col}">{m['total_score']}<span style="color:var(--dim);font-size:.8rem">/100</span></span>
|
|
<span>tests: {tests_html}</span>
|
|
</div>
|
|
</header>
|
|
{caveat_html}
|
|
<div class="mini-grid">{metrics_block}</div>
|
|
|
|
<div class="section">
|
|
<h2>▮ PILLAR BREAKDOWN</h2>
|
|
<div class="pillar-list">{pillar_bars}</div>
|
|
</div>
|
|
|
|
<div class="grid2">
|
|
<div class="section" style="margin-top:0">
|
|
<h2 style="color:var(--lime);text-shadow:0 0 10px rgba(182,255,58,0.4)">✓ WENT RIGHT</h2>
|
|
<ul class="clean rights">{rights_html}</ul>
|
|
</div>
|
|
<div class="section" style="margin-top:0">
|
|
<h2 style="color:var(--red);text-shadow:0 0 10px rgba(255,59,92,0.4)">✗ WENT WRONG</h2>
|
|
<ul class="clean wrongs">{wrongs_html}</ul>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>▮ CRITICAL BUGS</h2>
|
|
<ul class="clean bugs">{bugs_html or '<li class="dim">None flagged.</li>'}</ul>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>▮ RECOMMENDED USE</h2>
|
|
<div class="best-box">{esc(m['best_for'])}</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h2>▮ REFACTORED PATCH</h2>
|
|
<pre class="code">{esc(patch)}</pre>
|
|
</div>
|
|
|
|
<footer>Source: <span class="mono">{esc(m['filename'])}</span> · graded {esc(m.get('timestamp','')[:10])}</footer>
|
|
</div>
|
|
{FOOT}"""
|
|
|
|
def main():
|
|
with open(DATA) as f:
|
|
data = json.load(f)
|
|
os.makedirs(PAGES_DIR, exist_ok=True)
|
|
with open(OUT_DASH, "w") as f:
|
|
f.write(render_dashboard(data))
|
|
for m in data["models"]:
|
|
with open(os.path.join(PAGES_DIR, m["id"] + ".html"), "w") as f:
|
|
f.write(render_detail(m, data))
|
|
print(f"OK: dashboard.html + {len(data['models'])} pages in pages/")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|