Grade 3 more models + dashboard v2 layout (quant/format as first-class)
New graded (11 total now):
gemma4-26b-a4b-8bit-mlx 82 Minor Flaws (tied top local; delta-based tx freq)
qwen3.6-27b-8bit-mlx 78 Minor Flaws (clean; anom. slow generation flagged)
qwen3-coder-30b-6bit-mlx 50 Critical (asyncio.Lock used with sync with -> crash)
Dashboard redesign:
- Bar chart is now the full-width hero row (was cramped half-width)
- 4 stat tiles squished 2x2 beside the radar up top
- Quant + Format are dedicated columns in the leaderboard (MLX/GGUF/CLOUD chips)
- New 'Format & Quant Showdown' panel: groups same-family variants so
GGUF-vs-MLX and quant-depth comparisons are side by side
- Bar-chart axis labels now include the quant so duplicate model names
are distinguishable, with rotation for readability
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
+144
-20
@@ -4,7 +4,7 @@ Generator: builds dashboard.html + pages/<slug>..html from data/benchmark_histor
|
||||
Re-run after each grading batch to regenerate everything.
|
||||
Cyberpunk-terminal aesthetic. Pure stdlib + Chart.js via CDN.
|
||||
"""
|
||||
import json, html, os, sys
|
||||
import json, html, os, sys, re as _re
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DATA = os.path.join(HERE, "data", "benchmark_history.json")
|
||||
@@ -98,11 +98,28 @@ h1{font-size:1.5rem; margin:.3em 0 0; letter-spacing:.02em; text-shadow:0 0 16px
|
||||
.stat::after{content:"";position:absolute;left:0;top:0;bottom:0;width:3px;background:var(--cyan);box-shadow:0 0 12px var(--cyan)}
|
||||
.stat .lbl{color:var(--dim);font-size:.68rem;letter-spacing:.18em;text-transform:uppercase}
|
||||
.stat .num{font-size:1.6rem;margin-top:4px;color:var(--ink)}
|
||||
/* hero top row: squished stat tiles beside the radar */
|
||||
.toprow{display:grid;grid-template-columns:1.35fr 1fr;gap:18px;align-items:stretch;margin:18px 0 22px}
|
||||
.stats-squish{display:grid;grid-template-columns:repeat(2,1fr);gap:12px;align-content:start}
|
||||
.stats-squish .stat{padding:16px 18px}
|
||||
.stats-squish .stat .num{font-size:1.9rem}
|
||||
.radar-panel{display:flex;flex-direction:column}
|
||||
.radar-panel .chart-box{flex:1;min-height:220px}
|
||||
/* compact top-performers list */
|
||||
.toplist{display:flex;flex-direction:column;gap:6px}
|
||||
a.tr{display:grid;grid-template-columns:28px 1fr auto auto auto;align-items:center;gap:10px;padding:8px 12px;background:rgba(0,0,0,0.2);border:1px solid rgba(255,255,255,0.05);border-radius:5px;transition:all .16s;cursor:pointer;text-decoration:none;font-size:.84rem}
|
||||
a.tr:hover{border-color:rgba(0,255,200,0.4);background:rgba(0,255,200,0.05)}
|
||||
.tr-r{color:var(--dim)}
|
||||
.tr-n{color:var(--ink)}
|
||||
.tr-q{color:var(--dim);font-size:.74rem}
|
||||
.tr-s{color:var(--mag);font-size:.78rem}
|
||||
.tr-sc{font-size:1rem;font-weight:600;text-align:right;min-width:30px}
|
||||
@media(max-width:900px){.toprow{grid-template-columns:1fr}.grid2{grid-template-columns:1fr}.stats,.stats-squish{grid-template-columns:repeat(2,1fr)}}
|
||||
.grid2{display:grid;grid-template-columns:1.1fr .9fr;gap:18px;margin-bottom:26px}
|
||||
.panel{background:var(--panel);border:1px solid rgba(255,255,255,0.07);border-radius:8px;padding:18px}
|
||||
.panel h2{font-size:.95rem;letter-spacing:.12em;text-transform:uppercase;color:var(--cyan);margin:0 0 14px;text-shadow:0 0 10px rgba(0,255,200,0.35)}
|
||||
.chart-box{position:relative;height:340px}
|
||||
@media(max-width:900px){.grid2{grid-template-columns:1fr}.stats{grid-template-columns:repeat(2,1fr)}}
|
||||
@media(max-width:900px){.grid2{grid-template-columns:1fr}}
|
||||
/* leaderboard */
|
||||
table{width:100%;border-collapse:collapse;font-size:.86rem}
|
||||
thead th{text-align:left;color:var(--dim);font-size:.66rem;letter-spacing:.16em;text-transform:uppercase;border-bottom:1px solid rgba(0,255,200,0.2);padding:8px 10px}
|
||||
@@ -122,6 +139,23 @@ tbody tr:hover{background:rgba(0,255,200,0.05);box-shadow:inset 0 0 0 1px rgba(0
|
||||
tr:hover .score-bar > i{box-shadow:0 0 12px currentColor}
|
||||
.caveat{color:var(--amber);font-size:.72rem}
|
||||
.cloud-tag{color:var(--blue);font-size:.7rem;border:1px solid rgba(91,139,255,.4);padding:1px 6px;border-radius:3px;margin-left:6px}
|
||||
.quant-cell{color:var(--ink);font-size:.82rem;white-space:nowrap}
|
||||
.fmt-chip{display:inline-block;font-size:.66rem;letter-spacing:.08em;padding:2px 7px;border-radius:3px;border:1px solid currentColor;font-family:'Fira Code',monospace}
|
||||
.fmt-mlx{color:var(--cyan);background:rgba(0,255,200,0.08)}
|
||||
.fmt-gguf{color:var(--mag);background:rgba(255,43,214,0.08)}
|
||||
.fmt-cloud{color:var(--blue);background:rgba(91,139,255,0.08)}
|
||||
/* format/quant showdown cards */
|
||||
.fcard{background:var(--panel2);border:1px solid rgba(255,255,255,0.06);border-radius:6px;padding:12px 14px;margin-bottom:10px}
|
||||
.fcard-h{display:flex;justify-content:space-between;align-items:baseline;gap:10px;flex-wrap:wrap;margin-bottom:9px}
|
||||
.fcard-n{color:var(--ink);font-weight:600;font-family:'Fira Code',monospace;font-size:.92rem}
|
||||
.fcard-meta{color:var(--dim);font-size:.72rem}
|
||||
.fcard-v{display:flex;flex-wrap:wrap;gap:8px}
|
||||
a.fv{display:grid;grid-template-columns:auto auto auto auto;align-items:center;gap:10px;padding:7px 11px;background:rgba(0,0,0,0.25);border:1px solid rgba(0,255,200,0.15);border-radius:5px;transition:all .18s;cursor:pointer;text-decoration:none}
|
||||
a.fv:hover{border-color:var(--cyan);box-shadow:0 0 12px rgba(0,255,200,0.3);background:rgba(0,255,200,0.06)}
|
||||
.fv-q{color:var(--ink);font-size:.82rem;min-width:64px}
|
||||
.fv-f{font-size:.7rem;letter-spacing:.08em;font-family:'Fira Code',monospace}
|
||||
.fv-s{color:var(--dim);font-size:.78rem}
|
||||
.fv-sc{font-size:1rem;font-weight:600;min-width:28px;text-align:right}
|
||||
footer{color:var(--dim);font-size:.74rem;margin-top:40px;border-top:1px solid rgba(255,255,255,0.06);padding-top:14px;text-align:center}
|
||||
@media (prefers-reduced-motion: reduce){*{animation:none!important;transition:none!important}}
|
||||
"""
|
||||
@@ -165,21 +199,38 @@ def render_dashboard(data):
|
||||
caveat = ""
|
||||
if m.get("speed_caveat"):
|
||||
caveat = '<div class="caveat">⚠ speed suspect</div>'
|
||||
cloud = '<span class="cloud-tag">CLOUD</span>' if m.get("format") == "cloud" else ""
|
||||
fmt = (m.get("format") or "").lower()
|
||||
if fmt == "cloud":
|
||||
fmt_chip = '<span class="fmt-chip fmt-cloud">CLOUD</span>'
|
||||
elif fmt == "gguf":
|
||||
fmt_chip = '<span class="fmt-chip fmt-gguf">GGUF</span>'
|
||||
elif fmt == "mlx":
|
||||
fmt_chip = '<span class="fmt-chip fmt-mlx">MLX</span>'
|
||||
else:
|
||||
fmt_chip = f'<span class="fmt-chip">{esc(m.get("format") or "—")}</span>'
|
||||
bar_color = col
|
||||
rows.append(f"""<tr>
|
||||
<td class="rank {'top' if i<=3 else ''}">#{i}</td>
|
||||
<td><div class="model-name">{esc(m['model_name'])}{cloud}</div><div class="quant">{esc(m['quant'])}</div>{caveat}</td>
|
||||
<td><div class="model-name">{esc(m['model_name'])}</div>{caveat}</td>
|
||||
<td><span class="quant-cell mono">{esc(m['quant'])}</span></td>
|
||||
<td>{fmt_chip}</td>
|
||||
<td class="mono">{speed_str(m)} <span style="color:var(--dim);font-size:.72rem">t/s</span></td>
|
||||
<td><div class="bar-cell"><span class="mono" style="width:34px;color:{col}">{m['total_score']}</span>
|
||||
<span class="score-bar"><i style="width:{m['total_score']}%;background:{bar_color};color:{bar_color}"></i></span></div></td>
|
||||
<td><span class="badge verdict-chip" style="color:{col}">{chip}</span></td>
|
||||
<td style="color:var(--dim);font-size:.8rem">{esc(m['best_for'])[:70]}…</td>
|
||||
<td style="color:var(--dim);font-size:.8rem">{esc(m['best_for'])[:60]}…</td>
|
||||
<td><a class="btn" href="pages/{esc(m['id'])}.html">DECODE ▸</a></td>
|
||||
</tr>""")
|
||||
|
||||
# JSON for charts
|
||||
bar_labels = json.dumps([m["model_name"].split("(")[0].strip()[:18] for m in local_sorted])
|
||||
def short_label(m):
|
||||
# family + quant so duplicates (same model, different quant) are distinguishable
|
||||
fam = m["model_name"].split("(")[0].strip()
|
||||
fam = _re.sub(r"(?i)\s+(uncensored|heretic|aggressive|hauhaucs).*$", "", fam)
|
||||
q = (m.get("quant") or "").strip()
|
||||
name = f"{fam} · {q}" if q else fam
|
||||
return name[:28]
|
||||
bar_labels = json.dumps([short_label(m) for m in local_sorted])
|
||||
bar_speed = json.dumps([m["tok_sec"] for m in local_sorted])
|
||||
bar_score = json.dumps([m["total_score"] for m in local_sorted])
|
||||
|
||||
@@ -195,14 +246,87 @@ def render_dashboard(data):
|
||||
})
|
||||
radar_json = json.dumps(radar_sets)
|
||||
|
||||
stats = f"""
|
||||
stats_tiles = f"""
|
||||
<div class="stat"><div class="lbl">Models Tested</div><div class="num mono">{n}</div></div>
|
||||
<div class="stat"><div class="lbl">Top Score</div><div class="num mono" style="color:var(--lime)">{top['total_score'] if top else '—'}</div></div>
|
||||
<div class="stat"><div class="lbl">Average</div><div class="num mono" style="color:var(--amber)">{avg:.1f}</div></div>
|
||||
<div class="stat"><div class="lbl">Prod-Ready</div><div class="num mono" style="color:var(--cyan)">{prod}/{n}</div></div>
|
||||
"""
|
||||
stats = f'<div class="stats">{stats_tiles}</div>'
|
||||
top_name = esc(top["model_name"]) if top else "—"
|
||||
|
||||
# compact top-5 list for the right-of-radar panel
|
||||
trows = []
|
||||
for i, m in enumerate(models[:5], 1):
|
||||
col, _ = verdict_meta(m["verdict"])
|
||||
speed = f"{m['tok_sec']:.1f}" if m.get("tok_sec") is not None else "—"
|
||||
trows.append(
|
||||
f'<a class="tr" href="pages/{esc(m["id"])}.html">'
|
||||
f'<span class="tr-r mono">#{i}</span>'
|
||||
f'<span class="tr-n">{esc(m["model_name"][:20])}</span>'
|
||||
f'<span class="tr-q mono">{esc(m["quant"])}</span>'
|
||||
f'<span class="tr-s mono">{speed}</span>'
|
||||
f'<span class="tr-sc mono" style="color:{col}">{m["total_score"]}</span></a>'
|
||||
)
|
||||
toplist = ''.join(trows)
|
||||
|
||||
# ---- Format / Quant showdown: group local models by base family ----
|
||||
def family_of(m):
|
||||
# strip parentheticals, uncensored/merge tags, quant words, format, params
|
||||
name = m["model_name"].split("(")[0].strip()
|
||||
name = _re.sub(r"(?i)\b(uncensored|heretic|aggressive|hauhaucs|coder|composer|fable5|v\d+\.\d+|merge)\b", "", name)
|
||||
name = _re.sub(r"\b\d+(\.\d+)?[bB](-[aA]\d+[bB])?\b", "", name) # sizes: 35B, 26B-A4B
|
||||
name = _re.sub(r"\s+", " ", name).strip(" -")
|
||||
# collapse known families
|
||||
for fam in ["Qwen 3.6", "Qwen3", "Gemma 4", "KAT-Coder", "DeepSeek"]:
|
||||
if _re.sub(r"\s+", "", name).lower().startswith(_re.sub(r"\s+", "", fam).lower()):
|
||||
return fam
|
||||
return name or m["model_name"]
|
||||
|
||||
families = {}
|
||||
for m in models:
|
||||
if m.get("format") == "cloud":
|
||||
continue
|
||||
f = family_of(m)
|
||||
families.setdefault(f, []).append(m)
|
||||
# only show families with >=2 variants (the interesting comparisons)
|
||||
multi = {f: ms for f, ms in families.items() if len(ms) >= 2}
|
||||
|
||||
if multi:
|
||||
fcards = []
|
||||
for fam, ms in sorted(multi.items(), key=lambda kv: -max(x["total_score"] for x in kv[1])):
|
||||
ms_sorted = sorted(ms, key=lambda x: -x["total_score"])
|
||||
best = ms_sorted[0]
|
||||
spread = max(x["total_score"] for x in ms) - min(x["total_score"] for x in ms)
|
||||
variants = []
|
||||
for x in ms_sorted:
|
||||
fmt = (x.get("format") or "").upper()
|
||||
fcol = "var(--cyan)" if x.get("format") == "mlx" else ("var(--mag)" if x.get("format") == "gguf" else "var(--dim)")
|
||||
col, _ = verdict_meta(x["verdict"])
|
||||
speed = f"{x['tok_sec']:.0f} t/s" if x.get("tok_sec") is not None else "—"
|
||||
variants.append(
|
||||
f'<a class="fv" href="pages/{esc(x["id"])}.html">'
|
||||
f'<span class="fv-q mono">{esc(x["quant"])}</span>'
|
||||
f'<span class="fv-f" style="color:{fcol}">{fmt}</span>'
|
||||
f'<span class="fv-s mono">{speed}</span>'
|
||||
f'<span class="fv-sc mono" style="color:{col}">{x["total_score"]}</span>'
|
||||
f'</a>'
|
||||
)
|
||||
fcards.append(f"""
|
||||
<div class="fcard">
|
||||
<div class="fcard-h"><span class="fcard-n">{esc(fam)}</span>
|
||||
<span class="fcard-meta">{len(ms)} variants · score spread <b class="mono">{spread}</b> · best <b class="mono" style="color:var(--lime)">{best["total_score"]}</b> ({esc(best["quant"])})</span></div>
|
||||
<div class="fcard-v">{''.join(variants)}</div>
|
||||
</div>""")
|
||||
family_panel = f"""
|
||||
<div class="panel" style="margin-bottom:26px">
|
||||
<h2>▮ FORMAT & QUANT SHOWDOWN — same family, different quants/formats</h2>
|
||||
<div style="color:var(--dim);font-size:.76rem;margin-bottom:12px">Families with 2+ variants. Click a row for the full audit. Compare how quant depth and MLX-vs-GGUF change the score.</div>
|
||||
{''.join(fcards)}
|
||||
</div>"""
|
||||
else:
|
||||
family_panel = ""
|
||||
|
||||
body = f"""
|
||||
{head_html("LLM Benchmark Suite")}
|
||||
<header class="hud-bar">
|
||||
@@ -211,27 +335,27 @@ def render_dashboard(data):
|
||||
<div class="subtitle">{n} models graded on a strict 5-pillar / 100-pt rubric · O(1) LFU + ACID transactions · M3 Max · LM Studio
|
||||
· <span style="color:var(--cyan)">TOP: {top_name}</span></div>
|
||||
</header>
|
||||
{stats}
|
||||
<div class="grid2">
|
||||
<div class="panel">
|
||||
<h2>▮ Score vs Throughput (tok/sec)</h2>
|
||||
<div class="chart-box"><canvas id="bar"></canvas></div>
|
||||
<div style="color:var(--dim);font-size:.72rem;margin-top:8px">Local models only — cloud baseline (DeepSeek) excluded from speed axis. Gemma 4 bars flagged ⚠ (GPU-offload suspect).</div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h2>▮ 5-Pillar Radar — Top 3</h2>
|
||||
<div class="chart-box"><canvas id="radar"></canvas></div>
|
||||
<div style="color:var(--dim);font-size:.72rem;margin-top:8px">Each pillar scored 0–20. Outer = stronger.</div>
|
||||
<div class="toprow">
|
||||
<div class="stats stats-squish">{stats_tiles}</div>
|
||||
<div class="panel radar-panel">
|
||||
<h2>▮ 5-PILLAR RADAR — TOP 3</h2>
|
||||
<div class="chart-box" style="height:230px"><canvas id="radar"></canvas></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel" style="margin-bottom:26px">
|
||||
<h2>▮ SCORE vs THROUGHPUT (tok/sec)</h2>
|
||||
<div class="chart-box" style="height:440px"><canvas id="bar"></canvas></div>
|
||||
<div style="color:var(--dim);font-size:.72rem;margin-top:8px">Local models only — cloud baseline (DeepSeek) excluded from the speed axis. Bars flagged ⚠ have suspected GPU-offload / inference issues (not representative of the model).</div>
|
||||
</div>
|
||||
<div class="panel" style="margin-bottom:26px">
|
||||
<h2>▮ LEADERBOARD</h2>
|
||||
<div style="overflow-x:auto">
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>Model</th><th>Speed</th><th>Score</th><th>Verdict</th><th>Best For</th><th></th></tr></thead>
|
||||
<thead><tr><th>#</th><th>Model</th><th>Quant</th><th>Format</th><th>Speed</th><th>Score</th><th>Verdict</th><th>Best For</th><th></th></tr></thead>
|
||||
<tbody>{''.join(rows)}</tbody>
|
||||
</table></div>
|
||||
</div>
|
||||
{family_panel}
|
||||
<footer>Generated from <span class="mono">data/benchmark_history.json</span> · re-run <span class="mono">generate_dashboard.py</span> to refresh · cyberpunk-terminal UI</footer>
|
||||
<script>
|
||||
const NEON={{cyan:'#00ffc8',mag:'#ff2bd6',lime:'#b6ff3a',amber:'#ffb000',red:'#ff3b5c',blue:'#5b8cff'}};
|
||||
@@ -245,7 +369,7 @@ new Chart(document.getElementById('bar'),{{
|
||||
options:{{maintainAspectRatio:false,responsive:true,
|
||||
plugins:{{legend:{{labels:{{color:'#d7e0e6',font:{{family:'Fira Code'}}}}}}}},
|
||||
scales:{{
|
||||
x:{{ticks:{{color:'#7a8590',font:{{family:'Fira Code',size:10}}}},grid:{{color:'rgba(255,255,255,0.05)'}}}},
|
||||
x:{{ticks:{{color:'#7a8590',font:{{family:'Fira Code',size:10}},maxRotation:50,minRotation:35,autoSkip:false}},grid:{{color:'rgba(255,255,255,0.05)'}}}},
|
||||
y:{{position:'left',max:100,title:{{display:true,text:'Score',color:'#00ffc8'}},ticks:{{color:'#7a8590'}},grid:{{color:'rgba(255,255,255,0.05)'}}}},
|
||||
y1:{{position:'right',title:{{display:true,text:'tok/sec',color:'#ff2bd6'}},grid:{{drawOnChartArea:false}},ticks:{{color:'#7a8590'}}}}
|
||||
}}}}
|
||||
|
||||
Reference in New Issue
Block a user