From de748f48548967ad17c110e07192dbac194901c1 Mon Sep 17 00:00:00 2001 From: aygea Date: Tue, 28 Jul 2026 17:12:56 -0700 Subject: [PATCH] Add Key Findings panel + Aygea Test prompt battery Findings panel: live stats from the data (5/10 run tests, quant dominates quality, concurrency is the killer pillar, 2/11 __slots__, 4/11 monotonic). Aygea Test (prompts/aygea_test_battery.md): 5-prompt battery derived from ~/dev + jirachi project shapes. Notes prompt_id schema for multi-prompt. Co-Authored-By: Claude --- generate_dashboard.py | 69 +++++++++++++++++++++ prompts/aygea_test_battery.md | 113 ++++++++++++++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 prompts/aygea_test_battery.md diff --git a/generate_dashboard.py b/generate_dashboard.py index 68f1d84..8d8bdfa 100644 --- a/generate_dashboard.py +++ b/generate_dashboard.py @@ -156,6 +156,19 @@ a.fv:hover{border-color:var(--cyan);box-shadow:0 0 12px rgba(0,255,200,0.3);back .fv-f{font-size:.7rem;letter-spacing:.08em;font-family:'Fira Code',monospace} .fv-s{color:var(--dim);font-size:.78rem} .fv-sc{font-size:1rem;font-weight:600;min-width:28px;text-align:right} +/* key findings */ +.findings{display:grid;grid-template-columns:repeat(4,1fr);gap:12px;margin-bottom:16px} +.fnd{background:var(--panel2);border:1px solid rgba(255,255,255,0.06);border-radius:6px;padding:14px 16px;position:relative;overflow:hidden} +.fnd::after{content:"";position:absolute;left:0;top:0;bottom:0;width:3px;background:var(--mag);box-shadow:0 0 12px var(--mag)} +.fnd-n{font-size:1.7rem;color:var(--cyan);text-shadow:0 0 10px rgba(0,255,200,0.3)} +.fnd-l{color:var(--ink);font-size:.72rem;letter-spacing:.12em;text-transform:uppercase;margin-top:4px;font-family:'Fira Code',monospace} +.fnd-s{color:var(--dim);font-size:.72rem;margin-top:6px;line-height:1.45} +ul.findings-notes{list-style:none;padding:0;margin:0} +ul.findings-notes li{padding:8px 0 8px 20px;border-bottom:1px solid rgba(255,255,255,0.05);position:relative;font-size:.84rem;color:var(--ink);line-height:1.5} +ul.findings-notes li:last-child{border-bottom:none} +ul.findings-notes li::before{content:"▸";position:absolute;left:0;color:var(--mag)} +ul.findings-notes code{background:rgba(0,255,200,0.1);color:var(--cyan);padding:1px 5px;border-radius:3px;font-family:'Fira Code',monospace;font-size:.8rem} +@media(max-width:900px){.findings{grid-template-columns:repeat(2,1fr)}} footer{color:var(--dim);font-size:.74rem;margin-top:40px;border-top:1px solid rgba(255,255,255,0.06);padding-top:14px;text-align:center} @media (prefers-reduced-motion: reduce){*{animation:none!important;transition:none!important}} """ @@ -327,6 +340,61 @@ def render_dashboard(data): else: family_panel = "" + # ---- Key findings: real stats computed from the data ---- + import ast as _ast, os as _os + def _scan_file(m): + fn = m.get("filename", "") + path = _os.path.join(HERE, fn) if not _os.path.isabs(fn) else fn + try: + txt = open(path).read() + parses = True + try: _ast.parse(txt) + except Exception: parses = False + return { + "slots": txt.count("__slots__") > 0, + "monotonic": txt.count("monotonic") > 0, + "linear": any(p in txt for p in ("sorted(", ".sort(", "heapq", "min(", "max(")), + "parses": parses, + } + except Exception: + return {"slots": False, "monotonic": False, "linear": False, "parses": None} + + scans = {m["id"]: _scan_file(m) for m in local} + n_run = sum(1 for m in local if m.get("tests_pass")) + n_crit = sum(1 for m in local if m["verdict"] == "Critical Bugs") + n_slots = sum(1 for m in local if scans[m["id"]]["slots"]) + n_mono = sum(1 for m in local if scans[m["id"]]["monotonic"]) + pillar_avg = {p: round(sum(m["breakdown"][p] for m in local)/len(local), 1) for p in PILLARS} + weakest = min(PILLARS, key=lambda p: pillar_avg[p]) + + def _fcard(num, label, sub): + return (f'
{num}
' + f'
{label}
' + f'
{sub}
') + + findings_tiles = "".join([ + _fcard(f"{n_run}/{len(local)}", "RUN THEIR OWN TESTS", + "Half of local models crash before completing — runnability is the real filter."), + _fcard(f"{n_crit}/{len(local)}", "CRITICAL BUGS", + "Cache corruption, evict-crashes, or fatal KeyErrors — not safe for systems work."), + _fcard(f"{n_slots}/{len(local)}", "DECLARE __slots__", + "Rubric explicitly required it for memory efficiency; nearly all models miss it."), + _fcard(f"{n_mono}/{len(local)}", "USE time.monotonic()", + "The rest use the system clock — NTP jumps corrupt TTL eviction."), + ]) + findings_panel = f""" +
+

▮ KEY FINDINGS — patterns across {len(local)} local models

+
{findings_tiles}
+
    +
  • Quant depth dominates quality. Same model, different quant: Qwen 3.6 35B-A3B scores 82 at 6-bit but 57 at 4-bit — a ~25-point drop. Aggressive quants cost real logic on systems code.
    • +
  • Speed ≠ quality. The 4-bit Qwen is the fastest (83 t/s) yet scores 57; the 6-bit is slower (69 t/s) but scores 82. Pick quants for correctness first, throughput second.
    • +
  • Concurrency is the killer pillar (avg {pillar_avg['concurrency']}/20). Local models most often break on async correctness — lock type mismatches, races, and non-reentrant-lock deadlocks.
    • +
  • The weakest pillar overall is {PILLAR_LABELS[weakest]} (avg {pillar_avg[weakest]}/20). Test suites that ship with crashing code validate nothing.
    • +
  • Only the cloud baseline (DeepSeek, 91) cleared Production-Ready. Best local scores cap at 82 — strong scaffolding, but every submission needs a human pass on __slots__, monotonic clocks, and lock granularity.
    • +
+
""" + body = f""" {head_html("LLM Benchmark Suite")}
@@ -347,6 +415,7 @@ def render_dashboard(data):
Local models only — cloud baseline (DeepSeek) excluded from the speed axis. Bars flagged ⚠ have suspected GPU-offload / inference issues (not representative of the model).
+{findings_panel}

▮ LEADERBOARD

diff --git a/prompts/aygea_test_battery.md b/prompts/aygea_test_battery.md new file mode 100644 index 0000000..5127db6 --- /dev/null +++ b/prompts/aygea_test_battery.md @@ -0,0 +1,113 @@ +# 🎯 The Aygea Test — a multi-prompt battery drawn from your real projects + +## Why this exists + +The LFU-cache exam (`prompts/lfu_cache_prompt.txt`) is an excellent probe for +**systems + async correctness** — O(1) data structures, locks, ACID, TTL. But it's +one narrow axis. It tells you nothing about whether a model can do the work you +*actually* do every day. + +So I surveyed `~/dev` (mewtwo) + `jirachi` and found your real workload clusters into a +handful of archetypes. This battery mirrors them. Run each model against all five and +you get a profile — "great at MCP, weak at real-time" — instead of a single score. + +## What your projects actually are (the evidence) + +From scanning `~/dev` + `jirachi`: + +| Archetype | Examples you have | What the code does | +|---|---|---| +| **MCP servers** (9!) | joplin-mcp, obsidian-mcp, mySupabaseMCP, project-rag, yt-video-summarizer-mcp, vault-mcp | tool defs, Zod/Pydantic schema validation, stdio/SSE/StreamableHTTP transport, input parsing | +| **TTS / audio pipelines** | Chatterbox-TTS-Server, aygea-tts-app, vr-to-tts, ffxiv-tts, echokraut-bridge | external HTTP APIs, streaming responses, queueing, device/audio edge cases | +| **Streaming / chat bridges** | aygeas-multistream, twitch-vod-to-youtube, twitch-discord-notifier, aygeas-chat-overlay | webhooks, OAuth, rate limits, real-time event handling | +| **Data / API services** | project-rag, mySupabaseMCP, PostgresHA, aygeas-dashboard | SQL, connection pooling, pagination, REST/JSON | +| **Automation / glue** | fix-tokens, notesCleanup, twitch-discord-notifier | cron-style tasks, idempotency, retries, partial-failure recovery | + +**Stack signal:** TypeScript/Node is dominant, Python second, async/await is in ~half of +all files, try/catch is everywhere, Zod (`z.string`/`z.object`) and Pydantic (`BaseModel`) +are your validation layer, Docker/compose is standard. + +The LFU exam tests *none* of that. These five prompts do. + +--- + +## The 5 prompts + +Each is scoped to ~1 file, runnable, and gradable on the same 5-pillar / 100-pt rubric. +Save each as `prompts/.txt` and feed it to the model. + +### 1. `mcp-server.txt` — Build an MCP tool server +**Probe:** tool/schema correctness, transport, error handling. Your most common project. +> Write a single-file MCP server (TypeScript `@modelcontextprotocol/sdk` OR Python `mcp`) that +> exposes 3 tools against a JSONPlaceholder REST API: +> `get_user(id)`, `list_posts_by_user(user_id, limit)`, `search_posts(query)`. +> Each tool must: validate inputs with a schema (Zod or Pydantic), return typed results, +> handle HTTP errors + timeouts gracefully (no silent failures), and not crash on bad input. +> Run over stdio transport. Include 3 runnable tests (happy path, bad-id 404, malformed input). +> No external state — pure stdlib + fetch/httpx + the MCP SDK. + +### 2. `tts-pipeline.txt` — Audio job queue with backpressure +**Probe:** async queues, streaming, external-API resilience. Your TTS/audio shape. +> Single-file async service (Python asyncio or Node) that accepts TTS "jobs" via an async +> `submit(text, voice)` function, queues them, and processes them through a mock synthesizer +> (`await mock_synthesize(text) -> bytes`, variable 50-300ms latency). Requirements: +> bounded concurrency (max 4 in-flight), backpressure (reject when queue > 100), per-job +> retry-on-failure (max 3, exponential backoff), a `drain()` that awaits all queued jobs, +> and clean cancellation. Emit job lifecycle events to a callback. Include a 50-job stress +> test proving the concurrency cap holds and no jobs are dropped on cancel. + +### 3. `webhook-bridge.txt` — Twitch/Discord event bridge +**Probe:** webhook signature verification, rate limiting, idempotency. Your bridge shape. +> Single-file HTTP service that receives Twitch EventSub webhooks (POST /webhook) and +> forwards chat events to Discord via a mock webhook. Requirements: +> HMAC-SHA256 signature verification of every request (reject 401 on mismatch), an +> in-memory idempotency store keyed by the event id (skip replays within 5 min), a +> token-bucket rate limiter capping Discord forwards to 5/sec, and graceful handling of +> Discord 429 (read Retry-After, back off). No framework deps beyond a stdlib http server. +> Include tests for: valid vs tampered signature, replayed event skipped, rate-limit trigger. + +### 4. `data-service.txt` — Paginated query service with a connection pool +**Probe:** SQL, pooling, pagination, resource cleanup. Your data-service shape. +> Single-file service wrapping a (mock) Postgres pool exposing: +> `get_users(page, page_size)`, `get_user_with_posts(user_id)`, and a bulk +> `relabel_users(id_label_pairs)`. Requirements: a real pooled-connection pattern (checkout / +> return / leak-proof), parameterized queries (no string-interpolated SQL), correct +> offset/limit pagination with a total-count, a transactional bulk update that rolls back on +> any failure, and connection-checkout timeouts. Mock the DB; include tests proving: no +> connection leak across 100 calls, pagination math, rollback on partial failure. + +### 5. `automation-glue.txt` — Idempotent batch job with retries +**Probe:** idempotency, partial-failure recovery, observability. Your automation shape. +> Single-file async batch processor that reads a list of "items", calls a flaky external +> `process(item)` (fails ~20% randomly), and must: be idempotent (re-running resumes from a +> checkpoint file, never reprocessing done items), retry failures with backoff (max 3), +> write a progress checkpoint after each item, log a structured JSON summary at the end +> (succeeded/failed/skipped counts + durations), and exit cleanly on SIGINT (flushing +> checkpoint). Include a test that kills mid-run and proves resume skips completed items. + +--- + +## How to score (reuse the existing rubric) + +Each prompt grades on the same 5 pillars (0–20 each, 100 total): +1. **Complexity / correctness** — does it actually work, edge cases handled? +2. **Async / concurrency** — locks, backpressure, cancellation, no races +3. **Error handling** — no silent failures, retries, timeouts, graceful degradation +4. **Resource / state safety** — connection leaks, idempotency, checkpoint integrity +5. **Test integrity** — real assertions vs always-pass; do the tests catch the bugs above? + +> Note: pillars 3–5 map cleanly to your repeated patterns (try/catch everywhere, +> retries, validation, "no silent failures" — your own recurring concern). + +## How the dashboard should evolve for this + +The current JSON schema assumes one prompt (`exam_prompt`). To support a battery: +- Add `prompt_id` to each model entry (e.g. `"lfu"`, `"mcp"`, `"tts"`). +- The leaderboard gets a **prompt filter** (default: show a model's average across all + prompts it has run). +- A new **per-model radar across prompts** shows the profile ("strong at MCP, weak at async + pipelines") — the real value of a battery over a single exam. + +When you're ready to run these, tell me which prompt + model and I'll wire up grading the +same way as the LFU set. The generator will need the `prompt_id` field + the filter; I can +do that in one pass once you have ≥1 result from a second prompt.