Grade muse-glimmer-28b full 7-prompt battery — new benchmark leader

7 entries (30→37 total). Muse Glimmer 28B (GGUF) avg 80.7 — the strongest model in the benchmark, 6/7 prompts Minor Logic Flaws: lfu 76 | webhook 81 | automation 89 | rust 85 | data 88 | tts 58 | mcp 88 Standout results: - rust 85 (KAT 36, Qwen3-Coder 54) — real tokio channels (mpsc::channel, not hallucinated mpsc::bounded), two-tier CancellationToken, zero clippy lints; one-line E0507 compile fix. - automation 89 — first model to print a correct summary (98/2/0/100); atomic temp+fsync+rename checkpointing. - data 88 edges out Gemma-26B's 86; mcp 88 sets the bar on a new prompt. Only weak spot: tts 58 (backpressure raises instead of awaits, like Qwen3-Coder). Captured via the native /api/v1/chat fix (real tok/sec + TTFT). Slow deep-thinker: ~17-19 t/s, 5-9 min/prompt, ~5-9k tokens incl. reasoning. Also gitignore checkpoint.json (automation test runtime artifact). Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-10 13:13:54 -07:00
parent 3507d33006
commit 2f99dd1e35
10 changed files with 1769 additions and 7 deletions
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""
+Async batch processor with checkpointing, idempotency and graceful SIGINT.
+Stdlib only, Python 3.11+.
+"""
+import asyncio
+import json
+import os
+import signal
+import time
+import random
+
+CHECKPOINT_PATH = "checkpoint.json"
+
+class ProcessingError(Exception):
+    pass
+
+async def process(item: str) -> str:
+    """Flaky external call ~80% success, 10-80ms latency."""
+    await asyncio.sleep(random.uniform(0.01, 0.08))
+    if random.random() < 0.2:
+        raise ProcessingError(f"flaky failure for {item}")
+    return f"ok:{item}"
+
+def load_checkpoint():
+    if not os.path.exists(CHECKPOINT_PATH):
+        return set(), set()
+    try:
+        with open(CHECKPOINT_PATH, "r") as f:
+            data = json.load(f)
+        return set(data.get("completed", [])), set(data.get("failed", []))
+    except Exception:
+        return set(), set()
+
+def save_checkpoint(completed, failed):
+    data = {"completed": list(completed), "failed": list(failed)}
+    tmp = CHECKPOINT_PATH + ".tmp"
+    with open(tmp, "w") as f:
+        json.dump(data, f)
+        f.flush()
+        os.fsync(f.fileno())
+    os.replace(tmp, CHECKPOINT_PATH)   # atomic
+
+async def handle_item(item, completed, failed, lock, sem, stop_event):
+    async with sem:
+        if stop_event.is_set():
+            return
+        backoff = 0.1
+        for attempt in range(3):
+            try:
+                await process(item)
+                async with lock:
+                    if item not in completed and item not in failed:
+                        completed.add(item)
+                        save_checkpoint(completed, failed)
+                return
+            except ProcessingError:
+                if attempt == 2:
+                    async with lock:
+                        if item not in completed and item not in failed:
+                            failed.add(item)
+                            save_checkpoint(completed, failed)
+                    return
+                await asyncio.sleep(backoff)
+                backoff *= 2
+
+async def run_batch(items):
+    completed, failed = load_checkpoint()
+    initially_done = len(completed) + len(failed)
+    pending = [i for i in items if i not in completed and i not in failed]
+
+    stop_event = asyncio.Event()
+    lock = asyncio.Lock()
+    sem = asyncio.Semaphore(8)
+
+    loop = asyncio.get_running_loop()
+    def _sigint():
+        if not stop_event.is_set():
+            stop_event.set()
+    try:
+        loop.add_signal_handler(signal.SIGINT, _sigint)
+    except (NotImplementedError, ValueError):
+        pass
+
+    start = time.monotonic()
+    q = asyncio.Queue()
+    for it in pending:
+        q.put_nowait(it)
+
+    async def worker():
+        while True:
+            if stop_event.is_set() and q.empty():
+                break
+            try:
+                item = await asyncio.wait_for(q.get(), timeout=0.1)
+            except asyncio.TimeoutError:
+                if q.empty():
+                    break
+                continue
+            await handle_item(item, completed, failed, lock, sem, stop_event)
+            q.task_done()
+
+    workers = [asyncio.create_task(worker()) for _ in range(8)]
+    await asyncio.gather(*workers, return_exceptions=True)
+
+    elapsed_ms = int((time.monotonic() - start) * 1000)
+    summary = {
+        "succeeded": len(completed),
+        "failed": len(failed),
+        "skipped": initially_done,
+        "total": len(items),
+        "elapsed_ms": elapsed_ms,
+    }
+    print(json.dumps(summary))
+    return summary
+
+def main():
+    items = [f"job-{i}" for i in range(100)]
+    asyncio.run(run_batch(items))
+
+# ----------------------------------------------------------------------
+# Tests
+async def run_tests():
+    if os.path.exists(CHECKPOINT_PATH):
+        os.remove(CHECKPOINT_PATH)
+    random.seed(0)
+    items = [f"job-{i}" for i in range(50)]
+
+    calls = {}
+    active = 0
+    max_active = 0
+    orig = process
+
+    async def counting(item):
+        nonlocal active, max_active
+        active += 1
+        if active > max_active:
+            max_active = active
+        calls[item] = calls.get(item, 0) + 1
+        try:
+            return await orig(item)
+        finally:
+            active -= 1
+
+    globals()["process"] = counting
+
+    # watcher for checkpoint validity
+    errors = []
+    async def watcher():
+        while True:
+            try:
+                with open(CHECKPOINT_PATH, "r") as f:
+                    json.load(f)
+            except FileNotFoundError:
+                pass
+            except Exception as e:
+                errors.append(e)
+                break
+            await asyncio.sleep(0.001)
+    w = asyncio.create_task(watcher())
+
+    # first partial run -> simulate interruption
+    try:
+        await asyncio.wait_for(run_batch(items), timeout=0.05)
+    except asyncio.TimeoutError:
+        pass
+
+    # second run to finish
+    await run_batch(items)
+    w.cancel()
+    try:
+        await w
+    except asyncio.CancelledError:
+        pass
+
+    completed, failed = load_checkpoint()
+    assert len(completed) + len(failed) == len(items), "not all done"
+    assert not errors, f"checkpoint corrupt: {errors}"
+    assert max_active <= 8, f"concurrency exceeded: {max_active}"
+    # idempotency: no item processed again after it was completed/failed
+    # (calls count includes retries, but completed items must not be retried)
+    print("tests passed")
+
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) > 1 and sys.argv[1] == "test":
+        asyncio.run(run_tests())
+    else:
+        main()