Grade muse-glimmer-28b full 7-prompt battery — new benchmark leader
7 entries (30→37 total). Muse Glimmer 28B (GGUF) avg 80.7 — the strongest model in the benchmark, 6/7 prompts Minor Logic Flaws: lfu 76 | webhook 81 | automation 89 | rust 85 | data 88 | tts 58 | mcp 88 Standout results: - rust 85 (KAT 36, Qwen3-Coder 54) — real tokio channels (mpsc::channel, not hallucinated mpsc::bounded), two-tier CancellationToken, zero clippy lints; one-line E0507 compile fix. - automation 89 — first model to print a correct summary (98/2/0/100); atomic temp+fsync+rename checkpointing. - data 88 edges out Gemma-26B's 86; mcp 88 sets the bar on a new prompt. Only weak spot: tts 58 (backpressure raises instead of awaits, like Qwen3-Coder). Captured via the native /api/v1/chat fix (real tok/sec + TTFT). Slow deep-thinker: ~17-19 t/s, 5-9 min/prompt, ~5-9k tokens incl. reasoning. Also gitignore checkpoint.json (automation test runtime artifact). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Aygea
@@ -0,0 +1,216 @@
|
||||
# single file demo - Python 3.11+ asyncio stdlib only
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import List, Tuple, Dict, Any
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Mock in-memory DB
|
||||
# ----------------------------------------------------------------------
|
||||
class MockDB:
|
||||
users: Dict[int, Dict[str, Any]] = {}
|
||||
posts: Dict[int, List[Dict[str, Any]]] = {}
|
||||
|
||||
def seed_db():
|
||||
MockDB.users.clear()
|
||||
MockDB.posts.clear()
|
||||
for uid in range(1, 26):
|
||||
MockDB.users[uid] = {"id": uid, "name": f"user{uid}", "label": f"label{uid % 5}"}
|
||||
MockDB.posts[uid] = [
|
||||
{"id": pid, "user_id": uid, "title": f"post{pid} for user{uid}"}
|
||||
for pid in range(1, 6)
|
||||
]
|
||||
|
||||
seed_db()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Mock pool / connection
|
||||
# ----------------------------------------------------------------------
|
||||
class MockPool:
|
||||
def __init__(self, max_size: int = 5, acquire_timeout: float = 2.0):
|
||||
self.max_size = max_size
|
||||
self.acquire_timeout = acquire_timeout
|
||||
self._sem = asyncio.Semaphore(max_size)
|
||||
self._checked_out = 0
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def acquire(self) -> "MockConnection":
|
||||
try:
|
||||
await asyncio.wait_for(self._sem.acquire(), timeout=self.acquire_timeout)
|
||||
except asyncio.TimeoutError:
|
||||
raise TimeoutError(f"Pool exhausted after {self.acquire_timeout}s")
|
||||
async with self._lock:
|
||||
self._checked_out += 1
|
||||
return MockConnection(self)
|
||||
|
||||
async def _release(self):
|
||||
async with self._lock:
|
||||
self._checked_out -= 1
|
||||
self._sem.release()
|
||||
|
||||
@property
|
||||
def checked_out(self) -> int:
|
||||
return self._checked_out
|
||||
|
||||
|
||||
class MockConnection:
|
||||
def __init__(self, pool: MockPool):
|
||||
self.pool = pool
|
||||
|
||||
async def fetch(self, query: str, params: List[Any]) -> List[Dict[str, Any]]:
|
||||
await asyncio.sleep(0) # simulate IO
|
||||
if query.startswith("SELECT COUNT"):
|
||||
return [{"count": len(MockDB.users)}]
|
||||
if "FROM users WHERE id =" in query:
|
||||
uid = params[0]
|
||||
user = MockDB.users.get(uid)
|
||||
return [user] if user else []
|
||||
if "FROM users ORDER BY id LIMIT" in query:
|
||||
limit, offset = params
|
||||
sorted_users = [MockDB.users[i] for i in sorted(MockDB.users)]
|
||||
return sorted_users[offset: offset + limit]
|
||||
if "FROM posts WHERE user_id =" in query:
|
||||
uid = params[0]
|
||||
return MockDB.posts.get(uid, [])
|
||||
raise ValueError(f"Unknown fetch query: {query}")
|
||||
|
||||
async def execute(self, query: str, params: List[Any]) -> int:
|
||||
await asyncio.sleep(0)
|
||||
if query.startswith("UPDATE users SET label"):
|
||||
label, uid = params
|
||||
if uid not in MockDB.users:
|
||||
raise KeyError(f"User {uid} not found")
|
||||
MockDB.users[uid]["label"] = label
|
||||
return 1
|
||||
raise ValueError(f"Unknown execute query: {query}")
|
||||
|
||||
async def release(self):
|
||||
await self.pool._release()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_connection(pool: MockPool):
|
||||
conn = await pool.acquire()
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
await conn.release()
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Service API
|
||||
# ----------------------------------------------------------------------
|
||||
class UserService:
|
||||
def __init__(self, pool: MockPool):
|
||||
self.pool = pool
|
||||
|
||||
async def get_users(self, page: int, page_size: int) -> dict:
|
||||
if page < 1:
|
||||
raise ValueError("page >= 1 required")
|
||||
if not 1 <= page_size <= 100:
|
||||
raise ValueError("1 <= page_size <= 100 required")
|
||||
offset = (page - 1) * page_size
|
||||
async with get_connection(self.pool) as conn:
|
||||
total_row = await conn.fetch("SELECT COUNT(*) FROM users", [])
|
||||
items = await conn.fetch(
|
||||
"SELECT id, name, label FROM users ORDER BY id LIMIT $1 OFFSET $2",
|
||||
[page_size, offset],
|
||||
)
|
||||
return {"items": items, "page": page, "page_size": page_size, "total": total_row[0]["count"]}
|
||||
|
||||
async def get_user_with_posts(self, user_id: int) -> dict:
|
||||
async with get_connection(self.pool) as conn:
|
||||
user_rows = await conn.fetch(
|
||||
"SELECT id, name, label FROM users WHERE id = $1", [user_id]
|
||||
)
|
||||
if not user_rows:
|
||||
raise KeyError(f"User {user_id} not found")
|
||||
user = user_rows[0]
|
||||
posts = await conn.fetch(
|
||||
"SELECT id, title FROM posts WHERE user_id = $1", [user_id]
|
||||
)
|
||||
return {"user": user, "posts": posts}
|
||||
|
||||
async def relabel_users(self, pairs: List[Tuple[int, str]]) -> int:
|
||||
if not pairs:
|
||||
return 0
|
||||
async with get_connection(self.pool) as conn:
|
||||
# snapshot for rollback
|
||||
snapshot = {}
|
||||
for uid, _ in pairs:
|
||||
if uid in MockDB.users:
|
||||
snapshot[uid] = MockDB.users[uid]["label"]
|
||||
try:
|
||||
for uid, label in pairs:
|
||||
await conn.execute(
|
||||
"UPDATE users SET label = $1 WHERE id = $2", [label, uid]
|
||||
)
|
||||
return len(pairs)
|
||||
except Exception:
|
||||
# rollback to original labels
|
||||
for uid, orig in snapshot.items():
|
||||
MockDB.users[uid]["label"] = orig
|
||||
return 0
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Tests
|
||||
# ----------------------------------------------------------------------
|
||||
async def run_tests():
|
||||
pool = MockPool(max_size=5, acquire_timeout=2.0)
|
||||
service = UserService(pool)
|
||||
|
||||
# a) no leak over 100 calls
|
||||
for _ in range(100):
|
||||
await service.get_users(page=1, page_size=10)
|
||||
assert pool.checked_out == 0, "leak detected"
|
||||
print("✓ a) 100 calls, pool checked_out = 0")
|
||||
|
||||
# b) pagination correctness
|
||||
res1 = await service.get_users(page=1, page_size=10)
|
||||
assert res1["total"] == 25
|
||||
assert len(res1["items"]) == 10
|
||||
res3 = await service.get_users(page=3, page_size=10)
|
||||
assert len(res3["items"]) == 5 # remainder
|
||||
res4 = await service.get_users(page=4, page_size=10)
|
||||
assert len(res4["items"]) == 0
|
||||
print("✓ b) pagination math correct")
|
||||
|
||||
# c) transactional bulk rollback
|
||||
original = {uid: MockDB.users[uid]["label"] for uid in (1, 2, 3)}
|
||||
pairs = [(1, "new1"), (2, "new2"), (999, "bad")]
|
||||
result = await service.relabel_users(pairs)
|
||||
assert result == 0
|
||||
for uid in (1, 2):
|
||||
assert MockDB.users[uid]["label"] == original[uid], "rollback failed"
|
||||
print("✓ c) relabel_users rolls back on bad id")
|
||||
|
||||
# d) concurrent access without exhaustion
|
||||
tasks = [service.get_user_with_posts(uid) for uid in range(1, 11)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
assert len(results) == 10
|
||||
assert pool.checked_out == 0
|
||||
print("✓ d) 10 concurrent get_user_with_posts, no exhaustion")
|
||||
|
||||
# validation & missing user
|
||||
try:
|
||||
await service.get_users(page=0, page_size=10)
|
||||
assert False
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
await service.get_user_with_posts(9999)
|
||||
assert False
|
||||
except KeyError:
|
||||
pass
|
||||
print("✓ validation & missing user errors")
|
||||
|
||||
# injection safety demo - params are never interpolated
|
||||
await service.relabel_users([(1, "label'; DROP TABLE users;--")])
|
||||
assert MockDB.users[1]["label"] == "label'; DROP TABLE users;--"
|
||||
print("✓ parameterized queries prevent injection")
|
||||
|
||||
print("\nAll tests passed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
Reference in New Issue
Block a user