diff --git a/.webhook.auth b/.webhook.auth new file mode 100644 index 0000000..75720f4 --- /dev/null +++ b/.webhook.auth @@ -0,0 +1 @@ +84f2a580ad552a437b90f1d19f861ab6f5f2b88434d3c5c4 diff --git a/TODO_submission_tool.md b/TODO_submission_tool.md new file mode 100644 index 0000000..b4044cc --- /dev/null +++ b/TODO_submission_tool.md @@ -0,0 +1,46 @@ +# TODO: a submission/grading tool (so grading is one command, not a manual pipeline) + +## The idea (from the user) +Right now grading a model output is a manual multi-step dance: +1. paste output → save to `outputs/.py` +2. run it, see if tests pass +3. manually audit 5 pillars +4. hand-write a JSON entry with tok/sec/tokens/TTFT/score/bugs/patch +5. append to `data/benchmark_history.json` +6. re-run `generate_dashboard.py` + redeploy + +The user wants a **tool** (likely an MCP server you can call from your editor/agent, +or a CLI) where you say: +> "grade this .py, model name = X, tok/sec = Y, tokens = Z, TTFT = W" +… and it runs the tests, captures pass/fail + crash output, and stages the JSON entry +(so I/the agent then do the actual *audit* — the subjective 5-pillar scoring + bug writeup — +on top of the auto-collected facts). + +## What it should auto-collect (deterministic, no judgment) +- [ ] Run the file; capture: parse OK? tests pass? stderr/crash output? +- [ ] Static scan: `__slots__` present? `time.monotonic()` used? any `min/max/sorted/heapq`? +- [ ] File metrics: line count, token-ish count +- [ ] Append a **draft** entry to the JSON with `total_score: null` + `verdict: "pending"` + and the auto-fields filled, so the human/agent only fills the subjective parts. + +## What stays human/agent (the actual audit — can't be automated honestly) +- The 5-pillar scores (0–20 each) +- The critical-bugs list + the patch code +- The verdict + best-for recommendation +- The `prompt_id` (which exam: lfu / tts / mcp / rust / data / automation) + +## Two build options +1. **CLI** (`./grade.sh outputs/foo.py --model "Qwen 6-bit" --tok 69 --tokens 4000 --ttft 0.9 --prompt tts`): + simplest, runs anywhere, no MCP setup. Prints the draft JSON entry + a summary. +2. **MCP server** (`grade_tool`, `list_results_tool`, `regenerate_dashboard_tool`): + callable from Claude Code / your agent so you can grade from inside a chat. Needs the MCP + server running (the same pattern as your joplin/vault MCPs). More powerful but more setup. + +## Recommended path +Start with the **CLI** (fast to build, works today, no LM-Studio/LiteLLM dependency). +Promote to an MCP server later once you've got LiteLLM set up for the MCP-prompt testing — +then the same MCP host can serve both the grading tool AND be the endpoint you test against. + +## Status +Not started. Build after the multi-prompt schema (`prompt_id`) is in place, since the tool +will need to tag which prompt an output is for. diff --git a/data/benchmark_history.json b/data/benchmark_history.json index 2479877..b4e15ce 100644 --- a/data/benchmark_history.json +++ b/data/benchmark_history.json @@ -12,7 +12,33 @@ "test_integrity" ], "max_per_pillar": 20, - "schema_version": 1 + "schema_version": 1, + "prompts": { + "lfu": { + "label": "LFU Cache + ACID (systems/async)", + "file": "prompts/lfu_cache_prompt.txt" + }, + "tts": { + "label": "TTS Pipeline (queues/backpressure)", + "file": "prompts/tts_pipeline.txt" + }, + "mcp": { + "label": "MCP Tool Server", + "file": "prompts/mcp_server.txt" + }, + "rust": { + "label": "Rust tokio Service", + "file": "prompts/rust_service.txt" + }, + "data": { + "label": "Data Service (pooling/SQL)", + "file": "prompts/data_service.txt" + }, + "automation": { + "label": "Automation Glue (idempotent batch)", + "file": "prompts/automation_glue.txt" + } + } }, "models": [ { @@ -44,7 +70,8 @@ "Lost-update risk: commit applies tx-local writes without any MVCC/version check, so a key modified by the background evictor or another committer between tx.get() and commit() is overwritten blindly.", "Tests dodge hard cases: 50-task stress uses unique keys with capacity 100, so no eviction-under-contention ever happens; no test for rollback-after-partial-application or mid-commit read isolation." ], - "patch_code": "# FIX 1: Add __slots__ for memory efficiency\n@dataclass\nclass Node:\n __slots__ = ('key', 'value', 'freq', 'expires_at', 'prev', 'next')\n key: Any\n value: Any\n freq: int\n expires_at: Optional[float]\n prev: Optional['Node'] = None\n next: Optional['Node'] = None\n\n# FIX 2: Use monotonic clock everywhere (get/put/_add_node/commit)\n# time.time() -> time.monotonic()\n# e.g.\nexpires_at = time.monotonic() + ttl_seconds if ttl_seconds else None\n\n# FIX 3: Make _cleanup_freq_lists O(1) \u2014 bump min_freq incrementally\n# instead of recomputing min() across all tiers:\n# In _update_freq, when emptying the min_freq bucket, only bump min_freq\n# if you're evicting from it; otherwise leave it. Delete the global\n# min(self.freq_map.keys()) scan. For background sweeps, prune empty\n# buckets lazily on next _evict() rather than scanning proactively.\n\n# FIX 4: Shrink commit critical section \u2014 apply writes into a staging\n# structure under the lock, then release; or use per-bucket locks so\n# readers on unrelated keys aren't blocked.\n\n# FIX 5: Add MVCC version to Node; in commit, raise/abort if the\n# stored version != the version seen at tx.get() time (lost-update detect)." + "patch_code": "# FIX 1: Add __slots__ for memory efficiency\n@dataclass\nclass Node:\n __slots__ = ('key', 'value', 'freq', 'expires_at', 'prev', 'next')\n key: Any\n value: Any\n freq: int\n expires_at: Optional[float]\n prev: Optional['Node'] = None\n next: Optional['Node'] = None\n\n# FIX 2: Use monotonic clock everywhere (get/put/_add_node/commit)\n# time.time() -> time.monotonic()\n# e.g.\nexpires_at = time.monotonic() + ttl_seconds if ttl_seconds else None\n\n# FIX 3: Make _cleanup_freq_lists O(1) \u2014 bump min_freq incrementally\n# instead of recomputing min() across all tiers:\n# In _update_freq, when emptying the min_freq bucket, only bump min_freq\n# if you're evicting from it; otherwise leave it. Delete the global\n# min(self.freq_map.keys()) scan. For background sweeps, prune empty\n# buckets lazily on next _evict() rather than scanning proactively.\n\n# FIX 4: Shrink commit critical section \u2014 apply writes into a staging\n# structure under the lock, then release; or use per-bucket locks so\n# readers on unrelated keys aren't blocked.\n\n# FIX 5: Add MVCC version to Node; in commit, raise/abort if the\n# stored version != the version seen at tx.get() time (lost-update detect).", + "prompt_id": "lfu" }, { "id": "qwen3.6-35b-a3b-4bit-mlx", @@ -76,7 +103,8 @@ "Commit is not atomic across exceptions: a crash mid-_apply loop leaves half-applied global state with no rollback.", "Uses time.time() (system clock) throughout instead of time.monotonic() \u2014 NTP jumps corrupt TTL eviction." ], - "patch_code": "# FIX 1 (the crash): _evict double-removes. pop() already unlinks,\n# so do NOT call _remove_node on a popped node. Either:\n# (a) pop and then only delete the key_map entry + min_freq bookkeeping:\ndef _evict(self):\n if not self.freq_to_list:\n return\n evict_list = self.freq_to_list[self.min_freq]\n if evict_list.size == 0: # guard against empty\n del self.freq_to_list[self.min_freq]\n return\n node = evict_list.pop() # pop() unlinks + nulls prev/next\n del self.key_to_node[node.key] # DON'T call _remove_node again\n if self.freq_to_list[self.min_freq].size == 0:\n del self.freq_to_list[self.min_freq]\n self.min_freq += 1\n\n# FIX 2: _FreqList.pop empty-guard\ndef pop(self) -> _Node:\n if self.size == 0:\n raise IndexError('pop from empty _FreqList')\n node = self.head.next\n self.remove(node)\n return node\n\n# FIX 3: kill the O(N) scan in background sweep \u2014 maintain a separate\n# set of keys that have a TTL, and iterate that set in batches:\nasync with self.lock:\n batch = list(self._ttl_keys)[:50]\n for k in batch:\n node = self.key_to_node.get(k)\n if node and 0 < node.expires_at <= time.monotonic():\n self._remove_node(node)\n\n# FIX 4: time.time() -> time.monotonic() everywhere.\n# FIX 5: add node.version; in tx._apply_put, abort/refresh if\n# cache.key_to_node[key] is a different node than original_node." + "patch_code": "# FIX 1 (the crash): _evict double-removes. pop() already unlinks,\n# so do NOT call _remove_node on a popped node. Either:\n# (a) pop and then only delete the key_map entry + min_freq bookkeeping:\ndef _evict(self):\n if not self.freq_to_list:\n return\n evict_list = self.freq_to_list[self.min_freq]\n if evict_list.size == 0: # guard against empty\n del self.freq_to_list[self.min_freq]\n return\n node = evict_list.pop() # pop() unlinks + nulls prev/next\n del self.key_to_node[node.key] # DON'T call _remove_node again\n if self.freq_to_list[self.min_freq].size == 0:\n del self.freq_to_list[self.min_freq]\n self.min_freq += 1\n\n# FIX 2: _FreqList.pop empty-guard\ndef pop(self) -> _Node:\n if self.size == 0:\n raise IndexError('pop from empty _FreqList')\n node = self.head.next\n self.remove(node)\n return node\n\n# FIX 3: kill the O(N) scan in background sweep \u2014 maintain a separate\n# set of keys that have a TTL, and iterate that set in batches:\nasync with self.lock:\n batch = list(self._ttl_keys)[:50]\n for k in batch:\n node = self.key_to_node.get(k)\n if node and 0 < node.expires_at <= time.monotonic():\n self._remove_node(node)\n\n# FIX 4: time.time() -> time.monotonic() everywhere.\n# FIX 5: add node.version; in tx._apply_put, abort/refresh if\n# cache.key_to_node[key] is a different node than original_node.", + "prompt_id": "lfu" }, { "id": "qwen3.6-35b-a3b-uncensored-hauhaucs-aggressive-gguf", @@ -108,7 +136,8 @@ "No __slots__ on _Node despite using a dataclass (rubric required it for memory efficiency).", "_remove_key will KeyError on self.freq_map[freq] if a concurrent operation already deleted that bucket." ], - "patch_code": "# FIX 1 (the parse error): assign first, then assert.\nval_d = await cache.get(\"D\")\nassert val_d, \"D should exist\"\nval_a = await cache.get(\"A\")\nassert val_a, \"A should exist (highest freq)\"\n\n# FIX 2: delete the broken while:pass loop. Bump min_freq incrementally:\n# only when the min_freq bucket empties, and only ever UP by 1 (a key\n# whose freq increased must land at min_freq+1). Never call min()/max().\nif not old_bucket:\n del self.freq_map[old_freq]\n if self.min_freq == old_freq:\n self.min_freq += 1 # next tier up; never scan\n\n# FIX 3: do ALL lazy eviction INSIDE the lock, not before it:\nasync def get(self, key):\n async with self._lock:\n if self.ttl_map.get(key, inf) <= time.monotonic():\n await self._remove_key(key) # now locked\n return None\n ...\n\n# FIX 4: make _remove_key a non-locking private helper, called from\n# inside already-locked public methods, so background_loop doesn't try\n# to re-acquire the non-reentrant asyncio.Lock.\n\n# FIX 5: add __slots__ = ('key','value','ttl_expiry') to _Node." + "patch_code": "# FIX 1 (the parse error): assign first, then assert.\nval_d = await cache.get(\"D\")\nassert val_d, \"D should exist\"\nval_a = await cache.get(\"A\")\nassert val_a, \"A should exist (highest freq)\"\n\n# FIX 2: delete the broken while:pass loop. Bump min_freq incrementally:\n# only when the min_freq bucket empties, and only ever UP by 1 (a key\n# whose freq increased must land at min_freq+1). Never call min()/max().\nif not old_bucket:\n del self.freq_map[old_freq]\n if self.min_freq == old_freq:\n self.min_freq += 1 # next tier up; never scan\n\n# FIX 3: do ALL lazy eviction INSIDE the lock, not before it:\nasync def get(self, key):\n async with self._lock:\n if self.ttl_map.get(key, inf) <= time.monotonic():\n await self._remove_key(key) # now locked\n return None\n ...\n\n# FIX 4: make _remove_key a non-locking private helper, called from\n# inside already-locked public methods, so background_loop doesn't try\n# to re-acquire the non-reentrant asyncio.Lock.\n\n# FIX 5: add __slots__ = ('key','value','ttl_expiry') to _Node.", + "prompt_id": "lfu" }, { "id": "kat-coder-v2.5-dev-xl-mlx", @@ -139,7 +168,8 @@ "_evict_node uses min(self._freq_map) (line 325) when the min-tier empties \u2014 a hidden O(F) scan, violating strict O(1).", "No MVCC/version check on transaction commit (lost-update possible if the global key is modified between tx.get and commit); commit is not exception-safe across the writes-vs-deletes loops." ], - "patch_code": "# FIX 1 (the fatal one-liner): use the helper that already exists.\n# line 305, in _put_internal, new-key branch:\n- self._freq_map[1].push_front(dll_node)\n+ self._ensure_freq_list(1).push_front(dll_node)\n# (This single change makes the cache and transactions functional.)\n\n# FIX 2: replace the O(F) min() scan with an incremental bump:\n# in _evict_node, when the min-tier bucket empties, min_freq is the\n# lowest remaining tier. Since freq only ever increments by 1, the\n# next min is almost always min_freq+1; track it incrementally rather\n# than scanning. Or, since this only happens on full eviction, accept\n# O(F) but only on the empty-cache edge \u2014 document it.\n\n# FIX 3: add __slots__ to all internal classes:\nclass _CacheNode:\n __slots__ = ('key','value','ttl_seconds','expiry_time','freq','dll_node')\n ...\n\n# FIX 4: wrap commit applies in try/except so a mid-commit exception\n# does not leave a half-applied global state; consider abort semantics.\n# FIX 5: add node.version; in tx commit, abort if the global node for\n# a key is not the one seen at tx.get() time." + "patch_code": "# FIX 1 (the fatal one-liner): use the helper that already exists.\n# line 305, in _put_internal, new-key branch:\n- self._freq_map[1].push_front(dll_node)\n+ self._ensure_freq_list(1).push_front(dll_node)\n# (This single change makes the cache and transactions functional.)\n\n# FIX 2: replace the O(F) min() scan with an incremental bump:\n# in _evict_node, when the min-tier bucket empties, min_freq is the\n# lowest remaining tier. Since freq only ever increments by 1, the\n# next min is almost always min_freq+1; track it incrementally rather\n# than scanning. Or, since this only happens on full eviction, accept\n# O(F) but only on the empty-cache edge \u2014 document it.\n\n# FIX 3: add __slots__ to all internal classes:\nclass _CacheNode:\n __slots__ = ('key','value','ttl_seconds','expiry_time','freq','dll_node')\n ...\n\n# FIX 4: wrap commit applies in try/except so a mid-commit exception\n# does not leave a half-applied global state; consider abort semantics.\n# FIX 5: add node.version; in tx commit, abort if the global node for\n# a key is not the one seen at tx.get() time.", + "prompt_id": "lfu" }, { "id": "gemma4-31b-gguf", @@ -172,7 +202,8 @@ "No MVCC/version check on commit (lost-update possible); commit is not exception-safe across the deletes-vs-puts loops.", "Tests pass but don't probe mid-commit read isolation or eviction-under-real-contention (capacity sized so all keys fit), so the isolation leak above goes undetected." ], - "patch_code": "# FIX 1 (the isolation leak): tx reads must NOT mutate global freq.\n# Add a read-only global lookup (no _update_frequency) and use it in tx.get:\nasync def _read_raw(self, key): # no freq bump\n node = self.cache.get(key)\n if node is None: return None\n if time.monotonic() > node.expiry:\n await self._delete_internal(key)\n return None\n return node.value\n# then in Transaction.get fallback:\n return await self._cache._read_raw(key) # NOT cache.get\n\n# FIX 2: time.time() -> time.monotonic() everywhere (get/put/_put_internal/bg loop).\n# FIX 3: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 4: prune empty freq buckets on delete, or have _evict_lfu drop them.\n# FIX 5: iterate a dedicated _ttl_keys set (batched) in the bg evictor\n# instead of list(self.cache.keys()) to stay O(batch), not O(N)." + "patch_code": "# FIX 1 (the isolation leak): tx reads must NOT mutate global freq.\n# Add a read-only global lookup (no _update_frequency) and use it in tx.get:\nasync def _read_raw(self, key): # no freq bump\n node = self.cache.get(key)\n if node is None: return None\n if time.monotonic() > node.expiry:\n await self._delete_internal(key)\n return None\n return node.value\n# then in Transaction.get fallback:\n return await self._cache._read_raw(key) # NOT cache.get\n\n# FIX 2: time.time() -> time.monotonic() everywhere (get/put/_put_internal/bg loop).\n# FIX 3: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 4: prune empty freq buckets on delete, or have _evict_lfu drop them.\n# FIX 5: iterate a dedicated _ttl_keys set (batched) in the bg evictor\n# instead of list(self.cache.keys()) to stay O(batch), not O(N).", + "prompt_id": "lfu" }, { "id": "gemma-4-31b-qat-gguf", @@ -206,7 +237,8 @@ "Background sweep does list(self.cache.keys()) = O(N) per interval.", "Tests pass but use the non-spec commit path and don't probe capacity breach or isolation leak." ], - "patch_code": "# FIX 1 (capacity breach): guard + prune empty buckets on eviction:\nwhile self.min_freq in self.freq_map and self.freq_map[self.min_freq].size == 0:\n del self.freq_map[self.min_freq]\n self.min_freq += 1\n if not self.freq_map:\n break\nif self.min_freq not in self.freq_map:\n return # nothing to evict\nevicted = self.freq_map[self.min_freq].pop_tail()\nif evicted and self.freq_map[self.min_freq].size == 0:\n del self.freq_map[self.min_freq]\n\n# FIX 2 (conformant API): add commit/rollback to Transaction:\nasync def commit(self):\n await self._cache.apply_transaction_changes(self._state)\n self._committed = True\ndef rollback(self):\n self._state.writes.clear()\n self._committed = True\n\n# FIX 3 (isolation leak): tx.get should use a read-only global lookup\n# (no _update_freq), not the public cache.get.\n# FIX 4: _get_now returns time.monotonic().\n# FIX 5: add __slots__ to all node/list classes." + "patch_code": "# FIX 1 (capacity breach): guard + prune empty buckets on eviction:\nwhile self.min_freq in self.freq_map and self.freq_map[self.min_freq].size == 0:\n del self.freq_map[self.min_freq]\n self.min_freq += 1\n if not self.freq_map:\n break\nif self.min_freq not in self.freq_map:\n return # nothing to evict\nevicted = self.freq_map[self.min_freq].pop_tail()\nif evicted and self.freq_map[self.min_freq].size == 0:\n del self.freq_map[self.min_freq]\n\n# FIX 2 (conformant API): add commit/rollback to Transaction:\nasync def commit(self):\n await self._cache.apply_transaction_changes(self._state)\n self._committed = True\ndef rollback(self):\n self._state.writes.clear()\n self._committed = True\n\n# FIX 3 (isolation leak): tx.get should use a read-only global lookup\n# (no _update_freq), not the public cache.get.\n# FIX 4: _get_now returns time.monotonic().\n# FIX 5: add __slots__ to all node/list classes.", + "prompt_id": "lfu" }, { "id": "gemma-4-12b-coder-heretic-mxfp8-mlx", @@ -238,7 +270,8 @@ "No __slots__; time.time() (not monotonic) throughout.", "Tests cannot run \u2014 crash at first put." ], - "patch_code": "# FIX 1 (the fatal KeyError): create the bucket before use.\n# In put(), new-key branch:\n- bucket = self.freq_buckets[self.min_freq]\n+ bucket = self.freq_buckets.setdefault(self.min_freq, DoublyLinkedListList())\n# Same fix in _update_freq and Transaction._update_local_freq (use setdefault).\n\n# FIX 2 (the broken evictor): make it a real async task that deletes:\nasync def _evict_loop(self, interval):\n while True:\n await asyncio.sleep(interval) # async, non-blocking\n now = time.monotonic()\n async with self.global_lock:\n expired = [k for k, n in list(self.cache.items()) if now > n.ttl_expiry]\n for k in expired:\n node = self.cache.pop(k, None)\n if node:\n self.freq_buckets[node.freq].remove(node) # DELETE, not re-put\n\nasync def start_evictor(self, interval=1.0):\n self.evictor_task = asyncio.create_task(self._evict_loop(interval))\n\n# FIX 3: delete expired keys; do NOT re-insert with TTL 0.\n# FIX 4: time.time() -> time.monotonic().\n# FIX 5: add __slots__ to Node / DoublyLinkedListList / ConcurrentLFUCache / Transaction." + "patch_code": "# FIX 1 (the fatal KeyError): create the bucket before use.\n# In put(), new-key branch:\n- bucket = self.freq_buckets[self.min_freq]\n+ bucket = self.freq_buckets.setdefault(self.min_freq, DoublyLinkedListList())\n# Same fix in _update_freq and Transaction._update_local_freq (use setdefault).\n\n# FIX 2 (the broken evictor): make it a real async task that deletes:\nasync def _evict_loop(self, interval):\n while True:\n await asyncio.sleep(interval) # async, non-blocking\n now = time.monotonic()\n async with self.global_lock:\n expired = [k for k, n in list(self.cache.items()) if now > n.ttl_expiry]\n for k in expired:\n node = self.cache.pop(k, None)\n if node:\n self.freq_buckets[node.freq].remove(node) # DELETE, not re-put\n\nasync def start_evictor(self, interval=1.0):\n self.evictor_task = asyncio.create_task(self._evict_loop(interval))\n\n# FIX 3: delete expired keys; do NOT re-insert with TTL 0.\n# FIX 4: time.time() -> time.monotonic().\n# FIX 5: add __slots__ to Node / DoublyLinkedListList / ConcurrentLFUCache / Transaction.", + "prompt_id": "lfu" }, { "id": "deepseekv4flash-cloud", @@ -269,7 +302,8 @@ "No explicit lost-update/conflict abort on commit (delta-based freq is applied unconditionally).", "Tests pass 20/20 but don't include a mid-commit read-isolation probe or adversarial eviction-under-contention stress." ], - "patch_code": "# These are minor refinements on an already production-ready file.\n\n# FIX 1: eliminate the recovery min() scans by keeping min_freq\n# strictly in sync on every insert/bump/remove (it already does on\n# the hot path), so the _evict_one recovery branch is unreachable and\n# can assert instead of scanning:\nassert self._min_freq in self._freq_to_list or not self._freq_to_list\n\n# FIX 2: extend __slots__ to Transaction and LFUCache.\nclass LFUCache(Generic[KT, VT]):\n __slots__ = ('_capacity','_key_to_node','_freq_to_list','_min_freq',\n '_lock','_ttl_index','_evictor_task','_closed')\n\n# FIX 3 (optional): on commit, if a key's global node changed since the\n# tx snapshot, raise LFUCacheError('lost update') instead of overwriting." + "patch_code": "# These are minor refinements on an already production-ready file.\n\n# FIX 1: eliminate the recovery min() scans by keeping min_freq\n# strictly in sync on every insert/bump/remove (it already does on\n# the hot path), so the _evict_one recovery branch is unreachable and\n# can assert instead of scanning:\nassert self._min_freq in self._freq_to_list or not self._freq_to_list\n\n# FIX 2: extend __slots__ to Transaction and LFUCache.\nclass LFUCache(Generic[KT, VT]):\n __slots__ = ('_capacity','_key_to_node','_freq_to_list','_min_freq',\n '_lock','_ttl_index','_evictor_task','_closed')\n\n# FIX 3 (optional): on commit, if a key's global node changed since the\n# tx snapshot, raise LFUCacheError('lost update') instead of overwriting.", + "prompt_id": "lfu" }, { "id": "gemma4-26b-a4b-8bit-mlx", @@ -299,7 +333,8 @@ "Background evictor does list(self.cache_data.keys()) = O(N) snapshot every interval \u2014 a linear scan, forbidden by strict O(1).", "No MVCC/version check on commit (lost-update possible if the global key changes between tx.get and commit); commit applies deletes->reads->puts without try/except, so a mid-commit exception leaves partial state." ], - "patch_code": "# FIX 1 (stale min_freq): recompute or invalidate when the min bucket empties.\n# In _remove_node_from_structures, replace the `pass`:\nif dll.size == 0:\n del self.freq_map[node.freq]\n if self.min_freq == node.freq:\n # bump to next existing tier (frequencies are contiguous under normal use)\n self.min_freq = self.min_freq + 1 if (self.min_freq + 1) in self.freq_map else min(self.freq_map, default=1)\n\n# FIX 2: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 3: background evictor \u2014 maintain a _ttl_keys set and iterate IT in\n# batches instead of list(self.cache_data.keys()) to stay O(batch).\n# FIX 4: wrap commit's three loops in try/except with rollback semantics on failure." + "patch_code": "# FIX 1 (stale min_freq): recompute or invalidate when the min bucket empties.\n# In _remove_node_from_structures, replace the `pass`:\nif dll.size == 0:\n del self.freq_map[node.freq]\n if self.min_freq == node.freq:\n # bump to next existing tier (frequencies are contiguous under normal use)\n self.min_freq = self.min_freq + 1 if (self.min_freq + 1) in self.freq_map else min(self.freq_map, default=1)\n\n# FIX 2: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 3: background evictor \u2014 maintain a _ttl_keys set and iterate IT in\n# batches instead of list(self.cache_data.keys()) to stay O(batch).\n# FIX 4: wrap commit's three loops in try/except with rollback semantics on failure.", + "prompt_id": "lfu" }, { "id": "qwen3.6-27b-8bit-mlx", @@ -331,7 +366,8 @@ "Background _evict_loop materializes list(self.nodes.keys()) (O(N)) before checking only batch_size keys \u2014 the break caps work but not the snapshot cost, a hidden O(N) per sweep.", "Commit re-implements put+evict inline (lines 102-118) duplicating the public path = duplicated bug surface; not wrapped in try/except so a mid-commit exception leaves partial state. No MVCC version check (lost-update possible)." ], - "patch_code": "# FIX 1 (isolation leak): add a read-only global lookup (no freq bump)\n# and use it in tx.get instead of the public cache.get:\nasync def _read_raw(self, key):\n async with self.lock:\n node = self.nodes.get(key)\n if node is None: return None\n if time.monotonic() > node.expires_at:\n self._remove_node(key); return None\n return node.value\n# then: return await self._cache._read_raw(key)\n# FIX 2: time.time() -> time.monotonic() everywhere.\n# FIX 3: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 4: maintain a _ttl_keys set; iterate IT (batched) in the bg loop\n# instead of list(self.nodes.keys()).\n# FIX 5: factor commit's put/evict to reuse the internal helpers; wrap\n# the commit loop in try/except with rollback-on-failure." + "patch_code": "# FIX 1 (isolation leak): add a read-only global lookup (no freq bump)\n# and use it in tx.get instead of the public cache.get:\nasync def _read_raw(self, key):\n async with self.lock:\n node = self.nodes.get(key)\n if node is None: return None\n if time.monotonic() > node.expires_at:\n self._remove_node(key); return None\n return node.value\n# then: return await self._cache._read_raw(key)\n# FIX 2: time.time() -> time.monotonic() everywhere.\n# FIX 3: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 4: maintain a _ttl_keys set; iterate IT (batched) in the bg loop\n# instead of list(self.nodes.keys()).\n# FIX 5: factor commit's put/evict to reuse the internal helpers; wrap\n# the commit loop in try/except with rollback-on-failure.", + "prompt_id": "lfu" }, { "id": "qwen3-coder-30b-6bit-mlx", @@ -363,7 +399,42 @@ "No __slots__ on CacheNode/FrequencyBucket/InMemoryLFUCache/Transaction \u2014 rubric required it.", "FrequencyBucket stores nodes in BOTH a DLL and a parallel `nodes` dict \u2014 redundant memory per bucket." ], - "patch_code": "# FIX 1 (the fatal crash): make begin_transaction async and use async with.\nasync def begin_transaction(self) -> 'Transaction':\n async with self._transaction_lock:\n self._transaction_counter += 1\n tx = Transaction(self)\n self._transactions[self._transaction_counter] = tx\n return tx\n# (and update callers: `tx = await cache.begin_transaction()`)\n# Alternative if sync creation is required: use threading.Lock for the\n# counter, but that is wrong in an asyncio codebase \u2014 go async.\n\n# FIX 2: remove unused `import threading` and `import weakref`.\n# FIX 3: time.time() -> time.monotonic() everywhere.\n# FIX 4: add __slots__ to all classes.\n# FIX 5: drop the redundant FrequencyBucket.nodes dict; the DLL already\n# tracks membership, so the dict is duplicate storage." + "patch_code": "# FIX 1 (the fatal crash): make begin_transaction async and use async with.\nasync def begin_transaction(self) -> 'Transaction':\n async with self._transaction_lock:\n self._transaction_counter += 1\n tx = Transaction(self)\n self._transactions[self._transaction_counter] = tx\n return tx\n# (and update callers: `tx = await cache.begin_transaction()`)\n# Alternative if sync creation is required: use threading.Lock for the\n# counter, but that is wrong in an asyncio codebase \u2014 go async.\n\n# FIX 2: remove unused `import threading` and `import weakref`.\n# FIX 3: time.time() -> time.monotonic() everywhere.\n# FIX 4: add __slots__ to all classes.\n# FIX 5: drop the redundant FrequencyBucket.nodes dict; the DLL already\n# tracks membership, so the dict is duplicate storage.", + "prompt_id": "lfu" + }, + { + "id": "qwen3.6-35b-a3b-6bit-mlx-tts", + "prompt_id": "tts", + "timestamp": "2026-07-29T00:25:00Z", + "model_name": "Qwen 3.6 35B-A3B", + "quant": "6-bit MLX", + "param_size": "35B-A3B (MoE)", + "format": "mlx", + "tok_sec": 69.87, + "total_tokens": 11996, + "ttft_sec": 0.84, + "filename": "outputs/qwen3.6-35b-a3b-6bit-mlx-tts.py", + "tests_pass": false, + "total_score": 49, + "breakdown": { + "complexity": 10, + "concurrency": 8, + "error_handling": 13, + "resource_safety": 12, + "test_integrity": 6 + }, + "verdict": "Critical Bugs", + "best_for": "NOT a safe offload for async-pipeline / queue work despite scoring 82 on the LFU exam. Same model+quant, different prompt: 82 -> 49. Use it for data-structure/ACID tasks; route queue/backpressure/retry work elsewhere (or to the cloud model).", + "critical_bugs": [ + "FATAL: file does not parse. 'async with self._callbacks_lock' (L103) and 'await coro' (L98) are inside on_event, a SYNC def. SyntaxError before any test runs.", + "Bounded-concurrency design is broken: only ONE worker task is created (start() does a single create_task), and the Semaphore(4) is acquired inside that single loop. Effective concurrency is 1, not 4 \u2014 the semaphore is decorative.", + "Test A asserts max_concurrency <= 4, but since real concurrency is ~1 it passes trivially and proves NOTHING about the cap actually holding under saturation.", + "Cooperative cancel only sets a flag; it cannot interrupt an in-flight mock_synthesize mid-sleep (acceptable, but not 'cancel at next safe checkpoint' for a long synth).", + "submit() after stop() silently enqueues to a dead worker \u2014 no rejection, jobs leak (never processed, never failed).", + "_final_states dict grows unbounded (no eviction) \u2014 memory leak for a long-running service.", + "Callback errors are silently swallowed (broad try/except Exception) \u2014 a silent-failure pattern; debug visibility lost." + ], + "patch_code": "# FIX 1 (the parse error): make on_event async (or don't await inside it).\n# Simplest correct version:\nasync def on_event(self, callback):\n async with self._callbacks_lock:\n self._callbacks.append(callback)\n\n# FIX 2 (real bounded concurrency): spawn N workers OR create_task per job\n# gated by the semaphore. Option B (concurrency from the semaphore itself):\nasync def _worker_loop(self):\n while self._running:\n job = await self._queue.get()\n # do NOT hold the semaphore in the single worker; instead launch\n # each job as its own task, gated so at most max_concurrency run:\n async def _run(j):\n async with self._semaphore:\n await self._process_job(j)\n self._queue.task_done()\n asyncio.create_task(_run(job))\n# (and add a test that submits >max_concurrency long jobs and asserts\n# exactly max_concurrency run at once.)\n\n# FIX 3: reject submit() after stop() (guard on self._running).\n# FIX 4: bound _final_states (e.g. keep last N, or evict terminal >TTL).\n# FIX 5: log callback errors instead of swallowing them silently." } ] } \ No newline at end of file diff --git a/generate_dashboard.py b/generate_dashboard.py index a3e31ac..15f80d5 100644 --- a/generate_dashboard.py +++ b/generate_dashboard.py @@ -11,14 +11,40 @@ DATA = os.path.join(HERE, "data", "benchmark_history.json") OUT_DASH = os.path.join(HERE, "dashboard.html") PAGES_DIR = os.path.join(HERE, "pages") -PILLARS = ["complexity", "concurrency", "isolation", "memory_edge_cases", "test_integrity"] -PILLAR_LABELS = { - "complexity": "Complexity (O(1))", - "concurrency": "Concurrency / Races", - "isolation": "Tx Isolation", - "memory_edge_cases": "Memory & Edges", - "test_integrity": "Test Integrity", +# Per-prompt pillar definitions. Each prompt is graded on 5 pillars (0-20 each = 100), +# but the pillar NAMES differ by prompt type. The generator reads the entry's own pillars. +PILLARS_BY_PROMPT = { + "lfu": ["complexity", "concurrency", "isolation", "memory_edge_cases", "test_integrity"], + "tts": ["complexity", "concurrency", "error_handling", "resource_safety", "test_integrity"], + "mcp": ["schema_io", "transport", "error_handling", "state_safety", "test_integrity"], + "rust": ["ownership", "concurrency", "error_handling", "cancellation", "test_integrity"], + "data": ["query_safety", "pooling", "transactions", "pagination", "test_integrity"], + "automation": ["idempotency", "retry_backoff", "checkpointing", "signal_handling", "test_integrity"], } +PILLARS = PILLARS_BY_PROMPT["lfu"] # default for any code that still references the global +PILLAR_LABELS = { + # lfu + "complexity": "Complexity (O(1))", "concurrency": "Concurrency / Races", + "isolation": "Tx Isolation", "memory_edge_cases": "Memory & Edges", + "test_integrity": "Test Integrity", + # tts / shared + "error_handling": "Error Handling", "resource_safety": "Resource & State Safety", + # mcp + "schema_io": "Schema / I/O", "transport": "Transport", + "state_safety": "State Safety", + # rust + "ownership": "Ownership / Types", "cancellation": "Cancellation / Shutdown", + # data + "query_safety": "Query Safety", "pooling": "Pooling", "transactions": "Transactions", "pagination": "Pagination", + # automation + "idempotency": "Idempotency", "retry_backoff": "Retry / Backoff", + "checkpointing": "Checkpointing", "signal_handling": "Signal Handling", +} + +def pillars_for(m): + """Return the 5 pillar keys for a model entry based on its prompt_id.""" + pid = m.get("prompt_id", "lfu") + return PILLARS_BY_PROMPT.get(pid, PILLARS) # ---- cyberpunk palette ---- NEON_CYAN = "#00ffc8" @@ -144,6 +170,13 @@ tr:hover .score-bar > i{box-shadow:0 0 12px currentColor} .fmt-mlx{color:var(--cyan);background:rgba(0,255,200,0.08)} .fmt-gguf{color:var(--mag);background:rgba(255,43,214,0.08)} .fmt-cloud{color:var(--blue);background:rgba(91,139,255,0.08)} +.prompt-chip{display:inline-block;font-size:.62rem;letter-spacing:.06em;padding:2px 6px;border-radius:3px;border:1px solid currentColor;font-family:'Fira Code',monospace;text-transform:uppercase} +.p-lfu{color:var(--cyan);background:rgba(0,255,200,0.06)} +.p-tts{color:var(--lime);background:rgba(182,255,58,0.06)} +.p-mcp{color:var(--blue);background:rgba(91,139,255,0.06)} +.p-rust{color:var(--amber);background:rgba(255,176,0,0.06)} +.p-data{color:var(--mag);background:rgba(255,43,214,0.06)} +.p-automation{color:#c084fc;background:rgba(192,132,252,0.06)} /* format/quant showdown cards */ .fcard{background:var(--panel2);border:1px solid rgba(255,255,255,0.06);border-radius:6px;padding:12px 14px;margin-bottom:10px} .fcard-h{display:flex;justify-content:space-between;align-items:baseline;gap:10px;flex-wrap:wrap;margin-bottom:9px} @@ -204,6 +237,7 @@ FOOT = """ """ def render_dashboard(data): + # leaderboard ranked by score (prompt badge distinguishes same-model entries across prompts) models = sorted(data["models"], key=lambda m: -m["total_score"]) n = len(models) avg = sum(m["total_score"] for m in models) / n if n else 0 @@ -232,9 +266,11 @@ def render_dashboard(data): else: fmt_chip = f'{esc(m.get("format") or "—")}' bar_color = col + pid = m.get("prompt_id", "lfu") + pchip = f'{pid}' if pid != "lfu" else 'lfu' rows.append(f""" #{i} -
{esc(m['model_name'])}
{caveat} +
{esc(m['model_name'])} {pchip}
{caveat} {esc(m['quant'])} {fmt_chip} {speed_str(m)} t/s @@ -257,9 +293,11 @@ def render_dashboard(data): bar_speed = json.dumps([m["tok_sec"] for m in local_sorted]) bar_score = json.dumps([m["total_score"] for m in local_sorted]) + # radar compares top models on the LFU exam (apples-to-apples, same 5 axes) + radar_lfu = [m for m in models if m.get("prompt_id", "lfu") == "lfu"][:3] radar_labels = json.dumps([PILLAR_LABELS[p] for p in PILLARS]) radar_sets = [] - for idx, m in enumerate(radar_models): + for idx, m in enumerate(radar_lfu): col = SERIES[idx % len(SERIES)] radar_sets.append({ "label": m["model_name"][:24], @@ -374,7 +412,9 @@ def render_dashboard(data): n_crit = sum(1 for m in local if m["verdict"] == "Critical Bugs") n_slots = sum(1 for m in local if scans[m["id"]]["slots"]) n_mono = sum(1 for m in local if scans[m["id"]]["monotonic"]) - pillar_avg = {p: round(sum(m["breakdown"][p] for m in local)/len(local), 1) for p in PILLARS} + # findings stats are LFU-exam-scoped (the common comparison set) + lfu_models = [m for m in local if m.get("prompt_id", "lfu") == "lfu"] + pillar_avg = {p: round(sum(m["breakdown"][p] for m in lfu_models)/len(lfu_models), 1) for p in PILLARS} weakest = min(PILLARS, key=lambda p: pillar_avg[p]) def _fcard(num, label, sub): @@ -519,14 +559,15 @@ new Chart(document.getElementById('radar'),{{ def render_detail(m, data): col, chip = verdict_meta(m["verdict"]) + m_pillars = pillars_for(m) # each entry uses its own prompt's 5 pillars # derive "what went right" from high pillars, "wrong" from low + critical_bugs bd = m["breakdown"] - ranked = sorted(PILLARS, key=lambda p: -bd[p]) + ranked = sorted(m_pillars, key=lambda p: -bd[p]) rights = [f"{PILLAR_LABELS[p]} ({bd[p]}/20)" for p in ranked if bd[p] >= 16] wrongs_pillars = [f"{PILLAR_LABELS[p]} ({bd[p]}/20)" for p in ranked if bd[p] <= 13] pillar_bars = "" - for p in PILLARS: + for p in m_pillars: v = bd[p] c = NEON_LIME if v >= 17 else (NEON_AMBER if v >= 13 else NEON_RED) pillar_bars += f""" diff --git a/outputs/qwen3.6-35b-a3b-6bit-mlx-tts.py b/outputs/qwen3.6-35b-a3b-6bit-mlx-tts.py new file mode 100644 index 0000000..8a88cbf --- /dev/null +++ b/outputs/qwen3.6-35b-a3b-6bit-mlx-tts.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +""" +Async TTS Job Pipeline (Stdlib Only) +------------------------------------ +Provides bounded concurrency, backpressure, exponential retry, +clean cancellation, and lifecycle callbacks. +""" + +import asyncio +import random +import uuid +from dataclasses import dataclass +from typing import Callable, Any, Dict, List + +# --------------------------------------------------------------------------- +# Mock Synthesizer +# --------------------------------------------------------------------------- +async def mock_synthesize(text: str) -> bytes: + """ + Simulates TTS synthesis: + - Sleeps 50-300ms + - Fails ~10% of the time + - Returns dummy bytes on success + """ + await asyncio.sleep(random.uniform(0.05, 0.30)) + if random.random() < 0.10: + raise RuntimeError("Synthesis failed (mock network/device error)") + return b"\x00" * len(text) + + +# --------------------------------------------------------------------------- +# Core Pipeline +# --------------------------------------------------------------------------- +@dataclass +class Job: + id: str + text: str + voice: str + state: str = "queued" + attempts: int = 0 + + +class TTSPipeline: + def __init__( + self, + max_concurrency: int = 4, + max_queue_size: int = 100, + max_retries: int = 3, + ): + self._queue: asyncio.Queue = asyncio.Queue() + self._semaphore = asyncio.Semaphore(max_concurrency) + self._jobs: Dict[str, Job] = {} + self._jobs_lock = asyncio.Lock() + self._max_queue_size = max_queue_size + self._max_retries = max_retries + self._callbacks: List[Callable[[str, str], Any]] = [] + self._callbacks_lock = asyncio.Lock() + + # Concurrency tracking + self._active_count = 0 + self._max_concurrency_seen = 0 + self._concurrency_lock = asyncio.Lock() + + # Worker management + self._worker_task: asyncio.Task | None = None + self._running = False + + # Final state tracking (for testing/observability) + self._final_states: Dict[str, str] = {} + self._final_states_lock = asyncio.Lock() + + # ----------------------------------------------------------------------- + # Lifecycle Management + # ----------------------------------------------------------------------- + async def start(self) -> None: + if self._running: + return + self._running = True + self._worker_task = asyncio.create_task(self._worker_loop()) + + async def stop(self) -> None: + if not self._running: + return + self._running = False + if self._worker_task: + self._worker_task.cancel() + try: + await self._worker_task + except asyncio.CancelledError: + pass + + def on_event(self, callback: Callable[[str, str], Any]) -> None: + """Register a callback for job lifecycle events.""" + async def _safe_callback(job_id: str, event: str) -> None: + try: + coro = callback(job_id, event) + if asyncio.iscoroutine(coro): + await coro + except Exception: + # Fail silently to prevent callback errors from breaking the pipeline + pass + + async with self._callbacks_lock: + self._callbacks.append(_safe_callback) + + async def _notify(self, job_id: str, event: str) -> None: + async with self._callbacks_lock: + tasks = [cb(job_id, event) for cb in self._callbacks] + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + # ----------------------------------------------------------------------- + # Public API + # ----------------------------------------------------------------------- + async def submit(self, text: str, voice: str) -> str: + """Enqueue a job. Rejects immediately if backpressure threshold is hit.""" + if self._queue.qsize() >= self._max_queue_size: + raise ValueError( + f"Queue full (backpressure): max {self._max_queue_size} jobs allowed" + ) + + job_id = uuid.uuid4().hex + job = Job(id=job_id, text=text, voice=voice) + + async with self._jobs_lock: + self._jobs[job_id] = job + + await self._queue.put(job) + await self._notify(job_id, "queued") + return job_id + + async def drain(self) -> None: + """Wait until all queued AND in-flight jobs are finished.""" + await self._queue.join() + + async def cancel(self, job_id: str) -> None: + """Cancel a job. Safe for queued or in-flight jobs.""" + async with self._jobs_lock: + job = self._jobs.get(job_id) + if job and job.state in ("queued", "started"): + job.state = "cancelled" + await self._notify(job_id, "cancelled") + + # ----------------------------------------------------------------------- + # Observability (for tests) + # ----------------------------------------------------------------------- + def get_max_concurrency(self) -> int: + return self._max_concurrency_seen + + async def get_job_states(self) -> Dict[str, str]: + async with self._final_states_lock: + return dict(self._final_states) + + # ----------------------------------------------------------------------- + # Internal Worker + # ----------------------------------------------------------------------- + async def _worker_loop(self) -> None: + while self._running: + try: + job = await self._queue.get() + except asyncio.CancelledError: + break + + async with self._semaphore: + await self._process_job(job) + self._queue.task_done() + + async def _process_job(self, job: Job) -> None: + # Track concurrency + async with self._concurrency_lock: + self._active_count += 1 + if self._active_count > self._max_concurrency_seen: + self._max_concurrency_seen = self._active_count + + try: + # Skip if cancelled while in queue + if job.state == "cancelled": + await self._notify(job.id, "cancelled") + async with self._final_states_lock: + self._final_states[job.id] = "cancelled" + return + + await self._notify(job.id, "started") + + max_attempts = self._max_retries + 1 # 1 initial + 3 retries + for attempt in range(max_attempts): + if job.state == "cancelled": + break + + try: + await mock_synthesize(job.text) + job.state = "completed" + await self._notify(job.id, "completed") + async with self._final_states_lock: + self._final_states[job.id] = "completed" + return + except Exception: + # Failure handling + if attempt < max_attempts - 1: + backoff = 0.1 * (2 ** attempt) # 0.1, 0.2, 0.4 + await asyncio.sleep(backoff) + else: + job.state = "failed" + await self._notify(job.id, "failed") + async with self._final_states_lock: + self._final_states[job.id] = "failed" + return + finally: + async with self._concurrency_lock: + self._active_count -= 1 + + +# --------------------------------------------------------------------------- +# Test Suite +# --------------------------------------------------------------------------- +async def main() -> None: + pipeline = TTSPipeline(max_concurrency=4, max_queue_size=100, max_retries=3) + await pipeline.start() + + # ----------------------------------------------------------------------- + # a) Bounded Concurrency Test + # ----------------------------------------------------------------------- + print("Test A: Bounded concurrency (50 jobs)...") + ids_a = [await pipeline.submit(f"text_{i}", "voice_1") for i in range(50)] + await pipeline.drain() + + max_c = pipeline.get_max_concurrency() + assert max_c <= 4, f"❌ Concurrency exceeded: {max_c}" + print(f" ✅ Max concurrency observed: {max_c}") + + # ----------------------------------------------------------------------- + # b) Backpressure Test + # ----------------------------------------------------------------------- + print("\nTest B: Backpressure cap (100 jobs)...") + ids_b = [await pipeline.submit(f"bp_{i}", "voice_1") for i in range(100)] + try: + await pipeline.submit("bp_overflow", "voice_1") + assert False, "❌ Should have raised ValueError" + except ValueError as e: + print(f" ✅ Backpressure triggered: {e}") + + # ----------------------------------------------------------------------- + # c) Cancellation & Pool Recovery Test + # ----------------------------------------------------------------------- + print("\nTest C: Cancellation mid-flight & pool recovery...") + ids_c = [await pipeline.submit(f"cancel_{i}", "voice_2") for i in range(10)] + # Cancel one immediately after submission + await pipeline.cancel(ids_c[3]) + await pipeline.drain() + + states = await pipeline.get_job_states() + assert len(states) == 10, f"❌ Job count mismatch: {len(states)}" + assert len(set(states.keys())) == 10, "❌ Duplicate job IDs found" + + terminal_states = {"completed", "failed", "cancelled"} + for jid, st in states.items(): + assert st in terminal_states, f"❌ Non-terminal state for {jid}: {st}" + + print(f" ✅ All 10 jobs reached terminal state.") + print(f" States: {states}") + + # Verify pool is still usable after cancellation + recover_id = await pipeline.submit("recover_test", "voice_3") + await pipeline.drain() + final_states = await pipeline.get_job_states() + assert final_states.get(recover_id) == "completed", "❌ Pool is broken after cancellation" + print(" ✅ Pool remains fully functional after cancellation.") + + await pipeline.stop() + print("\n✅ All tests passed.") + + +if __name__ == "__main__": + asyncio.run(main())