From 3507d3300644637e4201620756e0958c777be3e0 Mon Sep 17 00:00:00 2001 From: Aygea Date: Tue, 28 Jul 2026 21:17:08 -0700 Subject: [PATCH] Grade 12-model backlog + fix native-API TTFT capture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grading (12 new entries, 18→30 total in benchmark_history.json): - KAT-Coder v2.5 Dev XL: lfu 49 / tts 52 / webhook 72 / rust 36 / automation 60 - Qwen3 Coder 30B: lfu 45 / tts 44 / webhook 62 / rust 54 / automation 58 - Qwen 3.6 35B-A3B uncensored (automation): 46 - Gemma 4 26B-A4B (data): 86 [tests pass] All "coder" models scored Critical Bugs across prompts — plausible-looking async code with fatal bugs (broken LFU eviction, in-flight cancel no-op, submit() raising instead of backpressuring, un-awaited async read-through). grade_run.py: switch from OpenAI-compat /v1/chat/completions (empty stats) to native /api/v1/chat — returns full stats incl. time_to_first_token_seconds. Verified on Gemma-26B (53.5 t/s, ttft 1.13). Two native-API gotchas handled: input (string) not messages; max_output_tokens not max_tokens (that 400s). New capture: outputs/gemma-4-26b-a4b-data.py (native-API run). Co-Authored-By: Claude --- data/benchmark_history.json | 554 +++++++++++++++++++++++++---- outputs/.last_response_schema.json | 21 +- outputs/gemma-4-26b-a4b-data.py | 312 ++++++++++++++++ tools/grade_run.py | 64 ++-- 4 files changed, 842 insertions(+), 109 deletions(-) create mode 100644 outputs/gemma-4-26b-a4b-data.py diff --git a/data/benchmark_history.json b/data/benchmark_history.json index a952868..90450f8 100644 --- a/data/benchmark_history.json +++ b/data/benchmark_history.json @@ -1,6 +1,6 @@ { "meta": { - "project": "Local LLM Benchmark Suite \u2014 LFU Cache & ACID Audit", + "project": "Local LLM Benchmark Suite — LFU Cache & ACID Audit", "machine": "Apple M3 Max, 48GB unified memory, LM Studio", "exam_prompt": "prompts/lfu_cache_prompt.txt", "grading_rubric": "prompts/grading.txt", @@ -65,16 +65,16 @@ "test_integrity": 17 }, "verdict": "Minor Logic Flaws", - "best_for": "Solid daily-driver scaffolding for ACID/async patterns \u2014 produces runnable, well-structured code, but needs a human pass for __slots__, monotonic clocks, and lock granularity before production.", + "best_for": "Solid daily-driver scaffolding for ACID/async patterns — produces runnable, well-structured code, but needs a human pass for __slots__, monotonic clocks, and lock granularity before production.", "critical_bugs": [ - "No __slots__ declared on Node/Transaction/LFUCache \u2014 rubric explicitly required it for memory efficiency.", - "Uses time.time() (system clock) throughout instead of time.monotonic() \u2014 NTP adjustments can cause premature/incorrect TTL eviction.", - "_cleanup_freq_lists() performs a hidden O(F) scan (iterates all freq tiers + min()), called after every eviction, background batch, AND inside commit() \u2014 violates the strict O(1) requirement.", - "Transaction commit holds the single cache lock across all write/delete/bump loops + cleanup \u2014 coarse-grained, blocks all readers for the whole commit window; no fine-grained locking.", + "No __slots__ declared on Node/Transaction/LFUCache — rubric explicitly required it for memory efficiency.", + "Uses time.time() (system clock) throughout instead of time.monotonic() — NTP adjustments can cause premature/incorrect TTL eviction.", + "_cleanup_freq_lists() performs a hidden O(F) scan (iterates all freq tiers + min()), called after every eviction, background batch, AND inside commit() — violates the strict O(1) requirement.", + "Transaction commit holds the single cache lock across all write/delete/bump loops + cleanup — coarse-grained, blocks all readers for the whole commit window; no fine-grained locking.", "Lost-update risk: commit applies tx-local writes without any MVCC/version check, so a key modified by the background evictor or another committer between tx.get() and commit() is overwritten blindly.", "Tests dodge hard cases: 50-task stress uses unique keys with capacity 100, so no eviction-under-contention ever happens; no test for rollback-after-partial-application or mid-commit read isolation." ], - "patch_code": "# FIX 1: Add __slots__ for memory efficiency\n@dataclass\nclass Node:\n __slots__ = ('key', 'value', 'freq', 'expires_at', 'prev', 'next')\n key: Any\n value: Any\n freq: int\n expires_at: Optional[float]\n prev: Optional['Node'] = None\n next: Optional['Node'] = None\n\n# FIX 2: Use monotonic clock everywhere (get/put/_add_node/commit)\n# time.time() -> time.monotonic()\n# e.g.\nexpires_at = time.monotonic() + ttl_seconds if ttl_seconds else None\n\n# FIX 3: Make _cleanup_freq_lists O(1) \u2014 bump min_freq incrementally\n# instead of recomputing min() across all tiers:\n# In _update_freq, when emptying the min_freq bucket, only bump min_freq\n# if you're evicting from it; otherwise leave it. Delete the global\n# min(self.freq_map.keys()) scan. For background sweeps, prune empty\n# buckets lazily on next _evict() rather than scanning proactively.\n\n# FIX 4: Shrink commit critical section \u2014 apply writes into a staging\n# structure under the lock, then release; or use per-bucket locks so\n# readers on unrelated keys aren't blocked.\n\n# FIX 5: Add MVCC version to Node; in commit, raise/abort if the\n# stored version != the version seen at tx.get() time (lost-update detect).", + "patch_code": "# FIX 1: Add __slots__ for memory efficiency\n@dataclass\nclass Node:\n __slots__ = ('key', 'value', 'freq', 'expires_at', 'prev', 'next')\n key: Any\n value: Any\n freq: int\n expires_at: Optional[float]\n prev: Optional['Node'] = None\n next: Optional['Node'] = None\n\n# FIX 2: Use monotonic clock everywhere (get/put/_add_node/commit)\n# time.time() -> time.monotonic()\n# e.g.\nexpires_at = time.monotonic() + ttl_seconds if ttl_seconds else None\n\n# FIX 3: Make _cleanup_freq_lists O(1) — bump min_freq incrementally\n# instead of recomputing min() across all tiers:\n# In _update_freq, when emptying the min_freq bucket, only bump min_freq\n# if you're evicting from it; otherwise leave it. Delete the global\n# min(self.freq_map.keys()) scan. For background sweeps, prune empty\n# buckets lazily on next _evict() rather than scanning proactively.\n\n# FIX 4: Shrink commit critical section — apply writes into a staging\n# structure under the lock, then release; or use per-bucket locks so\n# readers on unrelated keys aren't blocked.\n\n# FIX 5: Add MVCC version to Node; in commit, raise/abort if the\n# stored version != the version seen at tx.get() time (lost-update detect).", "prompt_id": "lfu" }, { @@ -97,17 +97,17 @@ "test_integrity": 4 }, "verdict": "Critical Bugs", - "best_for": "Not recommended for systems code as-is. The 4-bit quant degrades logic sharply vs the 6-bit sibling (82->57) \u2014 usable only for boilerplate/scaffolding drafts that a human will heavily rewrite.", + "best_for": "Not recommended for systems code as-is. The 4-bit quant degrades logic sharply vs the 6-bit sibling (82->57) — usable only for boilerplate/scaffolding drafts that a human will heavily rewrite.", "critical_bugs": [ - "FATAL: _evict() double-removes nodes \u2014 pop() already calls remove() (nulling node.prev/next), then _remove_node() calls remove() AGAIN -> AttributeError: 'NoneType' on first eviction. The cache cannot survive reaching capacity.", + "FATAL: _evict() double-removes nodes — pop() already calls remove() (nulling node.prev/next), then _remove_node() calls remove() AGAIN -> AttributeError: 'NoneType' on first eviction. The cache cannot survive reaching capacity.", "Test suite never executes: test_lfu_eviction crashes at the first eviction, so the 'All tests passed' message is unreachable and assertions are effectively unverified.", - "Background _eviction_loop materializes list(self.key_to_node.keys())[:50] every sweep \u2014 an O(N) linear scan, forbidden by the strict O(1) requirement.", - "_FreqList.pop() has no empty-guard \u2014 calling pop() on an empty list dereferences self.head.next (the dummy tail) and corrupts the DLL.", + "Background _eviction_loop materializes list(self.key_to_node.keys())[:50] every sweep — an O(N) linear scan, forbidden by the strict O(1) requirement.", + "_FreqList.pop() has no empty-guard — calling pop() on an empty list dereferences self.head.next (the dummy tail) and corrupts the DLL.", "Transaction _apply_put applies the buffered value into the EXACT original_node captured at tx.put() time; if the global cache evicted/relocated that node between put and commit, you mutate a stale/dangling node (no MVCC/version check).", "Commit is not atomic across exceptions: a crash mid-_apply loop leaves half-applied global state with no rollback.", - "Uses time.time() (system clock) throughout instead of time.monotonic() \u2014 NTP jumps corrupt TTL eviction." + "Uses time.time() (system clock) throughout instead of time.monotonic() — NTP jumps corrupt TTL eviction." ], - "patch_code": "# FIX 1 (the crash): _evict double-removes. pop() already unlinks,\n# so do NOT call _remove_node on a popped node. Either:\n# (a) pop and then only delete the key_map entry + min_freq bookkeeping:\ndef _evict(self):\n if not self.freq_to_list:\n return\n evict_list = self.freq_to_list[self.min_freq]\n if evict_list.size == 0: # guard against empty\n del self.freq_to_list[self.min_freq]\n return\n node = evict_list.pop() # pop() unlinks + nulls prev/next\n del self.key_to_node[node.key] # DON'T call _remove_node again\n if self.freq_to_list[self.min_freq].size == 0:\n del self.freq_to_list[self.min_freq]\n self.min_freq += 1\n\n# FIX 2: _FreqList.pop empty-guard\ndef pop(self) -> _Node:\n if self.size == 0:\n raise IndexError('pop from empty _FreqList')\n node = self.head.next\n self.remove(node)\n return node\n\n# FIX 3: kill the O(N) scan in background sweep \u2014 maintain a separate\n# set of keys that have a TTL, and iterate that set in batches:\nasync with self.lock:\n batch = list(self._ttl_keys)[:50]\n for k in batch:\n node = self.key_to_node.get(k)\n if node and 0 < node.expires_at <= time.monotonic():\n self._remove_node(node)\n\n# FIX 4: time.time() -> time.monotonic() everywhere.\n# FIX 5: add node.version; in tx._apply_put, abort/refresh if\n# cache.key_to_node[key] is a different node than original_node.", + "patch_code": "# FIX 1 (the crash): _evict double-removes. pop() already unlinks,\n# so do NOT call _remove_node on a popped node. Either:\n# (a) pop and then only delete the key_map entry + min_freq bookkeeping:\ndef _evict(self):\n if not self.freq_to_list:\n return\n evict_list = self.freq_to_list[self.min_freq]\n if evict_list.size == 0: # guard against empty\n del self.freq_to_list[self.min_freq]\n return\n node = evict_list.pop() # pop() unlinks + nulls prev/next\n del self.key_to_node[node.key] # DON'T call _remove_node again\n if self.freq_to_list[self.min_freq].size == 0:\n del self.freq_to_list[self.min_freq]\n self.min_freq += 1\n\n# FIX 2: _FreqList.pop empty-guard\ndef pop(self) -> _Node:\n if self.size == 0:\n raise IndexError('pop from empty _FreqList')\n node = self.head.next\n self.remove(node)\n return node\n\n# FIX 3: kill the O(N) scan in background sweep — maintain a separate\n# set of keys that have a TTL, and iterate that set in batches:\nasync with self.lock:\n batch = list(self._ttl_keys)[:50]\n for k in batch:\n node = self.key_to_node.get(k)\n if node and 0 < node.expires_at <= time.monotonic():\n self._remove_node(node)\n\n# FIX 4: time.time() -> time.monotonic() everywhere.\n# FIX 5: add node.version; in tx._apply_put, abort/refresh if\n# cache.key_to_node[key] is a different node than original_node.", "prompt_id": "lfu" }, { @@ -132,11 +132,11 @@ "verdict": "Critical Bugs", "best_for": "Not recommended for production code. Reasonable API shape and correctly used time.monotonic(), but the module does not parse (syntax error), contains an infinite while:pass loop, and has data races. Avoid for systems/concurrency work.", "critical_bugs": [ - "SyntaxError: line 305 'assert val := await cache.get(...)' is invalid Python \u2014 walrus operator cannot appear in an assert statement. The ENTIRE module fails to compile, so nothing runs and no test can execute.", - "Infinite busy-loop: _evict_lfu lines 159-160 \u2014 'while self.min_freq in self.freq_map and self.min_freq < max(...): pass' has an empty body that never updates min_freq, recomputes max() (O(F)) each iteration, and can never terminate.", - "Hidden O(F) scan: min(self.freq_map.keys()) / max(self.freq_map.keys()) appears at 6 call sites (every eviction and removal) \u2014 violates the strict O(1) requirement.", + "SyntaxError: line 305 'assert val := await cache.get(...)' is invalid Python — walrus operator cannot appear in an assert statement. The ENTIRE module fails to compile, so nothing runs and no test can execute.", + "Infinite busy-loop: _evict_lfu lines 159-160 — 'while self.min_freq in self.freq_map and self.min_freq < max(...): pass' has an empty body that never updates min_freq, recomputes max() (O(F)) each iteration, and can never terminate.", + "Hidden O(F) scan: min(self.freq_map.keys()) / max(self.freq_map.keys()) appears at 6 call sites (every eviction and removal) — violates the strict O(1) requirement.", "Race condition: get() and put() perform lazy-TTL _remove_key() BEFORE acquiring the lock (lines 71-73, 107-108), mutating shared state unlocked while other coroutines read/write.", - "Deadlock risk: background_loop holds self._lock, then calls await self._remove_key() which is itself a lock-acquiring method \u2014 asyncio.Lock is NOT reentrant -> deadlock when the evictor runs.", + "Deadlock risk: background_loop holds self._lock, then calls await self._remove_key() which is itself a lock-acquiring method — asyncio.Lock is NOT reentrant -> deadlock when the evictor runs.", "No __slots__ on _Node despite using a dataclass (rubric required it for memory efficiency).", "_remove_key will KeyError on self.freq_map[freq] if a concurrent operation already deleted that bucket." ], @@ -165,14 +165,14 @@ "verdict": "Critical Bugs", "best_for": "Promising code-design instincts (cleanest abstractions and best transaction isolation design in the set) but undone by a single fatal one-line bug that stops it running. With the bug fixed it would likely score 80+; as-is, only useful as a structural reference.", "critical_bugs": [ - "FATAL: _put_internal line 305 inserts a NEW key with 'self._freq_map[1].push_front(...)' but never ensures the freq-1 bucket exists \u2014 KeyError: 1 on the very first put. The cache cannot store a single key. The _ensure_freq_list(1) helper it should use exists and is used correctly everywhere else (lines 294, 354).", - "Transaction.commit() calls _put_internal for buffered writes, so it hits the same KeyError: 1 \u2014 committed transactions crash too.", + "FATAL: _put_internal line 305 inserts a NEW key with 'self._freq_map[1].push_front(...)' but never ensures the freq-1 bucket exists — KeyError: 1 on the very first put. The cache cannot store a single key. The _ensure_freq_list(1) helper it should use exists and is used correctly everywhere else (lines 294, 354).", + "Transaction.commit() calls _put_internal for buffered writes, so it hits the same KeyError: 1 — committed transactions crash too.", "Test suite cannot execute: crashes at the first cache.put() in main(); the well-built test harness (pass/fail counter, 4 real scenarios) validates nothing.", "No __slots__ on _DLLNode/_CacheNode/_DoublyLinkedList despite the rubric requiring it for memory efficiency.", - "_evict_node uses min(self._freq_map) (line 325) when the min-tier empties \u2014 a hidden O(F) scan, violating strict O(1).", + "_evict_node uses min(self._freq_map) (line 325) when the min-tier empties — a hidden O(F) scan, violating strict O(1).", "No MVCC/version check on transaction commit (lost-update possible if the global key is modified between tx.get and commit); commit is not exception-safe across the writes-vs-deletes loops." ], - "patch_code": "# FIX 1 (the fatal one-liner): use the helper that already exists.\n# line 305, in _put_internal, new-key branch:\n- self._freq_map[1].push_front(dll_node)\n+ self._ensure_freq_list(1).push_front(dll_node)\n# (This single change makes the cache and transactions functional.)\n\n# FIX 2: replace the O(F) min() scan with an incremental bump:\n# in _evict_node, when the min-tier bucket empties, min_freq is the\n# lowest remaining tier. Since freq only ever increments by 1, the\n# next min is almost always min_freq+1; track it incrementally rather\n# than scanning. Or, since this only happens on full eviction, accept\n# O(F) but only on the empty-cache edge \u2014 document it.\n\n# FIX 3: add __slots__ to all internal classes:\nclass _CacheNode:\n __slots__ = ('key','value','ttl_seconds','expiry_time','freq','dll_node')\n ...\n\n# FIX 4: wrap commit applies in try/except so a mid-commit exception\n# does not leave a half-applied global state; consider abort semantics.\n# FIX 5: add node.version; in tx commit, abort if the global node for\n# a key is not the one seen at tx.get() time.", + "patch_code": "# FIX 1 (the fatal one-liner): use the helper that already exists.\n# line 305, in _put_internal, new-key branch:\n- self._freq_map[1].push_front(dll_node)\n+ self._ensure_freq_list(1).push_front(dll_node)\n# (This single change makes the cache and transactions functional.)\n\n# FIX 2: replace the O(F) min() scan with an incremental bump:\n# in _evict_node, when the min-tier bucket empties, min_freq is the\n# lowest remaining tier. Since freq only ever increments by 1, the\n# next min is almost always min_freq+1; track it incrementally rather\n# than scanning. Or, since this only happens on full eviction, accept\n# O(F) but only on the empty-cache edge — document it.\n\n# FIX 3: add __slots__ to all internal classes:\nclass _CacheNode:\n __slots__ = ('key','value','ttl_seconds','expiry_time','freq','dll_node')\n ...\n\n# FIX 4: wrap commit applies in try/except so a mid-commit exception\n# does not leave a half-applied global state; consider abort semantics.\n# FIX 5: add node.version; in tx commit, abort if the global node for\n# a key is not the one seen at tx.get() time.", "prompt_id": "lfu" }, { @@ -184,7 +184,7 @@ "tok_sec": 10.09, "total_tokens": 4536, "ttft_sec": 4.39, - "speed_caveat": "All Gemma 4 models ran abnormally slow (GPU offload appeared inactive despite being set), so tok/sec and TTFT are NOT representative of the model itself \u2014 likely an LM Studio/GGUF config issue. Treat speed numbers for the Gemma 4 batch as suspect.", + "speed_caveat": "All Gemma 4 models ran abnormally slow (GPU offload appeared inactive despite being set), so tok/sec and TTFT are NOT representative of the model itself — likely an LM Studio/GGUF config issue. Treat speed numbers for the Gemma 4 batch as suspect.", "filename": "outputs/gemma4-31b-gguf.py", "tests_pass": true, "total_score": 78, @@ -198,11 +198,11 @@ "verdict": "Minor Logic Flaws", "best_for": "Clean, correct, runnable code with solid O(1) structure and good concurrency granularity. A reliable pick for everyday caching/async work after a monotonic-clock + __slots__ pass.", "critical_bugs": [ - "Isolation leak: Transaction.get falls back to the PUBLIC cache.get, which calls _update_frequency \u2014 so reading a key inside a transaction mutates GLOBAL frequency state before commit, leaking uncommitted access patterns into global eviction order. Spec requires tx reads not to alter global freq.", - "Uses time.time() (system clock) throughout instead of time.monotonic() \u2014 NTP adjustments corrupt TTL eviction.", - "No __slots__ on Node/DoublyLinkedList/LFUCache/Transaction \u2014 rubric required it for memory efficiency.", + "Isolation leak: Transaction.get falls back to the PUBLIC cache.get, which calls _update_frequency — so reading a key inside a transaction mutates GLOBAL frequency state before commit, leaking uncommitted access patterns into global eviction order. Spec requires tx reads not to alter global freq.", + "Uses time.time() (system clock) throughout instead of time.monotonic() — NTP adjustments corrupt TTL eviction.", + "No __slots__ on Node/DoublyLinkedList/LFUCache/Transaction — rubric required it for memory efficiency.", "_delete_internal deliberately leaves empty frequency buckets in freq_map (documented but a minor memory leak: empty DoublyLinkedList objects accumulate).", - "Background evictor does list(self.cache.keys()) = O(N) snapshot every interval \u2014 a linear scan, forbidden by strict O(1).", + "Background evictor does list(self.cache.keys()) = O(N) snapshot every interval — a linear scan, forbidden by strict O(1).", "No MVCC/version check on commit (lost-update possible); commit is not exception-safe across the deletes-vs-puts loops.", "Tests pass but don't probe mid-commit read isolation or eviction-under-real-contention (capacity sized so all keys fit), so the isolation leak above goes undetected." ], @@ -233,10 +233,10 @@ "best_for": "Runnable and structurally sound, but the LFU eviction has a stale-min_freq capacity-breach path and the transaction API doesn't match the spec (no commit/rollback). Usable for prototypes if you fix eviction and re-skin transactions.", "critical_bugs": [ "Capacity breach: eviction does self.freq_map[self.min_freq].pop_tail() with NO guard that the bucket exists or is non-empty, and never prunes emptied freq buckets. After manual deletes empty the min-tier, pop_tail returns None silently -> eviction fails -> cache grows PAST capacity. This is the exact stale-min_freq capacity-breach the rubric flags.", - "Non-conformant transaction API: Transaction has NO commit() or rollback() method (both required by spec). Commit happens via a separate cache.apply_transaction_changes(tx._state) \u2014 wrong surface; the test only passes because it uses this internal path.", - "Isolation leak: Transaction.get falls back to the public cache.get, which bumps global frequency before commit \u2014 uncommitted tx reads alter global eviction order.", + "Non-conformant transaction API: Transaction has NO commit() or rollback() method (both required by spec). Commit happens via a separate cache.apply_transaction_changes(tx._state) — wrong surface; the test only passes because it uses this internal path.", + "Isolation leak: Transaction.get falls back to the public cache.get, which bumps global frequency before commit — uncommitted tx reads alter global eviction order.", "Duplicated eviction logic in apply_transaction_changes re-introduces the stale-min_freq bug at line 211.", - "Uses time.time() (system clock) via _get_now() instead of time.monotonic() \u2014 NTP jumps corrupt TTL (though _get_now is a clean single fix point).", + "Uses time.time() (system clock) via _get_now() instead of time.monotonic() — NTP jumps corrupt TTL (though _get_now is a clean single fix point).", "No __slots__ on Node/DoublyLinkedList/LFUCache/TransactionState/Transaction.", "Background sweep does list(self.cache.keys()) = O(N) per interval.", "Tests pass but use the non-spec commit path and don't probe capacity breach or isolation leak." @@ -264,15 +264,15 @@ "test_integrity": 4 }, "verdict": "Critical Bugs", - "best_for": "Not usable as-is \u2014 the cache cannot store its first key and the background evictor would crash the event loop. The smallest model in the set (12B) and lowest-quality output. Avoid for systems work.", + "best_for": "Not usable as-is — the cache cannot store its first key and the background evictor would crash the event loop. The smallest model in the set (12B) and lowest-quality output. Avoid for systems work.", "critical_bugs": [ "FATAL: put() line 112 does 'bucket = self.freq_buckets[self.min_freq]' after setting min_freq=1 but NEVER creates freq_buckets[1] -> KeyError: 1 on the very first put. Cache is unusable.", "Background evictor is fundamentally broken: start_evictor defines a SYNC 'def evict_loop' and passes it to create_task; inside it calls blocking time.sleep(interval) (freezes the event loop) AND asyncio.run(...) from within a running loop -> RuntimeError. Would crash hard if ever reached.", - "Eviction-by-re-put: expired keys are 'evicted' by re-inserting them with TTL 0 (line 122) instead of deleting them \u2014 wrong semantics and triggers immediate re-eviction.", + "Eviction-by-re-put: expired keys are 'evicted' by re-inserting them with TTL 0 (line 122) instead of deleting them — wrong semantics and triggers immediate re-eviction.", "Transaction duplicates the entire LFU machinery (local_cache + local_freq + local_min_freq) for snapshot isolation, but _update_local_freq has the same missing-bucket KeyError (line 140).", "Class name typo 'DoublyLinkedListList' (doubled word).", "No __slots__; time.time() (not monotonic) throughout.", - "Tests cannot run \u2014 crash at first put." + "Tests cannot run — crash at first put." ], "patch_code": "# FIX 1 (the fatal KeyError): create the bucket before use.\n# In put(), new-key branch:\n- bucket = self.freq_buckets[self.min_freq]\n+ bucket = self.freq_buckets.setdefault(self.min_freq, DoublyLinkedListList())\n# Same fix in _update_freq and Transaction._update_local_freq (use setdefault).\n\n# FIX 2 (the broken evictor): make it a real async task that deletes:\nasync def _evict_loop(self, interval):\n while True:\n await asyncio.sleep(interval) # async, non-blocking\n now = time.monotonic()\n async with self.global_lock:\n expired = [k for k, n in list(self.cache.items()) if now > n.ttl_expiry]\n for k in expired:\n node = self.cache.pop(k, None)\n if node:\n self.freq_buckets[node.freq].remove(node) # DELETE, not re-put\n\nasync def start_evictor(self, interval=1.0):\n self.evictor_task = asyncio.create_task(self._evict_loop(interval))\n\n# FIX 3: delete expired keys; do NOT re-insert with TTL 0.\n# FIX 4: time.time() -> time.monotonic().\n# FIX 5: add __slots__ to Node / DoublyLinkedListList / ConcurrentLFUCache / Transaction.", "prompt_id": "lfu" @@ -286,7 +286,7 @@ "tok_sec": null, "total_tokens": null, "ttft_sec": null, - "speed_caveat": "Cloud model (run via opencode, not LM Studio) \u2014 tok_sec/tokens/TTFT are N/A (not measured for cloud). Included as a quality baseline against the local models. NOTE: it took 3 attempts to produce any output and ~12 minutes of thinking before succeeding \u2014 so it is a QUALITY benchmark, not a speed/usability one.", + "speed_caveat": "Cloud model (run via opencode, not LM Studio) — tok_sec/tokens/TTFT are N/A (not measured for cloud). Included as a quality baseline against the local models. NOTE: it took 3 attempts to produce any output and ~12 minutes of thinking before succeeding — so it is a QUALITY benchmark, not a speed/usability one.", "filename": "deepseekv4flash.py", "tests_pass": true, "total_score": 91, @@ -298,10 +298,10 @@ "test_integrity": 18 }, "verdict": "Production Ready", - "best_for": "Reference-quality baseline (91/100) \u2014 the bar the local models are measured against. Only submission with __slots__ + time.monotonic() + delta-based transactional frequency accounting. Sets the ceiling for correctness, though its unreliability (3 attempts, 12-min think time) makes it a poor *local* daily-driver.", + "best_for": "Reference-quality baseline (91/100) — the bar the local models are measured against. Only submission with __slots__ + time.monotonic() + delta-based transactional frequency accounting. Sets the ceiling for correctness, though its unreliability (3 attempts, 12-min think time) makes it a poor *local* daily-driver.", "critical_bugs": [ - "Two min(self._freq_to_list) linear scans in _evict_one's defensive recovery path (lines 415, 429) \u2014 only triggered when min_freq desyncs, not per-operation, but still a non-O(1) path. Could be replaced with incremental tracking.", - "Single coarse lock held across the whole commit-apply loop \u2014 not the fine-grained locking the prompt asked for.", + "Two min(self._freq_to_list) linear scans in _evict_one's defensive recovery path (lines 415, 429) — only triggered when min_freq desyncs, not per-operation, but still a non-O(1) path. Could be replaced with incremental tracking.", + "Single coarse lock held across the whole commit-apply loop — not the fine-grained locking the prompt asked for.", "__slots__ present on _Node and _DLL but not extended to Transaction / LFUCache.", "No explicit lost-update/conflict abort on commit (delta-based freq is applied unconditionally).", "Tests pass 20/20 but don't include a mid-commit read-isolation probe or adversarial eviction-under-contention stress." @@ -332,12 +332,12 @@ "verdict": "Minor Logic Flaws", "best_for": "Tied top local scorer (82). The only local model to use delta-based transactional frequency accounting (freq bumps deferred to commit), matching the cloud baseline's isolation approach. Reliable for async/ACID-pattern work after a __slots__ + min_freq-edge pass.", "critical_bugs": [ - "Stale min_freq on manual delete: _remove_node_from_structures empties the min-freq bucket but does `pass` instead of recomputing min_freq (lines 199-202, documented as 'a simplification'). Correct only because eviction has a min_freq-in-freq_map guard + arbitrary-key fallback (lines 237-244) \u2014 latent fragility under concurrent deletes.", + "Stale min_freq on manual delete: _remove_node_from_structures empties the min-freq bucket but does `pass` instead of recomputing min_freq (lines 199-202, documented as 'a simplification'). Correct only because eviction has a min_freq-in-freq_map guard + arbitrary-key fallback (lines 237-244) — latent fragility under concurrent deletes.", "No __slots__ on Node/DoublyLinkedList/LFUCache/Transaction despite Generic dataclasses (rubric required it for memory efficiency).", - "Background evictor does list(self.cache_data.keys()) = O(N) snapshot every interval \u2014 a linear scan, forbidden by strict O(1).", + "Background evictor does list(self.cache_data.keys()) = O(N) snapshot every interval — a linear scan, forbidden by strict O(1).", "No MVCC/version check on commit (lost-update possible if the global key changes between tx.get and commit); commit applies deletes->reads->puts without try/except, so a mid-commit exception leaves partial state." ], - "patch_code": "# FIX 1 (stale min_freq): recompute or invalidate when the min bucket empties.\n# In _remove_node_from_structures, replace the `pass`:\nif dll.size == 0:\n del self.freq_map[node.freq]\n if self.min_freq == node.freq:\n # bump to next existing tier (frequencies are contiguous under normal use)\n self.min_freq = self.min_freq + 1 if (self.min_freq + 1) in self.freq_map else min(self.freq_map, default=1)\n\n# FIX 2: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 3: background evictor \u2014 maintain a _ttl_keys set and iterate IT in\n# batches instead of list(self.cache_data.keys()) to stay O(batch).\n# FIX 4: wrap commit's three loops in try/except with rollback semantics on failure.", + "patch_code": "# FIX 1 (stale min_freq): recompute or invalidate when the min bucket empties.\n# In _remove_node_from_structures, replace the `pass`:\nif dll.size == 0:\n del self.freq_map[node.freq]\n if self.min_freq == node.freq:\n # bump to next existing tier (frequencies are contiguous under normal use)\n self.min_freq = self.min_freq + 1 if (self.min_freq + 1) in self.freq_map else min(self.freq_map, default=1)\n\n# FIX 2: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 3: background evictor — maintain a _ttl_keys set and iterate IT in\n# batches instead of list(self.cache_data.keys()) to stay O(batch).\n# FIX 4: wrap commit's three loops in try/except with rollback semantics on failure.", "prompt_id": "lfu" }, { @@ -350,7 +350,7 @@ "tok_sec": 12.29, "total_tokens": 12790, "ttft_sec": 2.9, - "speed_caveat": "This model 'thought' for 12m48s before producing output and ran at 12.29 tok/sec \u2014 anomalously slow for an 8-bit MLX on M3 Max. Likely an inference/quant issue worth investigating; the slow generation is NOT representative of normal 27B-8bit throughput.", + "speed_caveat": "This model 'thought' for 12m48s before producing output and ran at 12.29 tok/sec — anomalously slow for an 8-bit MLX on M3 Max. Likely an inference/quant issue worth investigating; the slow generation is NOT representative of normal 27B-8bit throughput.", "filename": "outputs/qwen3.6-27b-8bit-mlx.py", "tests_pass": true, "total_score": 78, @@ -362,12 +362,12 @@ "test_integrity": 16 }, "verdict": "Minor Logic Flaws", - "best_for": "Clean, correct, runnable \u2014 same tier as Gemma 4 31B (78). Good O(1) structure and concurrency granularity. Reliable for everyday async/caching work after a monotonic-clock + __slots__ + isolation pass. Caveat: was anomalously slow to generate.", + "best_for": "Clean, correct, runnable — same tier as Gemma 4 31B (78). Good O(1) structure and concurrency granularity. Reliable for everyday async/caching work after a monotonic-clock + __slots__ + isolation pass. Caveat: was anomalously slow to generate.", "critical_bugs": [ - "Isolation leak: Transaction.get falls back to the PUBLIC cache.get (line 78), which calls _update_freq \u2014 so reading a key inside a transaction mutates GLOBAL frequency state before commit, leaking uncommitted access patterns into global eviction order.", - "Uses time.time() (system clock) throughout instead of time.monotonic() \u2014 NTP adjustments corrupt TTL eviction.", - "No __slots__ on Node/DoublyLinkedList/LFUCache/Transaction \u2014 rubric required it for memory efficiency.", - "Background _evict_loop materializes list(self.nodes.keys()) (O(N)) before checking only batch_size keys \u2014 the break caps work but not the snapshot cost, a hidden O(N) per sweep.", + "Isolation leak: Transaction.get falls back to the PUBLIC cache.get (line 78), which calls _update_freq — so reading a key inside a transaction mutates GLOBAL frequency state before commit, leaking uncommitted access patterns into global eviction order.", + "Uses time.time() (system clock) throughout instead of time.monotonic() — NTP adjustments corrupt TTL eviction.", + "No __slots__ on Node/DoublyLinkedList/LFUCache/Transaction — rubric required it for memory efficiency.", + "Background _evict_loop materializes list(self.nodes.keys()) (O(N)) before checking only batch_size keys — the break caps work but not the snapshot cost, a hidden O(N) per sweep.", "Commit re-implements put+evict inline (lines 102-118) duplicating the public path = duplicated bug surface; not wrapped in try/except so a mid-commit exception leaves partial state. No MVCC version check (lost-update possible)." ], "patch_code": "# FIX 1 (isolation leak): add a read-only global lookup (no freq bump)\n# and use it in tx.get instead of the public cache.get:\nasync def _read_raw(self, key):\n async with self.lock:\n node = self.nodes.get(key)\n if node is None: return None\n if time.monotonic() > node.expires_at:\n self._remove_node(key); return None\n return node.value\n# then: return await self._cache._read_raw(key)\n# FIX 2: time.time() -> time.monotonic() everywhere.\n# FIX 3: add __slots__ to Node, DoublyLinkedList, LFUCache, Transaction.\n# FIX 4: maintain a _ttl_keys set; iterate IT (batched) in the bg loop\n# instead of list(self.nodes.keys()).\n# FIX 5: factor commit's put/evict to reuse the internal helpers; wrap\n# the commit loop in try/except with rollback-on-failure.", @@ -394,16 +394,16 @@ "test_integrity": 4 }, "verdict": "Critical Bugs", - "best_for": "Not usable as-is \u2014 transactions crash immediately due to an async/sync lock mismatch. The coder-specialist produced terse, fast output (2779 tok, 72.7 t/s) with competent freq-bucket structure, but fumbled the async primitive. Fix the one lock bug and it would likely score 70+.", + "best_for": "Not usable as-is — transactions crash immediately due to an async/sync lock mismatch. The coder-specialist produced terse, fast output (2779 tok, 72.7 t/s) with competent freq-bucket structure, but fumbled the async primitive. Fix the one lock bug and it would likely score 70+.", "critical_bugs": [ "FATAL: _transaction_lock is an asyncio.Lock() (line 101) but begin_transaction() is a SYNC def that uses synchronous `with self._transaction_lock:` (line 199). asyncio.Lock does not support the sync context-manager protocol -> TypeError on the first transaction, crashing the entire test suite.", "Test suite cannot run: crashes at cache.begin_transaction() in main(); the transaction and concurrency assertions never execute.", - "Unused `import threading` and `import weakref` \u2014 vestigial confusion between threading and asyncio primitives.", - "Uses time.time() (system clock) throughout instead of time.monotonic() \u2014 NTP jumps corrupt TTL eviction.", - "No __slots__ on CacheNode/FrequencyBucket/InMemoryLFUCache/Transaction \u2014 rubric required it.", - "FrequencyBucket stores nodes in BOTH a DLL and a parallel `nodes` dict \u2014 redundant memory per bucket." + "Unused `import threading` and `import weakref` — vestigial confusion between threading and asyncio primitives.", + "Uses time.time() (system clock) throughout instead of time.monotonic() — NTP jumps corrupt TTL eviction.", + "No __slots__ on CacheNode/FrequencyBucket/InMemoryLFUCache/Transaction — rubric required it.", + "FrequencyBucket stores nodes in BOTH a DLL and a parallel `nodes` dict — redundant memory per bucket." ], - "patch_code": "# FIX 1 (the fatal crash): make begin_transaction async and use async with.\nasync def begin_transaction(self) -> 'Transaction':\n async with self._transaction_lock:\n self._transaction_counter += 1\n tx = Transaction(self)\n self._transactions[self._transaction_counter] = tx\n return tx\n# (and update callers: `tx = await cache.begin_transaction()`)\n# Alternative if sync creation is required: use threading.Lock for the\n# counter, but that is wrong in an asyncio codebase \u2014 go async.\n\n# FIX 2: remove unused `import threading` and `import weakref`.\n# FIX 3: time.time() -> time.monotonic() everywhere.\n# FIX 4: add __slots__ to all classes.\n# FIX 5: drop the redundant FrequencyBucket.nodes dict; the DLL already\n# tracks membership, so the dict is duplicate storage.", + "patch_code": "# FIX 1 (the fatal crash): make begin_transaction async and use async with.\nasync def begin_transaction(self) -> 'Transaction':\n async with self._transaction_lock:\n self._transaction_counter += 1\n tx = Transaction(self)\n self._transactions[self._transaction_counter] = tx\n return tx\n# (and update callers: `tx = await cache.begin_transaction()`)\n# Alternative if sync creation is required: use threading.Lock for the\n# counter, but that is wrong in an asyncio codebase — go async.\n\n# FIX 2: remove unused `import threading` and `import weakref`.\n# FIX 3: time.time() -> time.monotonic() everywhere.\n# FIX 4: add __slots__ to all classes.\n# FIX 5: drop the redundant FrequencyBucket.nodes dict; the DLL already\n# tracks membership, so the dict is duplicate storage.", "prompt_id": "lfu" }, { @@ -431,12 +431,12 @@ "best_for": "NOT a safe offload for async-pipeline / queue work despite scoring 82 on the LFU exam. Same model+quant, different prompt: 82 -> 49. Use it for data-structure/ACID tasks; route queue/backpressure/retry work elsewhere (or to the cloud model).", "critical_bugs": [ "FATAL: file does not parse. 'async with self._callbacks_lock' (L103) and 'await coro' (L98) are inside on_event, a SYNC def. SyntaxError before any test runs.", - "Bounded-concurrency design is broken: only ONE worker task is created (start() does a single create_task), and the Semaphore(4) is acquired inside that single loop. Effective concurrency is 1, not 4 \u2014 the semaphore is decorative.", + "Bounded-concurrency design is broken: only ONE worker task is created (start() does a single create_task), and the Semaphore(4) is acquired inside that single loop. Effective concurrency is 1, not 4 — the semaphore is decorative.", "Test A asserts max_concurrency <= 4, but since real concurrency is ~1 it passes trivially and proves NOTHING about the cap actually holding under saturation.", "Cooperative cancel only sets a flag; it cannot interrupt an in-flight mock_synthesize mid-sleep (acceptable, but not 'cancel at next safe checkpoint' for a long synth).", - "submit() after stop() silently enqueues to a dead worker \u2014 no rejection, jobs leak (never processed, never failed).", - "_final_states dict grows unbounded (no eviction) \u2014 memory leak for a long-running service.", - "Callback errors are silently swallowed (broad try/except Exception) \u2014 a silent-failure pattern; debug visibility lost." + "submit() after stop() silently enqueues to a dead worker — no rejection, jobs leak (never processed, never failed).", + "_final_states dict grows unbounded (no eviction) — memory leak for a long-running service.", + "Callback errors are silently swallowed (broad try/except Exception) — a silent-failure pattern; debug visibility lost." ], "patch_code": "# FIX 1 (the parse error): make on_event async (or don't await inside it).\n# Simplest correct version:\nasync def on_event(self, callback):\n async with self._callbacks_lock:\n self._callbacks.append(callback)\n\n# FIX 2 (real bounded concurrency): spawn N workers OR create_task per job\n# gated by the semaphore. Option B (concurrency from the semaphore itself):\nasync def _worker_loop(self):\n while self._running:\n job = await self._queue.get()\n # do NOT hold the semaphore in the single worker; instead launch\n # each job as its own task, gated so at most max_concurrency run:\n async def _run(j):\n async with self._semaphore:\n await self._process_job(j)\n self._queue.task_done()\n asyncio.create_task(_run(job))\n# (and add a test that submits >max_concurrency long jobs and asserts\n# exactly max_concurrency run at once.)\n\n# FIX 3: reject submit() after stop() (guard on self._running).\n# FIX 4: bound _final_states (e.g. keep last N, or evict terminal >TTL).\n# FIX 5: log callback errors instead of swallowing them silently." }, @@ -463,13 +463,13 @@ "test_integrity": 6 }, "verdict": "Critical Bugs", - "best_for": "NOT usable for Rust as-is \u2014 7 compile errors. Same model that scored 82 on LFU now: TTS 49, Rust 50. The profile is now clear: Qwen 6-bit is strong on single-file Python data-structure work but repeatedly ships non-compiling/non-parsing code on multi-task async + typed-language prompts. Keep it on Python ACID/data-structure tasks; do NOT offload Rust or async-pipeline work to it.", + "best_for": "NOT usable for Rust as-is — 7 compile errors. Same model that scored 82 on LFU now: TTS 49, Rust 50. The profile is now clear: Qwen 6-bit is strong on single-file Python data-structure work but repeatedly ships non-compiling/non-parsing code on multi-task async + typed-language prompts. Keep it on Python ACID/data-structure tasks; do NOT offload Rust or async-pipeline work to it.", "critical_bugs": [ - "FATAL: does not compile \u2014 7 errors (verified with cargo 1.94). Key ones: mpsc::bounded(32) (tokio has no bounded(); should be mpsc::channel(32) \u2014 an async-std/flume API hallucination); no `main` fn (lib-style file, won't 'run directly' as the prompt required); u32/u64 type mismatch in Duration::from_millis(20 + id % 30).", + "FATAL: does not compile — 7 errors (verified with cargo 1.94). Key ones: mpsc::bounded(32) (tokio has no bounded(); should be mpsc::channel(32) — an async-std/flume API hallucination); no `main` fn (lib-style file, won't 'run directly' as the prompt required); u32/u64 type mismatch in Duration::from_millis(20 + id % 30).", "Ownership errors (would not compile): shutdown() tries to move e.join_handle out of &WatcherEntry, and drop(self.item_tx)/drop(self.health_tx) out of &self. Needs Option + &mut self or mem::take.", - "remove_watcher is broken: removes the HashMap entry but NEVER cancels that watcher's task. The orphaned task keeps polling until global shutdown() \u2014 contradicting the 'clean per-watcher removal' requirement.", - "Test (b) failure-injection is DEAD CODE: a global TEST_ALWAYS_FAIL flag is toggled in the test, but watcher_loop calls mock_fetch directly, not the test_fetch override that reads the flag. So the unhealthy-marking test relies on the natural ~15% failure rate over 400ms \u2014 flaky, may never reach 5 consecutive failures.", - "Test (a) asserts nothing real \u2014 the comment admits 'we trust the join'; prompt required asserting >0 items received and no hang.", + "remove_watcher is broken: removes the HashMap entry but NEVER cancels that watcher's task. The orphaned task keeps polling until global shutdown() — contradicting the 'clean per-watcher removal' requirement.", + "Test (b) failure-injection is DEAD CODE: a global TEST_ALWAYS_FAIL flag is toggled in the test, but watcher_loop calls mock_fetch directly, not the test_fetch override that reads the flag. So the unhealthy-marking test relies on the natural ~15% failure rate over 400ms — flaky, may never reach 5 consecutive failures.", + "Test (a) asserts nothing real — the comment admits 'we trust the join'; prompt required asserting >0 items received and no hang.", "mock_fetch randomness uses SystemTime nanos + id; across watchers polled in the same tick the high bits are shared, so failure/item counts cluster (poor randomness, not truly independent).", "Several `let _ = tx.send(...)` silently swallow channel-closed errors." ], @@ -500,12 +500,12 @@ "verdict": "Minor Logic Flaws", "best_for": "Best non-LFU result for this model (75 vs TTS 49, Rust 50). Runs clean, passes all 4 tests, implements HMAC + idempotency + token-bucket rate-limit + 429 backoff correctly. Safe to offload single-handler HTTP/bridge logic (webhooks, signature verification, rate-limited forwarding). AVOID for multi-task orchestration (TTS) and typed/compiled languages (Rust).", "critical_bugs": [ - "Clock inconsistency: IdempotencyStore uses time.time() (system clock) while TokenBucketLimiter uses time.monotonic() \u2014 an NTP jump could wrongly expire/replay events. Should be monotonic everywhere.", - "No max-body cap: handle_client does reader.readexactly(content_length) with no limit \u2014 a hostile Content-Length could force a huge allocation (DoS). The rate limiter doesn't protect pre-parse.", - "forward_timestamps list grows unbounded (append-only, only cleared in tests) \u2014 memory leak for a long-running service.", - "HTTP reason phrase is the raw message string (HTTP/1.1 200 Forwarded) \u2014 works for the bundled test client but is not valid HTTP for real clients/proxies.", + "Clock inconsistency: IdempotencyStore uses time.time() (system clock) while TokenBucketLimiter uses time.monotonic() — an NTP jump could wrongly expire/replay events. Should be monotonic everywhere.", + "No max-body cap: handle_client does reader.readexactly(content_length) with no limit — a hostile Content-Length could force a huge allocation (DoS). The rate limiter doesn't protect pre-parse.", + "forward_timestamps list grows unbounded (append-only, only cleared in tests) — memory leak for a long-running service.", + "HTTP reason phrase is the raw message string (HTTP/1.1 200 Forwarded) — works for the bundled test client but is not valid HTTP for real clients/proxies.", "No Content-Type validation on incoming requests (accepts any).", - "Idempotency eviction is lazy (only on is_seen) \u2014 a quiet store retains stale entries until next access; not a leak in steady state but imperfect.", + "Idempotency eviction is lazy (only on is_seen) — a quiet store retains stale entries until next access; not a leak in steady state but imperfect.", "Tests are mildly timing-flaky: 5% random 429 in discord_send + a tight '>1.0s' threshold; no tests for the 400 (bad JSON) or missing-signature 401 paths even though the code handles them." ], "patch_code": "# FIX 1 (clock): use monotonic for TTL too.\nclass IdempotencyStore:\n def is_seen(self, event_id):\n now = time.monotonic()\n ...\n def mark(self, event_id):\n self.store[event_id] = time.monotonic()\n\n# FIX 2 (body cap): reject oversized bodies before reading.\nMAX_BODY = 64 * 1024\ncontent_length = int(headers.get('content-length', 0))\nif content_length > MAX_BODY:\n writer.write(b'HTTP/1.1 413 Payload Too Large\\r\\nContent-Length: 0\\r\\n\\r\\n'); await writer.drain(); return\nbody = await reader.readexactly(content_length) if 0 < content_length <= MAX_BODY else b''\n\n# FIX 3 (leak): bound forward_timestamps (deque maxlen=N) or drop it if unused.\nfrom collections import deque\nself.forward_timestamps: deque = deque(maxlen=1000)\n\n# FIX 4 (reason phrase): use a fixed map.\nREASON = {200:'OK',400:'Bad Request',401:'Unauthorized',404:'Not Found',502:'Bad Gateway',500:'Internal Server Error'}\nresponse = f'HTTP/1.1 {status} {REASON.get(status,\"OK\")}\\r\\n...'\n\n# FIX 5: add tests for the 400 (malformed JSON) and missing-signature 401 paths." @@ -534,12 +534,12 @@ "test_integrity": 15 }, "verdict": "Minor Logic Flaws", - "best_for": "DECISIVE: passes the TTS pipeline that Qwen 6-bit couldn't even parse. Uses REAL bounded concurrency (N worker tasks + Semaphore) \u2014 exactly Qwen's fatal flaw avoided. Gemma generalizes to multi-task orchestration where Qwen fails. Strong offload candidate for queue/pipeline work.", + "best_for": "DECISIVE: passes the TTS pipeline that Qwen 6-bit couldn't even parse. Uses REAL bounded concurrency (N worker tasks + Semaphore) — exactly Qwen's fatal flaw avoided. Gemma generalizes to multi-task orchestration where Qwen fails. Strong offload candidate for queue/pipeline work.", "critical_bugs": [ - "LM Studio API TTFT not captured by grade_run.py (key-name mismatch) \u2014 needs a fix to the script.", + "LM Studio API TTFT not captured by grade_run.py (key-name mismatch) — needs a fix to the script.", "Detail audit pending a full read, but tests pass and the concurrency model is correct (N create_task workers + Semaphore, proper drain via queue.join + running-count, cancel + gather)." ], - "patch_code": "# TTFT capture: LM Studio returns timing under 'stats' or 'timings' with keys like\n# 'time_to_first_token' / 'prompt_progress' \u2014 grade_run.py should dump resp['stats'] raw\n# once to find the right key, then parse it." + "patch_code": "# TTFT capture: LM Studio returns timing under 'stats' or 'timings' with keys like\n# 'time_to_first_token' / 'prompt_progress' — grade_run.py should dump resp['stats'] raw\n# once to find the right key, then parse it." }, { "id": "gemma4-26b-a4b-8bit-mlx-rust", @@ -567,8 +567,8 @@ "verdict": "Minor Logic Flaws", "best_for": "DECISIVE: Rust COMPILES CLEAN (0 errors with deps declared) where Qwen had 7 real compile errors (mpsc::bounded hallucination, ownership moves). The code is structurally sound Rust. Deductions only for undeclared deps + markdown wrapping. Gemma is the better Rust offload pick by a wide margin.", "critical_bugs": [ - "Uses tokio-util and rand crates WITHOUT declaring them in the dependency block (prompt said assume tokio/serde/thiserror only). Compiles clean once added \u2014 so it's a deps-list omission, not a code bug.", - "Wrapped output in markdown fences (```toml and ```rust) \u2014 required extraction to get runnable code. A submission-hygiene issue, not a logic one.", + "Uses tokio-util and rand crates WITHOUT declaring them in the dependency block (prompt said assume tokio/serde/thiserror only). Compiles clean once added — so it's a deps-list omission, not a code bug.", + "Wrapped output in markdown fences (```toml and ```rust) — required extraction to get runnable code. A submission-hygiene issue, not a logic one.", "Full logic audit pending; 0 compile errors with deps declared is the headline." ], "patch_code": "# FIX 1 (deps): add to Cargo.toml:\n# tokio-util = \"0.7\"\n# rand = \"0.8\"\n# FIX 2: grade_run.py extractor now prefers the ```rust fence; this won't recur." @@ -599,11 +599,11 @@ "verdict": "Critical Bugs", "best_for": "Violated the stdlib-only constraint: used aiohttp (2 imports) for the HTTP server, so it won't run as-is. Weaker than Qwen's 75 on the same prompt (Qwen used stdlib and passed). Not the offload pick for webhook work; Qwen is.", "critical_bugs": [ - "FATAL for the prompt: uses aiohttp (external dep) despite 'stdlib only' requirement. ModuleNotFoundError on import \u2014 won't run as delivered.", - "Chose aiohttp reasoning that stdlib http.server is synchronous/blocks the loop \u2014 a fair architectural point, but it violated the explicit constraint instead of using asyncio.start_server (stdlib, async) like Qwen did.", + "FATAL for the prompt: uses aiohttp (external dep) despite 'stdlib only' requirement. ModuleNotFoundError on import — won't run as delivered.", + "Chose aiohttp reasoning that stdlib http.server is synchronous/blocks the loop — a fair architectural point, but it violated the explicit constraint instead of using asyncio.start_server (stdlib, async) like Qwen did.", "Wrapped in markdown + prose preamble (needed extraction)." ], - "patch_code": "# FIX: replace aiohttp server with asyncio.start_server (stdlib, fully async) \u2014 exactly\n# what the Qwen webhook submission did. Then the logic (rate limit/idempotency/429) can run." + "patch_code": "# FIX: replace aiohttp server with asyncio.start_server (stdlib, fully async) — exactly\n# what the Qwen webhook submission did. Then the logic (rate limit/idempotency/429) can run." }, { "id": "gemma4-26b-a4b-8bit-mlx-automation", @@ -632,9 +632,415 @@ "best_for": "Doesn't run: SyntaxError (global processor declared after assignment in the signal-handler setup). A real but localized bug. First automation result for either model, so no head-to-head yet.", "critical_bugs": [ "FATAL: line 270 'global processor' declared AFTER processor is assigned earlier in the same function -> SyntaxError. The whole module fails to parse.", - "Signal-handler design (global instance for SIGINT access) is the root cause \u2014 globals-after-assignment is a classic Python footgun the model walked into." + "Signal-handler design (global instance for SIGINT access) is the root cause — globals-after-assignment is a classic Python footgun the model walked into." ], "patch_code": "# FIX: move 'global processor' to the FIRST line of the function that assigns it,\n# before any assignment to processor." + }, + { + "id": "kat-coder-v2.5-dev-xl-mlx-lfu", + "prompt_id": "lfu", + "model_name": "KAT-Coder v2.5 Dev XL", + "quant": "MLX", + "param_size": "XL", + "format": "mlx", + "lang": "python", + "filename": "outputs/kat-coder-v2.5-dev-xl-mlx-lfu.py", + "tok_sec": 65.26, + "ttft_sec": 7.42, + "total_tokens": 6172, + "tests_pass": false, + "total_score": 49, + "breakdown": { + "complexity": 11, + "concurrency": 10, + "isolation": 11, + "memory_edge_cases": 11, + "test_integrity": 6 + }, + "verdict": "Critical Bugs", + "best_for": "Strong architectural instincts for async data-structure design but unsafe to ship — needs fixes to token corruption, O(1) freq maintenance, and the sync/async transaction boundary before use.", + "critical_bugs": [ + "Line 223 `self._ bump_freq(node)` — SyntaxError (space in attribute name); file cannot parse or run at all, so every test fails to execute.", + "`min(self._freq_map)` linear scan used whenever the min-freq bucket empties (in _bump_freq, _apply_transaction, _evict_one, _evict_node) — breaks the strict O(1) requirement; should advance _min_freq incrementally.", + "Transaction.get calls `self._cache.get(key)` synchronously without await — LFUCache.get is async, so this returns a coroutine object instead of the value; read-your-own-writes fallthrough to global cache is broken.", + "stop_evictor uses `asyncio.get_event_loop().run_until_complete(...)` from within async code — deprecated; raises RuntimeError under asyncio.run. Should `await self._evictor_task` after cancel.", + "_Node is a @dataclass without slots=True; only _FreqList has __slots__." + ], + "patch_code": "# Fix 1 (line 223): remove the stray space in the attribute access\n- self._ bump_freq(node)\n+ self._bump_freq(node)\n\n# Fix 2: O(1) min_freq update — never rescan. On bump, when the old bucket\n# empties and it WAS the min, advance min_freq by 1 (the node just moved to freq+1).\ndef _bump_freq(self, node):\n old_freq = node.freq\n self._remove_from_freq_list(node)\n node.freq += 1\n self._add_to_freq_list(node)\n if old_freq == self._min_freq and self._freq_map[old_freq].is_empty():\n del self._freq_map[old_freq]\n self._min_freq += 1\n\n# Fix 3: make Transaction.get async and await the cache call\nasync def get(self, key):\n if self._committed or self._rolled_back:\n raise RuntimeError(\"Transaction already closed\")\n if key in self._deletes: return None\n if key in self._writes:\n value, expires_at = self._writes[key]\n if expires_at > 0 and time.monotonic() >= expires_at:\n self._deletes.add(key); return None\n return value\n return await self._cache.get(key)\n\n# Fix 4 (stop_evictor): await the cancelled task, don't run_until_complete\nasync def stop_evictor(self):\n if self._evictor_task is not None:\n self._evictor_task.cancel()\n try: await self._evictor_task\n except asyncio.CancelledError: pass\n self._evictor_task = None\n\n# Fix 5: add slots to _Node (Python 3.10+)\n@dataclass(slots=True)", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": null + }, + { + "id": "kat-coder-v2.5-dev-xl-mlx-tts", + "prompt_id": "tts", + "model_name": "KAT-Coder v2.5 Dev XL", + "quant": "MLX", + "param_size": "XL", + "format": "mlx", + "lang": "python", + "filename": "outputs/kat-coder-v2.5-dev-xl-mlx-tts.py", + "tok_sec": 65.26, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 52, + "breakdown": { + "complexity": 15, + "concurrency": 7, + "error_handling": 13, + "resource_safety": 7, + "test_integrity": 10 + }, + "verdict": "Critical Bugs", + "best_for": "Scaffolding async pipeline structure (queue+semaphore+retry+callbacks) when you intend to fix the cancellation and shutdown paths yourself.", + "critical_bugs": [ + "In-flight cancellation is a no-op: cancel() only sets _cancelled=True for STARTED jobs, but every cancellation check ALSO requires job.state==CANCELLED, which cancel() never sets for in-flight jobs (only QUEUED). In-flight jobs always run to completion — the spec's core requirement is unmet.", + "Hard 30s timeout: Test B drains 100 jobs at max_concurrency=1 (sequential); expected ~20.6s, p95 ~28s, with 10% retry backoffs it routinely exceeds 30s — a self-inflicted wall-clock sink.", + "Leaked tasks on shutdown: stop() cancels _worker_task but does NOT track or cancel the fire-and-forget _process_job tasks (asyncio.create_task). In-flight synthesis tasks are orphaned — no gather, no cancellation, semaphore slots held until they finish.", + "Latent deadlock: _process_job is spawned untracked; if it raises outside its try/except, task_done() is never called AND the job stays STARTED forever, so drain()'s terminal-state poll loops infinitely.", + "Cancellation test passes for the wrong reason: it asserts completed+cancelled+failed==20, but since in-flight cancel is a no-op, 'cancelled' jobs actually COMPLETE." + ], + "patch_code": "# FIX 1 — cancel() must set state=CANCELLED for in-flight jobs too:\nasync def cancel(self, job_id):\n async with self._lock:\n job = self._jobs.get(job_id)\n if job is None: raise KeyError(f\"Unknown job: {job_id}\")\n if job.state in (COMPLETED, FAILED, CANCELLED): return\n job._cancelled = True\n job.state = JobState.CANCELLED # set for BOTH queued AND in-flight\n await self._notify(job, \"cancelled\")\n\n# FIX 2 — gate cancellation checks on the flag, not the state:\n# replace every `if job._cancelled and job.state == JobState.CANCELLED`\n# with `if job._cancelled:`.\n\n# FIX 3 — track spawned tasks and cancel them on stop():\n# __init__: self._tasks = set()\n# _worker_loop:\n# t = asyncio.create_task(self._process_job(job))\n# self._tasks.add(t); t.add_done_callback(self._tasks.discard)\n# stop():\n# for t in list(self._tasks): t.cancel()\n# await asyncio.gather(*self._tasks, return_exceptions=True)\n\n# FIX 4 — don't self-DOS the backpressure test: use max_concurrency=1 AND\n# max_queue_size=10, submit 11, assert 11th rejected, then stop WITHOUT draining.", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "kat-coder-v2.5-dev-xl-mlx-webhook", + "prompt_id": "webhook", + "model_name": "KAT-Coder v2.5 Dev XL", + "quant": "MLX", + "param_size": "XL", + "format": "mlx", + "lang": "python", + "filename": "outputs/kat-coder-v2.5-dev-xl-mlx-webhook.py", + "tok_sec": 65.26, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 72, + "breakdown": { + "schema_io": 17, + "transport": 16, + "error_handling": 15, + "state_safety": 11, + "test_integrity": 13 + }, + "verdict": "Critical Bugs", + "best_for": "Reference design for the 5 required webhook mechanisms; needs the two bugs fixed before it can run or be trusted under concurrency.", + "critical_bugs": [ + "SyntaxError: `global SECRET, PORT` at line 628 (inside main()) is declared after SECRET (L35) and PORT (L36) are already assigned at module scope — Python rejects 'name used prior to global declaration'. File does not run; tests_pass=false.", + "Cross-thread data race: _BridgeHandler.do_POST runs forward() in a fresh per-request asyncio loop on a ThreadingMixIn worker thread, mutating shared IdempotencyStore._store and TokenBucket state with only loop-bound asyncio.Locks — they do NOT synchronize across OS threads, so concurrent POSTs can both pass is_duplicate() and race the rate-limiter token count.", + "Unguarded int(headers.get('Retry-After','1')) raises ValueError on a non-numeric header; falls through to the generic 500 handler instead of clamping to a safe default.", + "Test (b) prepares a bad_sig from a bad_body then ignores it and posts the original body with signature 'deadbeef'*8 — the prepared tampered path is dead code (copy-paste leftover)." + ], + "patch_code": "# 1. Fix SyntaxError — remove the redundant global decl in main(); SECRET/PORT\n# are already module-level (L35-36), just reassign directly.\ndef main():\n parser = argparse.ArgumentParser(description=\"Twitch EventSub Discord bridge\")\n parser.add_argument(\"--secret\", default=SECRET)\n parser.add_argument(\"--port\", default=PORT, type=int)\n parser.add_argument(\"--test\", action=\"store_true\")\n args = parser.parse_args()\n # global SECRET, PORT <-- DELETE (SyntaxError: name used prior to global)\n SECRET = args.secret\n PORT = args.port\n if args.test: sys.exit(run_tests(PORT))\n ...\n\n# 2. Make IdempotencyStore + TokenBucket thread-safe (threading.Lock, not asyncio.Lock)\nimport threading\nclass IdempotencyStore:\n def __init__(self, ttl=IDEMPOTENCY_TTL):\n self._ttl = ttl; self._store = {}; self._lock = threading.Lock()\n def is_duplicate(self, event_id):\n now = time.monotonic()\n with self._lock:\n self._store = {k:v for k,v in self._store.items() if now-v < self._ttl}\n if event_id in self._store: return True\n self._store[event_id] = now; return False\n\n# 3. Clamp Retry-After to a safe int\ntry: retry_after = max(1, int(float(headers.get(\"Retry-After\",\"1\"))))\nexcept (TypeError, ValueError): retry_after = 1", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "kat-coder-v2.5-dev-xl-mlx-rust", + "prompt_id": "rust", + "model_name": "KAT-Coder v2.5 Dev XL", + "quant": "MLX", + "param_size": "XL", + "format": "mlx", + "lang": "rust", + "filename": "outputs/kat-coder-v2.5-dev-xl-mlx-rust.rs", + "tok_sec": 65.26, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 36, + "breakdown": { + "ownership": 7, + "concurrency": 8, + "error_handling": 13, + "cancellation": 7, + "test_integrity": 1 + }, + "verdict": "Critical Bugs", + "best_for": "Generating idiomatic tokio building blocks (task functions, mock_fetch, error enums) but not a wired-up, compilable module — needs heavy human completion.", + "critical_bugs": [ + "JoinHandle::clone() in WatcherManager::shutdown (L365) — tokio JoinHandle is NOT Clone; `let handle = entry.handle.clone()` fails to compile.", + "tracing::info!/warn!/error!/trace! used 12x but `use tracing` is missing and tracing is not std — 12 unresolved macro calls.", + "Infallible used (L172) but never imported.", + "No main, no #[tokio::main], no #[tokio::test], zero assert! — the binary has no entry point and no tests.", + "WatcherManager V1 methods (new/watcher_count/is_healthy/add_watcher/remove_watcher/shutdown, L265-375) have empty or garbled bodies: floating `self.inner.read().await;` with no let/return, `entry in set.watchers.values() {` missing the `for` keyword, `set` used without a binding.", + "V1 run_consumer spawns a loop that immediately breaks ('approach flawed... break;') and never aggregates; V2 add_watcher/remove_watcher bodies are equally stubbed and reference a V2 run_consumer that doesn't exist." + ], + "patch_code": "// 1. Add missing imports at top:\nuse std::convert::Infallible;\n// and add `tracing = \"0.1\"` to Cargo.toml with `use tracing;` OR replace\n// all tracing::info!/warn!/error! with eprintln!/log macros.\n\n// 2. Fix V1 shutdown — JoinHandle is not Clone; move-join instead:\npub async fn shutdown(&self) -> Result<(), ServiceError> {\n let _ = self.shutdown_tx.send(());\n let mut set = self.inner.write().await;\n let entries: Vec<(_, tokio::task::JoinHandle<()>)> =\n set.watchers.drain().map(|(k,v)| (k, v.handle)).collect();\n drop(set); // release lock before awaiting joins\n for (id, handle) in entries {\n if handle.await.is_err() { tracing::warn!(watcher_id = id, \"watcher panicked\"); }\n }\n Ok(())\n}\n\n// 3. Finish V2 add_watcher (the design is sound; just complete it):\npub async fn add_watcher(&self, id: u32) -> Result<(), ServiceError> {\n let mut set = self.inner.write().await;\n if set.watchers.contains_key(&id) { return Err(ServiceError::WatcherNotFound(id)); }\n let health_flag = Arc::new(AtomicBool::new(true));\n set.health_flags.insert(id, health_flag.clone());\n let tx = self.shared_tx.clone();\n let mut shutdown_rx = self.shutdown_tx.subscribe();\n let handle = tokio::spawn(async move {\n let mut tick = tokio::time::interval(Duration::from_millis(80));\n let mut fails = 0u32;\n loop {\n tokio::select! {\n _ = shutdown_rx.recv() => break,\n _ = tick.tick() => match mock_fetch(id).await {\n Ok(vs) => { fails = 0; for v in vs { let _ = tx.send(WatchedItem{watcher_id:id, value:v}).await; } }\n Err(_) => { fails += 1; if fails > 5 { health_flag.store(false, Ordering::Relaxed); break; } }\n }\n }\n }\n });\n set.watchers.insert(id, WatcherEntryV2 { id, handle });\n Ok(())\n}\n\n// 4. Add a real main + a backpressure/shutdown test:\n#[tokio::main]\nasync fn main() {\n let (mgr, mut out_rx) = WatcherManagerV2::new();\n mgr.add_watcher(1).await.unwrap();\n tokio::time::sleep(Duration::from_millis(500)).await;\n mgr.shutdown().await.unwrap();\n if let Some(o) = out_rx.recv().await { println!(\"items={}.\", o.total_items); }\n}", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "kat-coder-v2.5-dev-xl-mlx-automation", + "prompt_id": "automation", + "model_name": "KAT-Coder v2.5 Dev XL", + "quant": "MLX", + "param_size": "XL", + "format": "mlx", + "lang": "python", + "filename": "outputs/kat-coder-v2.5-dev-xl-mlx-automation.py", + "tok_sec": 65.26, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 60, + "breakdown": { + "idempotency": 17, + "retry_backoff": 16, + "checkpointing": 14, + "signal_handling": 11, + "test_integrity": 2 + }, + "verdict": "Critical Bugs", + "best_for": "Reference scaffold for an asyncio checkpointed batch processor if you add a test harness and fix the checkpoint error-path close bug.", + "critical_bugs": [ + "No tests at all: process() is defined but never called by any async main/harness; file requires CLI `items` arg so `python file.py` errors out (tests_pass=false, violates self-contained-runnable contract).", + "Checkpoint error-path double-close: `os.close(fd) if not None else None` — `not None` is always True so it always calls os.close(fd); if the fd was already closed this re-raises OSError [Errno 9] and masks the original exception, also skipping the os.unlink(tmp) cleanup. Should be `if fd is not None` with a closed-flag.", + "SIGINT drain logic is broken: run() does ONE asyncio.wait(FIRST_COMPLETED), then cancels pending tasks and swallows CancelledError via gather(return_exceptions=True) rather than letting in-flight work finish.", + "No explicit exit 0 on graceful shutdown: _on_signal sets an Event and run() returns normally, but a second SIGINT calls sys.exit(1) (hard exit, no checkpoint flush)." + ], + "patch_code": "# 1. Fix checkpoint error-path close (use a closed flag)\nasync def save_checkpoint(cp, path):\n parent = path.parent; parent.mkdir(parents=True, exist_ok=True)\n loop = asyncio.get_event_loop()\n def _write():\n fd, tmp = tempfile.mkstemp(suffix='.tmp', dir=parent); closed = False\n try:\n payload = json.dumps(asdict(cp), indent=2, ensure_ascii=False) + '\\n'\n os.write(fd, payload.encode('utf-8')); os.close(fd); closed = True\n os.replace(tmp, str(path))\n except BaseException:\n if not closed:\n try: os.close(fd)\n except OSError: pass\n try: os.unlink(tmp)\n except OSError: pass\n raise\n await loop.run_in_executor(None, _write)\n\n# 2. Fix SIGINT drain: let in-flight finish, flush, exit 0\nasync def run(self):\n self._start_time = time.monotonic()\n self._install_signal_handlers()\n pending = [it for it in self.items if not self.cp.is_done(it)]\n skipped = len(self.items) - len(pending)\n self._semaphore = asyncio.Semaphore(self.max_concurrency)\n async def _bounded(item):\n async with self._semaphore:\n if self._shutdown_requested.is_set(): return\n await self._process_one(item)\n workers = [asyncio.create_task(_bounded(it)) for it in pending]\n try:\n await asyncio.gather(*workers, return_exceptions=True)\n await save_checkpoint(self.cp, self.checkpoint_path)\n finally:\n self._remove_signal_handlers()\n elapsed_ms = int((time.monotonic() - self._start_time) * 1000)\n return {'succeeded': len(self.cp.succeeded), 'failed': len(self.cp.failed),\n 'skipped': skipped, 'total': len(self.items), 'elapsed_ms': elapsed_ms}\n\n# 3. Add a self-contained async main() test harness (no CLI args)\nasync def main():\n items = [f'job-{i}' for i in range(20)]\n bp = BatchProcessor(items, checkpoint_path='cp.json')\n s = await bp.run(); print(json.dumps(s))\nif __name__ == '__main__': asyncio.run(main())", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "qwen3-coder-30b-6bit-mlx-lfu", + "prompt_id": "lfu", + "model_name": "Qwen3 Coder 30B", + "quant": "6-bit MLX", + "param_size": "30B", + "format": "mlx", + "lang": "python", + "filename": "outputs/qwen3-coder-30b-6bit-mlx-lfu.py", + "tok_sec": 72.7, + "ttft_sec": 0.9, + "total_tokens": 2779, + "tests_pass": false, + "total_score": 45, + "breakdown": { + "complexity": 6, + "concurrency": 10, + "isolation": 9, + "memory_edge_cases": 5, + "test_integrity": 15 + }, + "verdict": "Critical Bugs", + "best_for": "Drafting async scaffolding and transaction API shapes when you can fix the eviction/await bugs by hand.", + "critical_bugs": [ + "_evict_lfu: min(self._freq_buckets.keys()) picks the lowest bucket number, but _put_internal/_update_freq leave a stale EMPTY bucket 0 behind after every insertion (freq starts at 0 then immediately moves to 1, bucket 0 never deleted). So min() always returns 0, the bucket is empty, `if bucket:` is False, and eviction NEVER fires — the cache grows unbounded. This is exactly why the test fails on 'Should have evicted b'.", + "_evict_lfu uses min(self._freq_buckets.keys()) — an O(k) linear scan, violating strict O(1). No incremental min_freq tracking; no frequency-bucket + DLL structure (OrderedDict only).", + "Transaction.get calls self.cache.get(key) which is async def — the call is never awaited, so it returns a coroutine object instead of the cached value. Read-through to the global cache from inside a transaction is broken.", + "_evictor_loop runs _cleanup_expired via run_in_executor on a ThreadPoolExecutor WITHOUT acquiring self._lock, while async get/put mutate the same dicts under the lock — a real thread-vs-event-loop data race on shared state.", + "All timestamps use time.time() instead of time.monotonic() (vulnerable to wall-clock/NTP jumps). No __slots__; empty frequency buckets never pruned." + ], + "patch_code": "# 1. Track min_freq incrementally and prune empty buckets in _update_freq/_delete_internal\ndef _update_freq(self, key):\n old = self._key_to_freq[key]; new = old + 1\n self._key_to_freq[key] = new\n old_b = self._freq_buckets[old]; del old_b[key]\n if not old_b and old == self._min_freq: # prune + advance min_freq\n del self._freq_buckets[old]; self._min_freq = new\n elif not old_b:\n del self._freq_buckets[old]\n self._freq_buckets.setdefault(new, OrderedDict())[key] = None\n\ndef _evict_lfu(self):\n bucket = self._freq_buckets.get(self._min_freq)\n if not bucket: return\n victim = next(iter(bucket)) # oldest in min-freq bucket = LRU tie-break\n self._delete_internal(victim)\n\n# 2. On new insertion set self._min_freq = 1 (after 0->1 bump).\n# 3. Fix Transaction read-through: make get async and await the cache.\nasync def get(self, key):\n if self._rolled_back: raise RuntimeError('rolled back')\n if key in self._deletes: return None\n if key in self._writes: return self._writes[key].value\n return await self.cache.get(key) # was: self.cache.get(key) -> coroutine\n# 4. Evictor must hold the lock: drop the ThreadPoolExecutor; just:\nasync def _evictor_loop(self):\n while self._evictor_running:\n await asyncio.sleep(1.0)\n async with self._lock: self._cleanup_expired()\n# 5. Replace time.time() with time.monotonic() everywhere; add __slots__.", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": null + }, + { + "id": "qwen3-coder-30b-6bit-mlx-tts", + "prompt_id": "tts", + "model_name": "Qwen3 Coder 30B", + "quant": "6-bit MLX", + "param_size": "30B", + "format": "mlx", + "lang": "python", + "filename": "outputs/qwen3-coder-30b-6bit-mlx-tts.py", + "tok_sec": 72.7, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 44, + "breakdown": { + "complexity": 13, + "concurrency": 6, + "error_handling": 10, + "resource_safety": 8, + "test_integrity": 7 + }, + "verdict": "Critical Bugs", + "best_for": "Scaffolding async job pipelines when you intend to rewrite the backpressure layer; shows decent retry/callback structure but fails the spec's central mechanic.", + "critical_bugs": [ + "submit() raises Exception('Queue is full') instead of awaiting a slot (no backpressure) — submitting past queue_limit aborts the whole pipeline; this is why Test c crashes with an uncaught Queue-is-full error.", + "Bounded concurrency is enforced two ways (manual len(_active_tasks) RATE_LIMIT_WINDOW so sub-window requests leak refill credit inconsistently.", + "HMAC and idempotency state are module globals mutated without a lock from the single request thread — works only because HTTPServer is serial." + ], + "patch_code": "# 1) Evict stale idempotency entries (bounded store + monotonic clock)\nimport time\nIDEMPOTENCY_TTL = 300\ndef is_idempotent(event_id):\n now = time.monotonic()\n stale = [k for k, ts in list(idempotency_store.items()) if now - ts > IDEMPOTENCY_TTL]\n for k in stale[:64]: del idempotency_store[k]\n if event_id in idempotency_store and now - idempotency_store[event_id] < IDEMPOTENCY_TTL:\n idempotency_store[event_id] = now; return True\n idempotency_store[event_id] = now; return False\n\n# 2) Cap 429 retries so a stuck Discord never hangs the server\nMAX_429_RETRIES = 1\nasync def forward_to_discord(self, data):\n while not rate_limit(): await asyncio.sleep(0.05)\n payload = {\"content\": f\"Event {data.get('type')}: {data.get('data', {}).get('message', '')}\"}\n attempts = 0\n while True:\n try: await discord_send(payload); return\n except Exception as e:\n if 'HTTP 429' in str(e) and attempts < MAX_429_RETRIES:\n m = re.search(r'Retry-After (\\d+)', str(e))\n await asyncio.sleep(int(m.group(1)) if m else 1); attempts += 1; continue\n raise # let caller return a clear 502, do not swallow\n\n# 3) SO_REUSEADDR + clean shutdown (fixes EADDRINUSE)\nclass ReusableHTTPServer(HTTPServer): allow_reuse_address = True\nif __name__ == '__main__':\n unittest.main(argv=[''], exit=False, verbosity=2)\n server = ReusableHTTPServer(('localhost', 8080), WebhookHandler)\n try: server.serve_forever()\n finally: server.server_close()", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "qwen3-coder-30b-6bit-mlx-rust", + "prompt_id": "rust", + "model_name": "Qwen3 Coder 30B", + "quant": "6-bit MLX", + "param_size": "30B", + "format": "mlx", + "lang": "rust", + "filename": "outputs/qwen3-coder-30b-6bit-mlx-rust.rs", + "tok_sec": 72.7, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 54, + "breakdown": { + "ownership": 15, + "concurrency": 10, + "error_handling": 14, + "cancellation": 9, + "test_integrity": 6 + }, + "verdict": "Critical Bugs", + "best_for": "Sketching idiomatic Rust type/trait/error-enum shapes when the channel lifecycle and task-join discipline will be added by a human.", + "critical_bugs": [ + "No consumer: WatcherManager::new() does `let (consumer_tx, _) = mpsc::unbounded_channel()` — the Receiver is dropped immediately; every consumer_tx.send() succeeds into a dead channel and the data is silently lost.", + "Backpressure pillar failed: channel is mpsc::unbounded_channel — spec requires a BOUNDED channel with documented capacity and full behavior. There is no bound and no backpressure anywhere.", + "remove_watcher(id) deletes the map entry but does NOT cancel or await the watcher's spawned task — the orphaned task keeps polling mock_fetch and sending on consumer_tx until global shutdown. Leaked/hung task.", + "shutdown() has no JoinHandles (tasks spawned detached), so it cannot confirm termination — it cancels the token then guesses with sleep(100ms).", + "No main()/no #[tokio::main] — file is a library only; as submitted it cannot run as a service.", + "test_unhealthy_watcher is statistically broken: 15% per-poll failure with 10ms interval over 100ms (~10 polls) expects ~1.5 failures — it almost never reaches the 5-consecutive threshold, so the assert!(healthy==false) will flakily fail." + ], + "patch_code": "// 1) Hold the receiver; use a BOUNDED channel (backpressure).\npub struct WatcherManager {\n watchers: Arc>>,\n consumer_tx: mpsc::Sender, // bounded\n join: Arc>>>, // track tasks for clean shutdown\n shutdown_token: CancellationToken,\n}\nimpl WatcherManager {\n pub fn new(bound: usize) -> (Self, mpsc::Receiver) {\n let (tx, rx) = mpsc::channel::(bound);\n (Self { watchers: Arc::new(RwLock::new(HashMap::new())),\n consumer_tx: tx, join: Arc::new(Mutex::new(Vec::new())),\n shutdown_token: CancellationToken::new() }, rx)\n }\n pub async fn shutdown(&self) {\n self.shutdown_token.cancel();\n let mut handles = self.join.lock().await;\n for h in handles.drain(..) { let _ = tokio::time::timeout(Duration::from_secs(1), h).await; }\n }\n}\n// 2) Consumer that actually drains the bounded channel:\nasync fn consumer_task(mut rx: mpsc::Receiver, shutdown: CancellationToken) {\n loop {\n tokio::select! {\n _ = shutdown.cancelled() => break,\n ev = rx.recv() => match ev {\n Some(WatcherEvent::Items(id, items)) => { /* aggregate */ }\n None => break,\n }\n }\n }\n}\n// 3) Deterministic unhealthy test — inject failures instead of relying on rand:\nasync fn failing_fetch(_id: u32) -> Result, FetchError> { Err(FetchError::MockFetchFailed) }", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "qwen3-coder-30b-6bit-mlx-automation", + "prompt_id": "automation", + "model_name": "Qwen3 Coder 30B", + "quant": "6-bit MLX", + "param_size": "30B", + "format": "mlx", + "lang": "python", + "filename": "outputs/qwen3-coder-30b-6bit-mlx-automation.py", + "tok_sec": 72.7, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": true, + "total_score": 58, + "breakdown": { + "idempotency": 12, + "retry_backoff": 16, + "checkpointing": 14, + "signal_handling": 10, + "test_integrity": 6 + }, + "verdict": "Critical Bugs", + "best_for": "Throwaway single-shot batch jobs where you only care that items ran once, not accurate status accounting.", + "critical_bugs": [ + "Skipped items never counted: _process_items filters out completed/failed items BEFORE counting them as skipped; the only skip-counter lives inside _process_item behind a re-check that is unreachable for pre-filtered items. Result: any idempotent re-run (or any run loading a checkpoint) prints succeeded:0, failed:0, skipped:0, total:N — the summary is meaningless. This is the 0/0/0/20 observed.", + "All N coroutines are created up front in the list comprehension; only execution is semaphore-bounded, not submission. On large inputs this defeats bounded-concurrency memory expectations and means 'stop accepting new work' on SIGINT is impossible — everything is already queued.", + "run()'s finally only sets shutdown_event; it never calls save_checkpoint. If an in-flight item is cancelled before its own save_checkpoint, completed work can be lost on shutdown.", + "NamedTemporaryFile(delete=False) leaves an orphan .tmp file on crash between create and os.replace; no cleanup.", + "Tests B and C are vacuous: B only json.loads (asserts nothing about content), C prints 'Concurrency control: OK' without measuring max in-flight. Test A doesn't verify in-flight tasks actually finished before exit." + ], + "patch_code": "# Fix 1: count skipped items at filter time\nasync def _process_items(self):\n done = self.checkpoint.completed | self.checkpoint.failed\n to_process = []\n for item in self.items:\n if item in done: self.results['skipped'] += 1 # count here\n else: to_process.append(item)\n tasks = [asyncio.create_task(self._process_item(i)) for i in to_process]\n for coro in asyncio.as_completed(tasks): await coro\n self._update_summary()\n save_checkpoint(self.checkpoint, self.checkpoint_path)\n print(json.dumps({'succeeded': self.results['succeeded'], 'failed': self.results['failed'],\n 'skipped': self.results['skipped'], 'total': len(self.items),\n 'elapsed_ms': int((time.time()-self.start_time)*1000)}))\n\n# Fix 2: bounded submission + checkpoint flush on shutdown\n pending = [i for i in self.items if i not in done]\n sem = self.semaphore\n async def run_one(item):\n async with sem:\n if not self.running: return\n await self._process_item(item)\n bg = [asyncio.create_task(run_one(i)) for i in pending]\n try:\n for t in asyncio.as_completed(bg): await t\n finally:\n save_checkpoint(self.checkpoint, self.checkpoint_path) # flush on any exit", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "qwen3.6-35b-a3b-uncensored-hauhaucs-aggressive-automation", + "prompt_id": "automation", + "model_name": "Qwen 3.6 35B-A3B (uncensored hauhaucs aggressive)", + "quant": "GGUF", + "param_size": "35B-A3B (MoE)", + "format": "gguf", + "lang": "python", + "filename": "outputs/qwen3.6-35b-a3b-uncensored-hauhaucs-aggressive-automation.py", + "tok_sec": 62.54, + "ttft_sec": null, + "total_tokens": null, + "tests_pass": false, + "total_score": 46, + "breakdown": { + "idempotency": 6, + "retry_backoff": 11, + "checkpointing": 14, + "signal_handling": 7, + "test_integrity": 8 + }, + "verdict": "Critical Bugs", + "best_for": "Generating plausible-looking async scaffolding that passes casual reading but fails under any actual execution — a cautionary example of why integration tests must run the happy path, not just the interrupt path.", + "critical_bugs": [ + "run_batch never awaits the tasks it creates in the normal path: the for-loop calls asyncio.create_task(process_item(item)) but the only `await asyncio.gather(*running_tasks)` is gated behind `if stop_event.is_set()`. A normal uninterrupted run returns immediately with succeeded=0, no checkpoint written, zero items processed — the processor is a no-op on the happy path.", + "Summary double-counts on resume: succeeded_count/failed_count are seeded from checkpoint['summary'] (cumulative) while skipped_count is recomputed by scanning completed_items, so a converged resume run reports succeeded+skipped twice the total. The test_resumability assertion catches this.", + "SIGINT graceful-exit guarantee is unenforced: save_checkpoint is called inside process_item after each item with no batching/flush-on-signal; the gather-on-stop path runs ALL queued tasks to completion rather than cancelling in-flight ones.", + "process() failure rate (~20%) combined with 4 total attempts means genuinely-failing items sometimes pass by luck; no distinction between transient and permanent failures." + ], + "patch_code": "# FIX 1 (fatal): always await created tasks, not only on stop\n for item in items:\n if stop_event.is_set(): break\n status = completed_items.get(item)\n if status in (success, failed): skipped_count += 1; continue\n task = asyncio.create_task(process_item(item))\n running_tasks.append(task)\n if stop_event.is_set():\n for t in running_tasks: t.cancel() # cancel in-flight per spec\n await asyncio.gather(*running_tasks, return_exceptions=True) # ALWAYS await\n\n# FIX 2 (double-count): reset counters to current-run disposition, don't carry cumulative\n succeeded_count = 0; failed_count = 0\n # inside process_item on success: succeeded_count += 1 (not += cumulative)\n # store cumulative totals separately from the per-run summary that must\n # satisfy succeeded+failed+skipped == total.\n\n# FIX 3 (summary correctness): derive summary from final checkpoint state\n terminal = {k: v for k, v in completed_items.items()}\n succeeded_count = sum(1 for v in terminal.values() if v == success)\n failed_count = sum(1 for v in terminal.values() if v == failed)\n skipped_count = sum(1 for it in items if it in terminal and terminal[it] in (success, failed))\n summary = {succeeded: succeeded_count, failed: failed_count,\n skipped: skipped_count, total: len(items), elapsed_ms: elapsed_ms}", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": "captured via old OpenAI-compat API (stats empty) — TTFT unknown; speed is the model's known rate" + }, + { + "id": "gemma-4-26b-a4b-data", + "prompt_id": "data", + "model_name": "Gemma 4 26B-A4B", + "quant": "MLX", + "param_size": "26B-A4B (MoE)", + "format": "mlx", + "lang": "python", + "filename": "outputs/gemma-4-26b-a4b-data.py", + "tok_sec": 53.48, + "ttft_sec": 1.127, + "total_tokens": 5871, + "tests_pass": true, + "total_score": 86, + "breakdown": { + "query_safety": 18, + "pooling": 18, + "transactions": 18, + "pagination": 18, + "test_integrity": 14 + }, + "verdict": "Minor Logic Flaws", + "best_for": "Production-shaped async data-access layer with correct pooling, parameterization, pagination, and transactional rollback — a reliable template for a real Postgres-backed service.", + "critical_bugs": [ + "Test D is mislabeled as 'pool exhaustion' but never triggers it: 10 concurrent get_user_with_posts calls on a pool of 5 each hold the connection for only ~10ms (asyncio.sleep(0.01)), so all 10 complete within the 2s acquire timeout and PoolExhaustedError is never raised or asserted.", + "No positive acquire-timeout test: nothing holds max_size connections and then asserts a (max_size+1)th acquire raises PoolExhaustedError, so the timeout path is unverified.", + "MockConnection.execute's injection guard ('if \"'\" in query and not any string params') is inverted/naive — it flags any query containing a quote when no string params are present, which would false-positive on legitimate DDL.", + "_in_use_count in MockPool is maintained as a separate counter from the semaphore — correct today but a manual-invariant drift hazard; in_use should be derived from the semaphore." + ], + "patch_code": "# 1. Add a real acquire-timeout / pool-exhaustion test:\nasync def test_pool_exhaustion():\n state = MockDatabaseState()\n pool = MockPool(max_size=5, state=state)\n held = [await pool.acquire() for _ in range(5)]\n try:\n with pytest.raises(PoolExhaustedError):\n await asyncio.wait_for(pool.acquire(), timeout=3.0)\n finally:\n for c in held: await c.release()\n assert pool.in_use == 0\n\n# 2. Derive in_use from the semaphore so the counter cannot drift:\n@property\ndef in_use(self) -> int:\n return self.max_size - self._semaphore._value", + "timestamp": "2026-07-29T04:30:00Z", + "ttft_note": null } ] } \ No newline at end of file diff --git a/outputs/.last_response_schema.json b/outputs/.last_response_schema.json index f0f2e5d..6f2d9ba 100644 --- a/outputs/.last_response_schema.json +++ b/outputs/.last_response_schema.json @@ -1,12 +1,15 @@ { - "usage": { - "prompt_tokens": 593, - "completion_tokens": 10029, - "total_tokens": 10622, - "completion_tokens_details": { - "reasoning_tokens": 7460 - } + "model_instance_id": "google/gemma-4-26b-a4b", + "stats": { + "input_tokens": 718, + "total_output_tokens": 5153, + "reasoning_output_tokens": 1919, + "tokens_per_second": 53.47535329547263, + "time_to_first_token_seconds": 1.127 }, - "stats": {}, - "system_fingerprint": "qwen3.6-35b-a3b-uncensored-hauhaucs-aggressive" + "response_id": "resp_d8fafc117fa4e51b927d8cd531ded49984ae59e49c39d3eb", + "_output_types": [ + "reasoning", + "message" + ] } \ No newline at end of file diff --git a/outputs/gemma-4-26b-a4b-data.py b/outputs/gemma-4-26b-a4b-data.py new file mode 100644 index 0000000..f5d0d34 --- /dev/null +++ b/outputs/gemma-4-26b-a4b-data.py @@ -0,0 +1,312 @@ +import asyncio +import dataclasses +from typing import Any, Dict, List, Tuple, Optional + +# ============================================================================= +# EXCEPTIONS +# ============================================================================= + +class DatabaseError(Exception): + """Base class for database errors.""" + pass + +class PoolExhaustedError(DatabaseError): + """Raised when the connection pool cannot provide a connection within timeout.""" + pass + +class TransactionError(DatabaseError): + """Raised when a transaction fails and is rolled back.""" + pass + +class ValidationError(ValueError): + """Raised when input parameters fail validation.""" + pass + +# ============================================================================= +# MOCK DATABASE LAYER +# ============================================================================= + +@dataclasses.dataclass +class User: + id: int + name: str + label: str + +@dataclasses.dataclass +class Post: + id: int + user_id: int + content: str + +class MockDatabaseState: + """Holds the actual 'in-memory' data.""" + def __init__(self): + self.users: Dict[int, User] = { + i: User(id=i, name=f"User_{i}", label="default") for i in range(1, 26) + } + self.posts: List[Post] = [] + for u_id in self.users: + for p_idx in range(5): + self.posts.append(Post(id=len(self.posts) + 1, user_id=u_id, content=f"Post {p_idx} by {u_id}")) + +class MockConnection: + """Simulates a single database connection.""" + def __init__(self, pool: 'MockPool', state: MockDatabaseState): + self.pool = pool + self.state = state + self._transaction_buffer: List[Tuple[int, str]] = [] # Stores (user_id, new_label) + self._in_transaction = False + + async def execute(self, query: str, params: Tuple[Any, ...]) -> int: + """Simulates executing a command. Returns number of rows affected.""" + # Simulate network latency + await asyncio.sleep(0.01) + + # Parameterized Query Check: + # In a real driver, the query string contains placeholders (?), not values. + # We simulate an error if someone tries to pass a query that looks like it was interpolated. + if "'" in query and not any(isinstance(p, str) for p in params): + # This is a naive check to demonstrate the concept of preventing injection + pass + + if "UPDATE users SET label =" in query: + new_label, user_id = params + if user_id not in self.state.users: + return 0 # No rows affected (user doesn't exist) + + if self._in_transaction: + self._transaction_buffer.append((user_id, new_label)) + else: + # Autocommit mode + self.state.users[user_id].label = new_label + return 1 + + return 0 + + async def fetch(self, query: str, params: Tuple[Any, ...]) -> List[Dict[str, Any]]: + """Simulates fetching rows.""" + await asyncio.sleep(0.01) + + if "SELECT * FROM users" in query: + # Handle Pagination logic via params (LIMIT, OFFSET) + # Note: In a real app, we'd parse the SQL. Here we simulate based on params. + if "WHERE id =" in query: + u_id = params[0] + user = self.state.users.get(u_id) + return [{"id": user.id, "name": user.name, "label": user.label}] if user else [] + + # Pagination: params might be (limit, offset) or just empty + limit = params[0] if len(params) > 0 else 10 + offset = params[1] if len(params) > 1 else 0 + + all_users = list(self.state.users.values()) + sliced = all_users[offset : offset + limit] + return [{"id": u.id, "name": u.name, "label": u.label} for u in sliced] + + if "SELECT COUNT(*) FROM users" in query: + return [{"count": len(self.state.users)}] + + if "SELECT * FROM posts WHERE user_id =" in query: + u_id = params[0] + user_posts = [p for p in self.state.posts if p.user_id == u_id] + return [{"id": p.id, "content": p.content} for p in user_posts] + + return [] + + async def begin(self): + self._in_transaction = True + self._transaction_buffer = [] + + async def commit(self): + if not self._in_transaction: + return + # Apply buffered updates to the real state + for user_id, new_label in self._transaction_buffer: + self.state.users[user_id].label = new_label + self._in_transaction = False + self._transaction_buffer = [] + + async def rollback(self): + self._transaction_buffer = [] + self._in_transaction = False + + async def release(self): + await self.pool._release_connection(self) + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.release() + +class MockPool: + """Simulates a connection pool with a fixed size.""" + def __init__(self, max_size: int, state: MockDatabaseState): + self.max_size = max_size + self.state = state + self._semaphore = asyncio.Semaphore(max_size) + self._in_use_count = 0 + + async def acquire(self) -> MockConnection: + try: + # Requirement 5: Respect max-wait (2s) + await asyncio.wait_for(self._semaphore.acquire(), timeout=2.0) + self._in_use_count += 1 + return MockConnection(self, self.state) + except asyncio.TimeoutError: + raise PoolExhaustedError("Could not acquire connection: Pool exhausted.") + + async def _release_connection(self, conn: MockConnection): + self._in_use_count -= 1 + self._semaphore.release() + + @property + def in_use(self) -> int: + return self._in_use_count + +# ============================================================================= +# DATA ACCESS SERVICE +# ============================================================================= + +class UserService: + def __init__(self, pool: MockPool): + self.pool = pool + + async def get_users(self, page: int, page_size: int) -> Dict[str, Any]: + """Requirement 1 & 3: Pagination and Connection Discipline.""" + if page < 1 or not (1 <= page_size <= 100): + raise ValidationError("Invalid page or page_size. Page >= 1, 1 <= size <= 100.") + + async with await self.pool.acquire() as conn: + # Get total count + count_res = await conn.fetch("SELECT COUNT(*) FROM users", ()) + total = count_res[0]["count"] + + # Calculate offset + offset = (page - 1) * page_size + + # Get items + items = await conn.fetch("SELECT * FROM users LIMIT ? OFFSET ?", (page_size, offset)) + + return { + "items": items, + "page": page, + "page_size": page_size, + "total": total + } + + async def get_user_with_posts(self, user_id: int) -> Dict[str, Any]: + """Requirement 2: Efficient fetching (no N+1).""" + async with await self.pool.acquire() as conn: + user_res = await conn.fetch("SELECT * FROM users WHERE id = ?", (user_id,)) + if not user_res: + raise KeyError(f"User with id {user_id} not found.") + + user = user_res[0] + # Fetch posts in the same connection context + posts = await conn.fetch("SELECT * FROM posts WHERE user_id = ?", (user_id,)) + + return {**user, "posts": posts} + + async def relabel_users(self, pairs: List[Tuple[int, str]]) -> int: + """Requirement 4: Transactional bulk update.""" + async with await self.pool.acquire() as conn: + await conn.begin() + try: + total_updated = 0 + for user_id, new_label in pairs: + # Requirement 2: Parameterized queries (passing params as tuple) + rows_affected = await conn.execute( + "UPDATE users SET label = ? WHERE id = ?", + (new_label, user_id) + ) + if rows_affected == 0: + # If any update fails (user doesn't exist), trigger rollback + raise TransactionError(f"Update failed: User {user_id} not found.") + total_updated += rows_affected + + await conn.commit() + return total_updated + except Exception as e: + await conn.rollback() + # Re-raise or return 0 based on requirement. We raise to be explicit. + raise e + +# ============================================================================= +# TEST SUITE +# ============================================================================= + +async def run_tests(): + print("🚀 Starting Tests...\n") + state = MockDatabaseState() + pool = MockPool(max_size=5, state=state) + service = UserService(pool) + + # --- Test A: Connection Leak Check --- + print("Test A: Checking for connection leaks (100 sequential calls)...") + for _ in range(100): + await service.get_users(page=1, page_size=10) + assert pool.in_use == 0, f"Leak detected! Connections in use: {pool.in_use}" + print("✅ No leaks detected.") + + # --- Test B: Pagination Math --- + print("\nTest B: Verifying pagination math...") + # Total users is 25. Page size 10. + # Page 1: 1-10, Page 2: 11-20, Page 3: 21-25 + res_p1 = await service.get_users(page=1, page_size=10) + assert res_p1["total"] == 25 + assert len(res_p1["items"]) == 10 + + res_p3 = await service.get_users(page=3, page_size=10) + assert len(res_p3["items"]) == 5 # Remainder + + res_out = await service.get_users(page=10, page_size=10) + assert len(res_out["items"]) == 0 # Out of range + print("✅ Pagination math is correct.") + + # --- Test C: Transactional Integrity (Rollback) --- + print("\nTest C: Verifying transactional rollback...") + # Initial state check for user 1 + user_1_before = (await service.get_user_with_posts(1))["label"] + + # Attempt bulk update: User 2 is valid, User 999 is invalid. + # This should cause the whole batch to fail. + try: + await service.relabel_users([(2, "new_label"), (999, "fail_label")]) + except TransactionError: + pass # Expected + + user_1_after = (await service.get_user_with_posts(1))["label"] + user_2_after = (await service.get_user_with_posts(2))["label"] + + assert user_1_after == user_1_before, "User 1 changed despite transaction failure!" + assert user_2_after == "default", "User 2 changed despite transaction failure!" + print("✅ Transaction rolled back successfully. No partial updates applied.") + + # --- Test D: Concurrency & Pool Exhaustion --- + print("\nTest D: Verifying concurrency (10 concurrent requests on pool of 5)...") + # We trigger 10 tasks. Since max_size is 5, some will wait for the semaphore. + # This tests that they don't crash and eventually complete. + tasks = [service.get_user_with_posts(i) for i in range(1, 11)] + results = await asyncio.gather(*tasks) + assert len(results) == 10 + assert pool.in_use == 0 + print("✅ Concurrent requests completed successfully.") + + # --- Test E: Parameterized Query / Injection Simulation --- + print("\nTest E: Verifying parameterized query usage...") + # We check if the service correctly handles a "malicious" string by passing it as a param + # rather than interpolating it. + malicious_label = "'; DROP TABLE users; --" + await service.relabel_users([(5, malicious_label)]) + user_5 = await service.get_user_with_posts(5) + assert user_5["label"] == malicious_label, "The label was not treated as a literal string!" + print("✅ Parameterized query logic verified.") + + print("\n✨ ALL TESTS PASSED! ✨") + +if __name__ == "__main__": + try: + asyncio.run(run_tests()) + except KeyboardInterrupt: + pass \ No newline at end of file diff --git a/tools/grade_run.py b/tools/grade_run.py index ac801cb..08eb0c1 100755 --- a/tools/grade_run.py +++ b/tools/grade_run.py @@ -25,6 +25,13 @@ WHAT IT DOES 5. Optionally --append writes a placeholder entry to data/benchmark_history.json with total_score:null, verdict:"pending", so the audit is the only manual bit. +STATS COME FROM THE NATIVE API, NOT OPENAI-COMPAT. + LM Studio's OpenAI-compatible endpoint (/v1/chat/completions) returns an EMPTY + stats object — use the native /api/v1/chat endpoint instead, which returns a + full stats block including time_to_first_token_seconds. Request shape differs + from OpenAI: pass {"model":..., "input": "", ...} (input is a + string or array of input items, NOT messages). + NOTE: This only COLLECTS facts (runnability + metrics + file). The actual 5-pillar audit is still done by the grader (me/you) — that part is subjective and shouldn't be faked. @@ -154,37 +161,41 @@ def main(): sys.exit(f"prompt file not found: {ppath}") prompt_text = open(ppath).read() - print(f"→ POST {args.lmstudio}/v1/chat/completions model={args.model} prompt={args.prompt}") + # NATIVE endpoint /api/v1/chat (not OpenAI-compat). The OpenAI endpoint + # /v1/chat/completions returns an EMPTY stats object; the native endpoint + # returns full stats including time_to_first_token_seconds. Request shape: + # {"model":..., "input": "", ...} — input is a string, NOT messages. + # If stream:true is ever needed, parse the SSE `chat.end` event's result.stats + # (identical schema to the non-streaming stats block below). + print(f"→ POST {args.lmstudio}/api/v1/chat model={args.model} prompt={args.prompt}") payload = { "model": args.model, - "messages": [{"role": "user", "content": prompt_text}], + "input": prompt_text, "temperature": 0.2, - "max_tokens": args.max_tokens, + "max_output_tokens": args.max_tokens, # native key (NOT max_tokens — that 400s) "stream": False, } try: - resp, wall = api(args.lmstudio, "/v1/chat/completions", payload) + resp, wall = api(args.lmstudio, "/api/v1/chat", payload) except urllib.error.URLError as e: sys.exit(f"LM Studio not reachable at {args.lmstudio} — is the server started and on 0.0.0.0? ({e})") - content = resp["choices"][0]["message"]["content"] - usage = resp.get("usage", {}) - # LM Studio reports timing under a stats/timings object — key names vary by - # version, so self-discover: walk known containers and pick the first numeric hit. - def _find(obj, keys): - for k in keys: - v = obj.get(k) - if isinstance(v, (int, float)): - return v - return None + # native response: { model_instance_id, output:[ {type:"message",content:...}, ... ], + # stats:{ input_tokens, total_output_tokens, reasoning_output_tokens, + # tokens_per_second, time_to_first_token_seconds, model_load_time_seconds }, + # response_id } + out_items = resp.get("output", []) + content = "" + for it in out_items: + if it.get("type") == "message": + content = it.get("content", "") + break stats = resp.get("stats") or {} - timings = resp.get("timings") or {} - # try stats then timings for each metric - tok_sec = _find(stats, ["tokens_per_second", "tokensPerSecond", "predicted_per_second", "predicted_tokens_per_second"]) \ - or _find(timings, ["predicted_per_second", "tokens_per_second", "predicted_n"]) - ttft = _find(stats, ["time_to_first_token", "timeToFirstToken", "ttft", "first_token_time"]) \ - or _find(timings, ["time_to_first_token", "prompt_n", "prompt_per_second", "first_token"]) - comp_tokens = usage.get("completion_tokens") or usage.get("completion_tokens_details", {}).get("reasoning_tokens", 0) + tok_sec = stats.get("tokens_per_second") + ttft = stats.get("time_to_first_token_seconds") + comp_tokens = stats.get("total_output_tokens") + input_tokens = stats.get("input_tokens") + total_tokens = (input_tokens or 0) + (comp_tokens or 0) if (input_tokens or comp_tokens) else None # fallback: derive tok/sec from wall time if the server didn't report it if tok_sec is None and comp_tokens and wall: tok_sec = round(comp_tokens / wall, 2) @@ -192,7 +203,8 @@ def main(): if not os.environ.get("GRADE_RUN_NO_SCHEMA_DUMP"): try: schema_path = os.path.join(HERE, "outputs", ".last_response_schema.json") - slim = {k: v for k, v in resp.items() if k not in ("choices", "id", "object", "model", "created")} + slim = {k: v for k, v in resp.items() if k != "output"} + slim["_output_types"] = [it.get("type") for it in out_items] json.dump(slim, open(schema_path, "w"), indent=2) except Exception: pass @@ -222,10 +234,10 @@ def main(): open(out_path, "w").write(text) print(f"✓ saved {len(text)} bytes -> {os.path.relpath(out_path, HERE)}") - print("\n=== METRICS (from API) ===") + print("\n=== METRICS (from native /api/v1/chat stats) ===") print(f" tok/sec : {tok_sec}") - print(f" total tokens : {usage.get('total_tokens')}") - print(f" TTFT : {ttft}") + print(f" total tokens : {total_tokens} (in={input_tokens} out={comp_tokens})") + print(f" TTFT (sec) : {ttft}") print("\n=== DRAFT JSON ENTRY (fill breakdown + audit by hand) ===") entry = { @@ -236,7 +248,7 @@ def main(): "quant": "TODO", "format": "mlx", "tok_sec": tok_sec, - "total_tokens": usage.get("total_tokens"), + "total_tokens": total_tokens, "ttft_sec": ttft, "filename": f"outputs/{args.name}.{ext}", "tests_pass": None, # grader runs the file