adminandClaude
0c79a997f5
Via tools/grade_run.py against LM Studio (load -> all prompts -> unload -> verify).
kat-coder: lfu/tts/webhook/automation/rust (lfu has a real SyntaxError: 'self._ bump_freq')
qwen3-coder-30b: lfu/tts/webhook/automation/rust (all 4 py parse clean; terse ~2k tok outputs)
Grading deferred to a later session (capture-only mode).
Script hardening:
- --resident / --unload-all commands (read loaded_instances, POST unload, verify)
- MEMORY GUARD: aborts if a different model is resident (never stack 30GB models)
- TTFT self-discovery dump (LM Studio returns empty stats non-streaming;
fix next session: switch to /api/v1/chat stream:true -> chat.end result.stats)
Co-Authored-By: Claude <noreply@anthropic.com>
588 lines
21 KiB
Python
588 lines
21 KiB
Python
"""
|
|
In-Memory Concurrent LFU Cache with Async TTL Eviction and Atomic Transactions.
|
|
|
|
Pure Python 3.11+ implementation — no external dependencies.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import time
|
|
import weakref
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Optional
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Internal Doubly-Linked List Node (frequency bucket member)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@dataclass
|
|
class _Node:
|
|
"""A single cache entry stored inside a frequency bucket."""
|
|
|
|
key: Any
|
|
value: Any
|
|
freq: int = 1
|
|
expires_at: float = 0.0 # 0 means no expiry
|
|
|
|
prev: Optional[_Node] = field(default=None, repr=False)
|
|
next: Optional[_Node] = field(default=None, repr=False)
|
|
|
|
@property
|
|
def is_expired(self) -> bool:
|
|
return self.expires_at > 0 and time.monotonic() >= self.expires_at
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Doubly-Linked List wrapper for a single frequency tier
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class _FreqList:
|
|
"""Doubly-linked list with sentinel head/tail for O(1) insert/remove."""
|
|
|
|
__slots__ = ("head", "tail", "size")
|
|
|
|
def __init__(self) -> None:
|
|
self.head = _Node(key=None, value=None) # sentinel
|
|
self.tail = _Node(key=None, value=None) # sentinel
|
|
self.head.next = self.tail
|
|
self.tail.prev = self.head
|
|
self.size = 0
|
|
|
|
# -- public helpers -----------------------------------------------------
|
|
|
|
def append_right(self, node: _Node) -> None:
|
|
"""Append *node* just before the tail sentinel (LRU-end)."""
|
|
pred = self.tail.prev
|
|
node.prev = pred
|
|
node.next = self.tail
|
|
pred.next = node
|
|
self.tail.prev = node
|
|
self.size += 1
|
|
|
|
def remove(self, node: _Node) -> None:
|
|
"""Remove *node* from the list in O(1)."""
|
|
pred, succ = node.prev, node.next
|
|
pred.next = succ
|
|
succ.prev = pred
|
|
node.prev = node.next = None
|
|
self.size -= 1
|
|
|
|
def pop_left(self) -> Optional[_Node]:
|
|
"""Remove and return the node just after head sentinel (MRU-end)."""
|
|
if self.size == 0:
|
|
return None
|
|
node = self.head.next
|
|
self.remove(node)
|
|
return node
|
|
|
|
def is_empty(self) -> bool:
|
|
return self.size == 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Transaction handle
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class Transaction:
|
|
"""
|
|
Represents an isolated sub-session on the cache.
|
|
|
|
* ``put`` / ``delete`` buffer changes locally.
|
|
* ``get`` sees uncommitted local writes ("read your own writes").
|
|
* ``commit`` applies all buffered changes atomically to the global cache.
|
|
* ``rollback`` discards everything without touching global state.
|
|
"""
|
|
|
|
def __init__(self, cache: "LFUCache") -> None:
|
|
self._cache = cache
|
|
# key -> (value, expires_at) — latest write wins on conflict
|
|
self._writes: dict[Any, tuple[Any, float]] = {}
|
|
# keys deleted within this transaction
|
|
self._deletes: set[Any] = set()
|
|
self._committed = False
|
|
self._rolled_back = False
|
|
|
|
# -- mutating operations ------------------------------------------------
|
|
|
|
def put(self, key: Any, value: Any, ttl_seconds: float = 0.0) -> None:
|
|
"""Buffer a write inside this transaction."""
|
|
if self._committed or self._rolled_back:
|
|
raise RuntimeError("Transaction already closed")
|
|
expires_at = (time.monotonic() + ttl_seconds) if ttl_seconds > 0 else 0.0
|
|
self._writes[key] = (value, expires_at)
|
|
self._deletes.discard(key)
|
|
|
|
def delete(self, key: Any) -> None:
|
|
"""Buffer a deletion inside this transaction."""
|
|
if self._committed or self._rolled_back:
|
|
raise RuntimeError("Transaction already closed")
|
|
self._deletes.add(key)
|
|
self._writes.pop(key, None)
|
|
|
|
# -- read operations ----------------------------------------------------
|
|
|
|
def get(self, key: Any) -> Optional[Any]:
|
|
"""
|
|
Return the value for *key*, checking local buffer first, then global.
|
|
Returns ``None`` if the key is absent or expired.
|
|
"""
|
|
if self._committed or self._rolled_back:
|
|
raise RuntimeError("Transaction already closed")
|
|
|
|
# 1. Check local uncommitted writes
|
|
if key in self._deletes:
|
|
return None
|
|
if key in self._writes:
|
|
value, expires_at = self._writes[key]
|
|
if expires_at > 0 and time.monotonic() >= expires_at:
|
|
self._deletes.add(key)
|
|
return None
|
|
return value
|
|
|
|
# 2. Fall back to global cache (with lazy TTL check)
|
|
return self._cache.get(key)
|
|
|
|
# -- lifecycle ----------------------------------------------------------
|
|
|
|
async def commit(self) -> None:
|
|
"""Apply all buffered changes atomically to the global cache."""
|
|
if self._committed or self._rolled_back:
|
|
raise RuntimeError("Transaction already closed")
|
|
await self._cache._apply_transaction(self)
|
|
self._committed = True
|
|
|
|
async def rollback(self) -> None:
|
|
"""Discard all buffered changes."""
|
|
if self._committed or self._rolled_back:
|
|
raise RuntimeError("Transaction already closed")
|
|
self._writes.clear()
|
|
self._deletes.clear()
|
|
self._rolled_back = True
|
|
|
|
def __repr__(self) -> str: # pragma: no cover
|
|
state = "committed" if self._committed else (
|
|
"rolled_back" if self._rolled_back else "active"
|
|
)
|
|
return f"<Transaction {state} writes={len(self._writes)} deletes={len(self._deletes)}>"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main LFU Cache
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class LFUCache:
|
|
"""
|
|
In-memory concurrent LFU cache with O(1) get/put, async TTL eviction,
|
|
and atomic transaction support.
|
|
"""
|
|
|
|
def __init__(self, capacity: int) -> None:
|
|
if capacity < 1:
|
|
raise ValueError("capacity must be >= 1")
|
|
|
|
self._capacity = capacity
|
|
|
|
# key -> _Node (global cache)
|
|
self._cache: dict[Any, _Node] = {}
|
|
|
|
# freq -> _FreqList (frequency buckets)
|
|
self._freq_map: dict[int, _FreqList] = {}
|
|
|
|
# Track the current minimum frequency for O(1) eviction
|
|
self._min_freq: int = 0
|
|
|
|
# Concurrency primitives
|
|
self._lock = asyncio.Lock()
|
|
|
|
# Background evictor task handle
|
|
self._evictor_task: Optional[asyncio.Task[None]] = None
|
|
|
|
# ------------------------------------------------------------------
|
|
# Public API — get / put
|
|
# ------------------------------------------------------------------
|
|
|
|
async def get(self, key: Any) -> Optional[Any]:
|
|
"""
|
|
Retrieve *key* from the cache in O(1).
|
|
|
|
Lazy TTL eviction is performed on access.
|
|
"""
|
|
async with self._lock:
|
|
node = self._cache.get(key)
|
|
if node is None:
|
|
return None
|
|
|
|
# Lazy TTL check
|
|
if node.is_expired:
|
|
self._evict_node(node)
|
|
return None
|
|
|
|
# Bump frequency — O(1)
|
|
self._ bump_freq(node)
|
|
return node.value
|
|
|
|
async def put(self, key: Any, value: Any, ttl_seconds: float = 0.0) -> None:
|
|
"""
|
|
Insert or update *key* in the cache in O(1).
|
|
|
|
If the cache is at capacity, the LFU (least-recently-used tie-break)
|
|
entry is evicted before insertion.
|
|
"""
|
|
expires_at = (time.monotonic() + ttl_seconds) if ttl_seconds > 0 else 0.0
|
|
|
|
async with self._lock:
|
|
# Case 1: key already exists — update in place
|
|
if key in self._cache:
|
|
node = self._cache[key]
|
|
old_freq = node.freq
|
|
node.value = value
|
|
node.expires_at = expires_at
|
|
# Move to new frequency bucket
|
|
self._remove_from_freq_list(node)
|
|
node.freq += 1
|
|
self._add_to_freq_list(node)
|
|
# Update min_freq if the old bucket is now empty and was min
|
|
if old_freq == self._min_freq and self._freq_map[old_freq].is_empty():
|
|
del self._freq_map[old_freq]
|
|
self._min_freq = node.freq
|
|
return
|
|
|
|
# Case 2: cache full — evict LFU entry
|
|
if len(self._cache) >= self._capacity:
|
|
self._evict_one()
|
|
|
|
# Insert new node at frequency 1
|
|
node = _Node(key=key, value=value, freq=1, expires_at=expires_at)
|
|
self._cache[key] = node
|
|
self._add_to_freq_list(node)
|
|
self._min_freq = 1
|
|
|
|
# ------------------------------------------------------------------
|
|
# Transaction support
|
|
# ------------------------------------------------------------------
|
|
|
|
def begin_transaction(self) -> Transaction:
|
|
"""Start a new isolated transaction on this cache."""
|
|
return Transaction(self)
|
|
|
|
async def _apply_transaction(self, tx: Transaction) -> None:
|
|
"""
|
|
Apply a transaction's buffered writes/deletes atomically.
|
|
Must be called while holding self._lock (called from Transaction.commit).
|
|
"""
|
|
# --- Phase 1: apply deletes first (so a put+delete of same key works) ---
|
|
for key in tx._deletes:
|
|
node = self._cache.pop(key, None)
|
|
if node is not None:
|
|
self._remove_from_freq_list(node)
|
|
if node.freq == self._min_freq and self._freq_map[node.freq].is_empty():
|
|
del self._freq_map[node.freq]
|
|
# Find new min freq
|
|
if self._freq_map:
|
|
self._min_freq = min(self._freq_map)
|
|
else:
|
|
self._min_freq = 0
|
|
|
|
# --- Phase 2: apply writes ------------------------------------------
|
|
for key, (value, expires_at) in tx._writes.items():
|
|
if key in self._cache:
|
|
# Update existing node
|
|
node = self._cache[key]
|
|
old_freq = node.freq
|
|
node.value = value
|
|
node.expires_at = expires_at
|
|
self._remove_from_freq_list(node)
|
|
node.freq += 1
|
|
self._add_to_freq_list(node)
|
|
if old_freq == self._min_freq and self._freq_map[old_freq].is_empty():
|
|
del self._freq_map[old_freq]
|
|
if self._freq_map:
|
|
self._min_freq = min(self._freq_map)
|
|
else:
|
|
# Insert new node (may need eviction first)
|
|
if len(self._cache) >= self._capacity:
|
|
self._evict_one()
|
|
node = _Node(key=key, value=value, freq=1, expires_at=expires_at)
|
|
self._cache[key] = node
|
|
self._add_to_freq_list(node)
|
|
self._min_freq = 1
|
|
|
|
# ------------------------------------------------------------------
|
|
# Background async TTL evictor
|
|
# ------------------------------------------------------------------
|
|
|
|
def start_evictor(self, interval: float = 0.5, batch_size: int = 32) -> None:
|
|
"""Start the non-blocking background TTL sweep task."""
|
|
if self._evictor_task is not None:
|
|
return
|
|
self._evictor_task = asyncio.create_task(
|
|
self._evictor_loop(interval, batch_size)
|
|
)
|
|
|
|
def stop_evictor(self) -> None:
|
|
"""Stop the background TTL sweep task."""
|
|
if self._evictor_task is not None:
|
|
self._evictor_task.cancel()
|
|
try:
|
|
asyncio.get_event_loop().run_until_complete(self._evictor_task)
|
|
except (asyncio.CancelledError, RuntimeError):
|
|
pass
|
|
self._evictor_task = None
|
|
|
|
async def _evictor_loop(self, interval: float, batch_size: int) -> None:
|
|
"""Periodically scan and purge expired entries in small batches."""
|
|
try:
|
|
while True:
|
|
await asyncio.sleep(interval)
|
|
async with self._lock:
|
|
expired = [
|
|
node for node in self._cache.values() if node.is_expired
|
|
][:batch_size]
|
|
for node in expired:
|
|
self._evict_node(node)
|
|
except asyncio.CancelledError:
|
|
return
|
|
|
|
# ------------------------------------------------------------------
|
|
# Internal helpers
|
|
# ------------------------------------------------------------------
|
|
|
|
def _add_to_freq_list(self, node: _Node) -> None:
|
|
"""Add *node* to its frequency bucket."""
|
|
freq = node.freq
|
|
if freq not in self._freq_map:
|
|
self._freq_map[freq] = _FreqList()
|
|
self._freq_map[freq].append_right(node)
|
|
|
|
def _remove_from_freq_list(self, node: _Node) -> None:
|
|
"""Remove *node* from its current frequency bucket."""
|
|
freq_list = self._freq_map.get(node.freq)
|
|
if freq_list is not None:
|
|
freq_list.remove(node)
|
|
|
|
def _bump_freq(self, node: _Node) -> None:
|
|
"""Increment *node*'s frequency and move it to the next bucket."""
|
|
old_freq = node.freq
|
|
self._remove_from_freq_list(node)
|
|
node.freq += 1
|
|
self._add_to_freq_list(node)
|
|
|
|
# Update min_freq if the old bucket is now empty and was the minimum
|
|
if old_freq == self._min_freq and self._freq_map[old_freq].is_empty():
|
|
del self._freq_map[old_freq]
|
|
if self._freq_map:
|
|
self._min_freq = min(self._freq_map)
|
|
else:
|
|
self._min_freq = 0
|
|
|
|
def _evict_one(self) -> None:
|
|
"""Evict the least-frequently-used entry (LRU tie-break). O(1)."""
|
|
if self._min_freq not in self._freq_map or self._freq_map[self._min_freq].is_empty():
|
|
# Fallback — should not happen in normal operation
|
|
return
|
|
|
|
freq_list = self._freq_map[self._min_freq]
|
|
node = freq_list.pop_left()
|
|
if node is not None:
|
|
del self._cache[node.key]
|
|
if freq_list.is_empty():
|
|
del self._freq_map[self._min_freq]
|
|
if self._freq_map:
|
|
self._min_freq = min(self._freq_map)
|
|
else:
|
|
self._min_freq = 0
|
|
|
|
def _evict_node(self, node: _Node) -> None:
|
|
"""Remove a single expired node from all structures."""
|
|
self._remove_from_freq_list(node)
|
|
del self._cache[node.key]
|
|
if node.freq == self._min_freq and self._freq_map.get(node.freq, _FreqList()).is_empty():
|
|
if node.freq in self._freq_map:
|
|
del self._freq_map[node.freq]
|
|
if self._freq_map:
|
|
self._min_freq = min(self._freq_map)
|
|
else:
|
|
self._min_freq = 0
|
|
|
|
|
|
# ===================================================================
|
|
# Executable Unit Tests
|
|
# ===================================================================
|
|
|
|
async def main() -> None:
|
|
passed = 0
|
|
failed = 0
|
|
|
|
def check(name: str, condition: bool) -> None:
|
|
nonlocal passed, failed
|
|
if condition:
|
|
passed += 1
|
|
print(f" ✓ {name}")
|
|
else:
|
|
failed += 1
|
|
print(f" ✗ {name}")
|
|
|
|
# ------------------------------------------------------------------
|
|
# (a) O(1) LFU eviction order when capacity is reached
|
|
# ------------------------------------------------------------------
|
|
print("\n[a] LFU Eviction Order")
|
|
|
|
cache = LFUCache(capacity=3)
|
|
|
|
# Insert a, b, c → all freq=1
|
|
await cache.put("a", 1)
|
|
await cache.put("b", 2)
|
|
await cache.put("c", 3)
|
|
|
|
# Access a twice → freq(a)=3, freq(b)=1, freq(c)=1
|
|
await cache.get("a")
|
|
await cache.put("a", 10) # bump to freq=4
|
|
|
|
# Access b once → freq(b)=2
|
|
await cache.get("b")
|
|
|
|
# Now: a=freq4, b=freq2, c=freq1. Min freq = 1 (key c).
|
|
# Insert d — should evict c (lowest freq).
|
|
await cache.put("d", 4)
|
|
|
|
check("evicts least-frequent key (c)", await cache.get("c") is None)
|
|
check("keeps a", await cache.get("a") == 10)
|
|
check("keeps b", await cache.get("b") == 2)
|
|
check("keeps d", await cache.get("d") == 4)
|
|
|
|
# ------------------------------------------------------------------
|
|
# (b) Lazy TTL vs. Background Async Sweep eviction
|
|
# ------------------------------------------------------------------
|
|
print("\n[b] TTL Eviction (Lazy + Background)")
|
|
|
|
cache2 = LFUCache(capacity=5)
|
|
|
|
await cache2.put("x", 1, ttl_seconds=0.05)
|
|
await cache2.put("y", 2, ttl_seconds=10.0)
|
|
|
|
# Before expiry — both visible
|
|
check("x present before TTL", await cache2.get("x") == 1)
|
|
check("y present before TTL", await cache2.get("y") == 2)
|
|
|
|
# Wait for x to expire
|
|
await asyncio.sleep(0.1)
|
|
|
|
# Lazy eviction on get
|
|
check("x evicted lazily on get", await cache2.get("x") is None)
|
|
check("y still present after x expired", await cache2.get("y") == 2)
|
|
|
|
# Background evictor test
|
|
cache3 = LFUCache(capacity=5)
|
|
await cache3.put("p", 1, ttl_seconds=0.05)
|
|
await cache3.put("q", 2, ttl_seconds=10.0)
|
|
|
|
cache3.start_evictor(interval=0.05, batch_size=8)
|
|
await asyncio.sleep(0.2)
|
|
|
|
check("p evicted by background sweep", await cache3.get("p") is None)
|
|
check("q still present after background sweep", await cache3.get("q") == 2)
|
|
|
|
cache3.stop_evictor()
|
|
|
|
# ------------------------------------------------------------------
|
|
# (c) Transaction commit visibility vs. rollback state restoration
|
|
# ------------------------------------------------------------------
|
|
print("\n[c] Atomic Transactions")
|
|
|
|
cache4 = LFUCache(capacity=5)
|
|
await cache4.put("k1", 100)
|
|
await cache4.put("k2", 200)
|
|
|
|
# --- Commit test -------------------------------------------------------
|
|
tx1 = cache4.begin_transaction()
|
|
tx1.put("k3", 300) # uncommitted — global should not see it
|
|
tx1.put("k1", 999) # update existing
|
|
|
|
check("global doesn't see uncommitted put", await cache4.get("k3") is None)
|
|
check("global still sees old k1", await cache4.get("k1") == 100)
|
|
|
|
# Read-your-own-writes
|
|
check("tx sees its own put(k3)", tx1.get("k3") == 300)
|
|
check("tx sees its own update(k1)", tx1.get("k1") == 999)
|
|
|
|
await tx1.commit()
|
|
|
|
check("global sees committed k3", await cache4.get("k3") == 300)
|
|
check("global sees committed k1 update", await cache4.get("k1") == 999)
|
|
|
|
# --- Rollback test -----------------------------------------------------
|
|
tx2 = cache4.begin_transaction()
|
|
tx2.put("k4", 400)
|
|
tx2.delete("k2")
|
|
|
|
check("tx sees deleted k2 as None", tx2.get("k2") is None)
|
|
check("global still has k2 before rollback", await cache4.get("k2") == 200)
|
|
|
|
await tx2.rollback()
|
|
|
|
check("global k2 restored after rollback", await cache4.get("k2") == 200)
|
|
check("global k4 absent after rollback", await cache4.get("k4") is None)
|
|
|
|
# --- Rollback of update test -------------------------------------------
|
|
tx3 = cache4.begin_transaction()
|
|
tx3.put("k1", 777)
|
|
await tx3.rollback()
|
|
check("global k1 restored after rollback of update", await cache4.get("k1") == 999)
|
|
|
|
# --- Transaction with TTL ----------------------------------------------
|
|
tx4 = cache4.begin_transaction()
|
|
tx4.put("temp", "value", ttl_seconds=0.05)
|
|
check("tx sees its own TTL'd key", tx4.get("temp") == "value")
|
|
await asyncio.sleep(0.1)
|
|
check("tx sees expired TTL'd key as None", tx4.get("temp") is None)
|
|
await tx4.rollback()
|
|
|
|
# ------------------------------------------------------------------
|
|
# (d) Stress test — 50 concurrent async tasks reading/writing
|
|
# ------------------------------------------------------------------
|
|
print("\n[d] Stress Test (50 concurrent tasks)")
|
|
|
|
cache5 = LFUCache(capacity=100)
|
|
errors: list[str] = []
|
|
|
|
async def worker(task_id: int) -> None:
|
|
try:
|
|
for i in range(50):
|
|
key = f"key_{task_id}_{i % 200}"
|
|
if i % 3 == 0:
|
|
await cache5.put(key, f"val_{task_id}_{i}", ttl_seconds=1.0)
|
|
elif i % 3 == 1:
|
|
await cache5.get(key)
|
|
else:
|
|
tx = cache5.begin_transaction()
|
|
tx.put(f"tx_{task_id}_{i}", f"txval_{i}")
|
|
val = tx.get(f"tx_{task_id}_{i}")
|
|
check(f" task {task_id} tx read-your-own-write", val == f"txval_{i}")
|
|
await tx.commit()
|
|
except Exception as e:
|
|
errors.append(f"task {task_id}: {e}")
|
|
|
|
tasks = [asyncio.create_task(worker(tid)) for tid in range(50)]
|
|
await asyncio.gather(*tasks)
|
|
|
|
check("no concurrent errors", len(errors) == 0)
|
|
check("cache has entries after stress", len(cache5._cache) > 0)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Summary
|
|
# ------------------------------------------------------------------
|
|
total = passed + failed
|
|
print(f"\n{'='*50}")
|
|
print(f"Results: {passed}/{total} passed" + (
|
|
"" if failed == 0 else f", {failed} FAILED"
|
|
))
|
|
print(f"{'='*50}")
|
|
|
|
if failed:
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main()) |