Initial benchmark suite: 8 graded models + cyberpunk dashboard generator
- prompts/: LFU cache exam + 5-pillar grading rubric - outputs/: 8 model .py outputs (local + cloud baseline) - data/benchmark_history.json: graded results (scores, metrics, bugs, patches) - generate_dashboard.py: builds dashboard.html + pages/*.html from JSON - Dockerfile + DEPLOY.md: Gitea→Coolify deploy (build-step, nginx static) - .gitignore: generated HTML excluded (built on deploy) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,343 @@
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional, Generic, TypeVar, Set
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
# --- Internal Data Structures for O(1) LFU ---
|
||||
|
||||
@dataclass
|
||||
class Node:
|
||||
"""A node in the LFU frequency doubly linked list."""
|
||||
key: Any
|
||||
value: Any
|
||||
freq: int = 1
|
||||
expiry: Optional[float] = None
|
||||
prev: Optional['Node'] = None
|
||||
next: Optional['Node'] = None
|
||||
|
||||
class DoublyLinkedList:
|
||||
"""A standard DLL to maintain nodes of the same frequency."""
|
||||
def __init__(self):
|
||||
self.head: Optional[Node] = None
|
||||
self.tail: Optional[Node] = None
|
||||
self.size = 0
|
||||
|
||||
def append(self, node: Node):
|
||||
"""Add node to the front (most recently used in this freq bucket)."""
|
||||
node.next = self.head
|
||||
node.prev = None
|
||||
if self.head:
|
||||
self.head.prev = node
|
||||
self.head = node
|
||||
if not self.tail:
|
||||
self.tail = node
|
||||
self.size += 1
|
||||
|
||||
def pop_tail(self) -> Optional[Node]:
|
||||
"""Remove and return the least recently used node in this freq bucket."""
|
||||
if not self.tail:
|
||||
return None
|
||||
node = self.tail
|
||||
self.remove(node)
|
||||
return node
|
||||
|
||||
def remove(self, node: Node):
|
||||
"""Remove a specific node from the list in O(1)."""
|
||||
if node.prev:
|
||||
node.prev.next = node.next
|
||||
else:
|
||||
self.head = node.next
|
||||
|
||||
if node.next:
|
||||
node.next.prev = node.prev
|
||||
else:
|
||||
self.tail = node.prev
|
||||
|
||||
node.next = None
|
||||
node.prev = None
|
||||
self.size -= 1
|
||||
|
||||
# --- Cache Implementation ---
|
||||
|
||||
class LFUCache:
|
||||
def __init__(self, capacity: int):
|
||||
if capacity <= 0:
|
||||
raise ValueError("Capacity must be greater than 0")
|
||||
|
||||
self.capacity = capacity
|
||||
self.cache: Dict[Any, Node] = {} # key -> Node
|
||||
self.freq_map: Dict[int, DoublyLinkedList] = {} # freq -> DLL
|
||||
self.min_freq = 0
|
||||
self.lock = asyncio.Lock()
|
||||
self._evictor_task: Optional[asyncio.Task] = None
|
||||
|
||||
def _get_now(self) -> float:
|
||||
return time.time()
|
||||
|
||||
def _is_expired(self, node: Node) -> bool:
|
||||
return node.expiry is not None and self._get_now() > node.expiry
|
||||
|
||||
async def _update_freq(self, node: Node):
|
||||
"""Moves a node to the next frequency bucket in O(1)."""
|
||||
old_freq = node.freq
|
||||
dll = self.freq_map[old_freq]
|
||||
dll.remove(node)
|
||||
|
||||
if old_freq == self.min_freq and dll.size == 0:
|
||||
self.min_freq += 1
|
||||
|
||||
node.freq += 1
|
||||
if node.freq not in self.freq_map:
|
||||
self.freq_map[node.freq] = DoublyLinkedList()
|
||||
self.freq_map[node.freq].append(node)
|
||||
|
||||
async def get(self, key: Any) -> Optional[Any]:
|
||||
"""O(1) Access with Lazy TTL check."""
|
||||
async with self.lock:
|
||||
if key not in self.cache:
|
||||
return None
|
||||
|
||||
node = self.cache[key]
|
||||
if self._is_expired(node):
|
||||
await self._delete_node(node)
|
||||
return None
|
||||
|
||||
await self._update_freq(node)
|
||||
return node.value
|
||||
|
||||
async def put(self, key: Any, value: Any, ttl_seconds: Optional[float] = None):
|
||||
"""O(1) Insertion with LFU eviction and Lazy TTL check."""
|
||||
async with self.lock:
|
||||
expiry = self._get_now() + ttl_seconds if ttl_seconds else None
|
||||
|
||||
if key in self.cache:
|
||||
node = self.cache[key]
|
||||
node.value = value
|
||||
node.expiry = expiry
|
||||
await self._update_freq(node)
|
||||
return
|
||||
|
||||
# Evict if capacity reached
|
||||
if len(self.cache) >= self.capacity:
|
||||
# Priority 1: Try to find an expired key first (optimistic)
|
||||
# Otherwise: Evict the LFU (min_freq tail)
|
||||
evicted = self.freq_map[self.min_freq].pop_tail()
|
||||
if evicted:
|
||||
del self.cache[evicted.key]
|
||||
|
||||
# Create new node
|
||||
new_node = Node(key=key, value=value, expiry=expiry)
|
||||
self.cache[key] = new_node
|
||||
self.min_freq = 1
|
||||
if 1 not in self.freq_map:
|
||||
self.freq_map[1] = DoublyLinkedList()
|
||||
self.freq_map[1].append(new_node)
|
||||
|
||||
async def delete(self, key: Any):
|
||||
"""O(1) deletion."""
|
||||
async with self.lock:
|
||||
if key in self.cache:
|
||||
await self._delete_node(self.cache[key])
|
||||
|
||||
async def _delete_node(self, node: Node):
|
||||
"""Internal helper to remove node from all tracking structures."""
|
||||
dll = self.freq_map.get(node.freq)
|
||||
if dll:
|
||||
dll.remove(node)
|
||||
if node.key in self.cache:
|
||||
del self.cache[node.key]
|
||||
|
||||
# --- Background Evictor ---
|
||||
|
||||
async def start_evictor(self, interval: float = 1.0, batch_size: int = 100):
|
||||
"""Starts a non-blocking background task to purge expired keys."""
|
||||
if self._evictor_task:
|
||||
return
|
||||
|
||||
async def evict_loop():
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
# Collect keys to check outside the lock to prevent blocking reads
|
||||
keys_to_check = list(self.cache.keys())
|
||||
|
||||
# Process in small batches to yield to the event loop
|
||||
for i in range(0, len(keys_to_check), batch_size):
|
||||
batch = keys_to_check[i : i + batch_size]
|
||||
async with self.lock:
|
||||
for k in batch:
|
||||
node = self.cache.get(k)
|
||||
if node and self._is_expired(node):
|
||||
await self._delete_node(node)
|
||||
# Yield control back to asyncio
|
||||
await asyncio.sleep(0)
|
||||
|
||||
self._evictor_task = asyncio.create_task(evict_loop())
|
||||
|
||||
async def stop_evictor(self):
|
||||
"""Stops the background eviction task."""
|
||||
if self._evictor_task:
|
||||
self._evictor_task.cancel()
|
||||
try:
|
||||
await self._evictor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._evictor_task = None
|
||||
|
||||
# --- Transaction Support ---
|
||||
|
||||
def begin_transaction(self) -> 'Transaction':
|
||||
return Transaction(self)
|
||||
|
||||
async def apply_transaction_changes(self, changes: 'TransactionState'):
|
||||
"""Atomically apply buffered transaction writes to the global state."""
|
||||
async with self.lock:
|
||||
for key, action in changes.writes.items():
|
||||
if action == "DELETE":
|
||||
if key in self.cache:
|
||||
await self._delete_node(self.cache[key])
|
||||
else:
|
||||
val, ttl = action
|
||||
# Using put logic inside the lock
|
||||
expiry = self._get_now() + ttl if ttl else None
|
||||
if key in self.cache:
|
||||
node = self.cache[key]
|
||||
node.value = val
|
||||
node.expiry = expiry
|
||||
await self._update_freq(node)
|
||||
else:
|
||||
if len(self.cache) >= self.capacity:
|
||||
evicted = self.freq_map[self.min_freq].pop_tail()
|
||||
if evicted:
|
||||
del self.cache[evicted.key]
|
||||
new_node = Node(key=key, value=val, expiry=expiry)
|
||||
self.cache[key] = new_node
|
||||
self.min_freq = 1
|
||||
if 1 not in self.freq_map:
|
||||
self.freq_map[1] = DoublyLinkedList()
|
||||
self.freq_map[1].append(new_node)
|
||||
|
||||
@dataclass
|
||||
class TransactionState:
|
||||
writes: Dict[Any, Any] = None # key -> (value, ttl) or "DELETE"
|
||||
|
||||
class Transaction:
|
||||
def __init__(self, cache: LFUCache):
|
||||
self._cache = cache
|
||||
self._state = TransactionState(writes={})
|
||||
self._committed = False
|
||||
|
||||
async def put(self, key: Any, value: Any, ttl_seconds: Optional[float] = None):
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
self._state.writes[key] = (value, ttl_seconds)
|
||||
|
||||
async def get(self, key: Any) -> Optional[Any]:
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
|
||||
# Read-Your-Own-Writes (RYOW)
|
||||
if key in self._state.writes:
|
||||
action = self._state.writes[key]
|
||||
if action == "DELETE":
|
||||
return None
|
||||
return action[0]
|
||||
|
||||
# Otherwise read from global cache
|
||||
return await self._cache.get(key)
|
||||
|
||||
async def delete(self, key: Any):
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
self._state.writes[key] = "DELETE"
|
||||
|
||||
async def commit(self):
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
await self._cache.apply_transaction_changes(self._state)
|
||||
self._committed = True
|
||||
|
||||
async def rollback(self):
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
self._state.writes.clear()
|
||||
self._committed = True
|
||||
|
||||
# --- Executable Unit Tests ---
|
||||
|
||||
async def main():
|
||||
print("🚀 Starting LFU Cache Suite...")
|
||||
|
||||
# 1. Test O(1) LFU Eviction
|
||||
print("\nTesting LFU Eviction Order...", end=" ")
|
||||
cache = LFUCache(capacity=2)
|
||||
await cache.put("a", 1)
|
||||
await cache.put("b", 2)
|
||||
await cache.get("a") # freq(a)=2, freq(b)=1
|
||||
await cache.put("c", 3) # b should be evicted
|
||||
assert await cache.get("b") is None
|
||||
assert await cache.get("a") == 1
|
||||
assert await cache.get("c") == 3
|
||||
print("✅ PASSED")
|
||||
|
||||
# 2. Test Dual-Layer TTL
|
||||
print("\nTesting TTL Eviction...", end=" ")
|
||||
ttl_cache = LFUCache(capacity=10)
|
||||
await ttl_cache.put("lazy", "val", ttl_seconds=0.1)
|
||||
await ttl_cache.put("async", "val", ttl_seconds=0.1)
|
||||
|
||||
# Lazy check
|
||||
await asyncio.sleep(0.2)
|
||||
assert await ttl_cache.get("lazy") is None, "Lazy eviction failed"
|
||||
|
||||
# Background check
|
||||
await ttl_cache.start_evictor(interval=0.1, batch_size=10)
|
||||
await asyncio.sleep(0.2)
|
||||
async with ttl_cache.lock:
|
||||
assert "async" not in ttl_cache.cache, "Background evictor failed"
|
||||
await ttl_cache.stop_evictor()
|
||||
print("✅ PASSED")
|
||||
|
||||
# 3. Test Atomic Transactions
|
||||
print("\nTesting Atomic Transactions...", end=" ")
|
||||
tx_cache = LFUCache(capacity=10)
|
||||
await tx_cache.put("base", 100)
|
||||
|
||||
# Test Commit
|
||||
tx1 = tx_cache.begin_transaction()
|
||||
await tx1.put("base", 200)
|
||||
await tx1.put("new", 300)
|
||||
assert await tx1.get("base") == 200 # RYOW
|
||||
assert await tx_cache.get("base") == 100 # Isolation
|
||||
await tx1.commit()
|
||||
assert await tx_cache.get("base") == 200
|
||||
assert await tx_cache.get("new") == 300
|
||||
|
||||
# Test Rollback
|
||||
tx2 = tx_cache.begin_transaction()
|
||||
await tx2.put("base", 999)
|
||||
await tx2.get("base")
|
||||
await tx2.rollback()
|
||||
assert await tx_cache.get("base") == 200
|
||||
print("✅ PASSED")
|
||||
|
||||
# 4. Stress Test: Concurrent Async Tasks
|
||||
print("\nStress Testing 50 concurrent tasks...", end=" ")
|
||||
stress_cache = LFUCache(capacity=20)
|
||||
|
||||
async def worker(worker_id):
|
||||
for i in range(100):
|
||||
key = f"key_{i % 30}" # Contention on keys
|
||||
if i % 3 == 0:
|
||||
await stress_cache.put(key, f"val_{worker_id}_{i}")
|
||||
elif i % 3 == 1:
|
||||
await stress_cache.get(key)
|
||||
else:
|
||||
tx = stress_cache.begin_transaction()
|
||||
await tx.put(key, f"tx_{worker_id}_{i}")
|
||||
await tx.commit()
|
||||
|
||||
await asyncio.gather(*(worker(i) for i in range(50)))
|
||||
print("✅ PASSED")
|
||||
|
||||
print("\n✨ All tests passed successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
Reference in New Issue
Block a user