Initial benchmark suite: 8 graded models + cyberpunk dashboard generator
- prompts/: LFU cache exam + 5-pillar grading rubric - outputs/: 8 model .py outputs (local + cloud baseline) - data/benchmark_history.json: graded results (scores, metrics, bugs, patches) - generate_dashboard.py: builds dashboard.html + pages/*.html from JSON - Dockerfile + DEPLOY.md: Gitea→Coolify deploy (build-step, nginx static) - .gitignore: generated HTML excluded (built on deploy) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,339 @@
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, Optional, Generic, TypeVar, Set
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
@dataclass
|
||||
class Node:
|
||||
"""A node in the LFU doubly linked list."""
|
||||
key: Any
|
||||
value: Any
|
||||
freq: int = 1
|
||||
expiry: float = float('inf')
|
||||
prev: Optional['Node'] = None
|
||||
next: Optional['Node'] = None
|
||||
|
||||
class DoublyLinkedList:
|
||||
"""Standard O(1) doubly linked list for maintaining frequency buckets."""
|
||||
def __init__(self):
|
||||
self.head = Node(None, None) # Sentinel head
|
||||
self.tail = Node(None, None) # Sentinel tail
|
||||
self.head.next = self.tail
|
||||
self.tail.prev = self.head
|
||||
self.size = 0
|
||||
|
||||
def append(self, node: Node):
|
||||
"""Adds a node to the end (most recently used in this frequency)."""
|
||||
node.next = self.tail
|
||||
node.prev = self.tail.prev
|
||||
self.tail.prev.next = node
|
||||
self.tail.prev = node
|
||||
self.size += 1
|
||||
|
||||
def remove(self, node: Node):
|
||||
"""Removes a specific node from the list."""
|
||||
if node.prev:
|
||||
node.prev.next = node.next
|
||||
if node.next:
|
||||
node.next.prev = node.prev
|
||||
node.prev = None
|
||||
node.next = None
|
||||
self.size -= 1
|
||||
|
||||
def pop_front(self) -> Optional[Node]:
|
||||
"""Removes and returns the oldest node (LFU candidate)."""
|
||||
if self.size == 0:
|
||||
return None
|
||||
node = self.head.next
|
||||
self.remove(node)
|
||||
return node
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.size == 0
|
||||
|
||||
|
||||
class LFUCache:
|
||||
"""
|
||||
In-Memory Concurrent LFU Cache with Async TTL Eviction and Atomic Transactions.
|
||||
Time Complexity: O(1) for get and put.
|
||||
Space Complexity: O(N).
|
||||
"""
|
||||
def __init__(self, capacity: int):
|
||||
if capacity <= 0:
|
||||
raise ValueError("Capacity must be greater than 0")
|
||||
|
||||
self.capacity = capacity
|
||||
self.cache: Dict[Any, Node] = {} # Key -> Node
|
||||
self.freq_map: Dict[int, DoublyLinkedList] = {} # Freq -> DLL
|
||||
self.min_freq = 0
|
||||
self._lock = asyncio.Lock()
|
||||
self._evictor_task: Optional[asyncio.Task] = None
|
||||
|
||||
async def get(self, key: Any) -> Optional[Any]:
|
||||
async with self._lock:
|
||||
return await self._get_internal(key)
|
||||
|
||||
async def _get_internal(self, key: Any) -> Optional[Any]:
|
||||
"""Internal get without lock for transaction use."""
|
||||
if key not in self.cache:
|
||||
return None
|
||||
|
||||
node = self.cache[key]
|
||||
|
||||
# Lazy Eviction check
|
||||
if time.time() > node.expiry:
|
||||
await self._delete_internal(key)
|
||||
return None
|
||||
|
||||
self._update_frequency(node)
|
||||
return node.value
|
||||
|
||||
async def put(self, key: Any, value: Any, ttl_seconds: Optional[float] = None):
|
||||
async with self._lock:
|
||||
await self._put_internal(key, value, ttl_seconds)
|
||||
|
||||
async def _put_internal(self, key: Any, value: Any, ttl_seconds: Optional[float] = None):
|
||||
"""Internal put without lock for transaction use."""
|
||||
expiry = time.time() + ttl_seconds if ttl_seconds is not None else float('inf')
|
||||
|
||||
if key in self.cache:
|
||||
node = self.cache[key]
|
||||
node.value = value
|
||||
node.expiry = expiry
|
||||
self._update_frequency(node)
|
||||
else:
|
||||
# Capacity Management
|
||||
if len(self.cache) >= self.capacity:
|
||||
await self._evict_lfu()
|
||||
|
||||
new_node = Node(key, value, freq=1, expiry=expiry)
|
||||
self.cache[key] = new_node
|
||||
self._add_to_freq_bucket(1, new_node)
|
||||
self.min_freq = 1
|
||||
|
||||
async def delete(self, key: Any):
|
||||
async with self._lock:
|
||||
await self._delete_internal(key)
|
||||
|
||||
async def _delete_internal(self, key: Any):
|
||||
"""Internal delete without lock for transaction use."""
|
||||
if key in self.cache:
|
||||
node = self.cache.pop(key)
|
||||
self.freq_map[node.freq].remove(node)
|
||||
# Note: We don't strictly need to update min_freq here because
|
||||
# _evict_lfu handles empty buckets by incrementing.
|
||||
|
||||
def _update_frequency(self, node: Node):
|
||||
"""Moves a node to the next frequency bucket in O(1)."""
|
||||
old_freq = node.freq
|
||||
self.freq_map[old_freq].remove(node)
|
||||
|
||||
if old_freq == self.min_freq and self.freq_map[old_freq].is_empty():
|
||||
self.min_freq += 1
|
||||
|
||||
node.freq += 1
|
||||
self._add_to_freq_bucket(node.freq, node)
|
||||
|
||||
def _add_to_freq_bucket(self, freq: int, node: Node):
|
||||
if freq not in self.freq_map:
|
||||
self.freq_map[freq] = DoublyLinkedList()
|
||||
self.freq_map[freq].append(node)
|
||||
|
||||
async def _evict_lfu(self):
|
||||
"""Evicts the least frequently used (and oldest within that freq) item."""
|
||||
while self.min_freq not in self.freq_map or self.freq_map[self.min_freq].is_empty():
|
||||
# This handles cases where items were deleted manually
|
||||
if not self.cache: return
|
||||
self.min_freq += 1 # Search for the next available freq bucket
|
||||
|
||||
evicted_node = self.freq_map[self.min_freq].pop_front()
|
||||
if evicted_node:
|
||||
del self.cache[evicted_node.key]
|
||||
|
||||
# --- Background Evictor ---
|
||||
|
||||
async def start_evictor(self, interval: float = 1.0):
|
||||
"""Starts the background task that purges expired keys."""
|
||||
if self._evictor_task is not None:
|
||||
return
|
||||
self._evictor_task = asyncio.create_task(self._eviction_loop(interval))
|
||||
|
||||
async def stop_evictor(self):
|
||||
"""Stops the background eviction task."""
|
||||
if self._evictor_task:
|
||||
self._evictor_task.cancel()
|
||||
try:
|
||||
await self._evictor_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._evictor_task = None
|
||||
|
||||
async def _eviction_loop(self, interval: float):
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
# Process in small batches to avoid blocking the main lock for too long
|
||||
now = time.time()
|
||||
keys_to_check = list(self.cache.keys())
|
||||
|
||||
batch_size = 100
|
||||
for i in range(0, len(keys_to_check), batch_size):
|
||||
async with self._lock:
|
||||
batch = keys_to_check[i : i + batch_size]
|
||||
for k in batch:
|
||||
if k in self.cache and now > self.cache[k].expiry:
|
||||
await self._delete_internal(k)
|
||||
|
||||
# --- Transaction Logic ---
|
||||
|
||||
def begin_transaction(self):
|
||||
return Transaction(self)
|
||||
|
||||
|
||||
class Transaction:
|
||||
"""
|
||||
Provides ACID-like atomic updates to the LFUCache.
|
||||
Implements 'Read Your Own Writes' (RYOW).
|
||||
"""
|
||||
def __init__(self, cache: LFUCache):
|
||||
self._cache = cache
|
||||
self._pending_puts: Dict[Any, tuple[Any, Optional[float]]] = {}
|
||||
self._pending_deletes: Set[Any] = set()
|
||||
self._committed = False
|
||||
|
||||
async def get(self, key: Any) -> Optional[Any]:
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
|
||||
# 1. Check local deletes first
|
||||
if key in self._pending_deletes:
|
||||
return None
|
||||
|
||||
# 2. Check local writes (RYOW)
|
||||
if key in self._pending_puts:
|
||||
return self._pending_puts[key][0]
|
||||
|
||||
# 3. Fallback to global cache
|
||||
return await self._cache.get(key)
|
||||
|
||||
async def put(self, key: Any, value: Any, ttl_seconds: Optional[float] = None):
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
self._pending_deletes.discard(key)
|
||||
self._pending_puts[key] = (value, ttl_seconds)
|
||||
|
||||
async def delete(self, key: Any):
|
||||
if self._committed: raise RuntimeError("Transaction already committed")
|
||||
self._pending_puts.pop(key, None)
|
||||
self._pending_deletes.add(key)
|
||||
|
||||
async def commit(self):
|
||||
"""Applies all pending changes to the global cache atomically."""
|
||||
if self._committed: return
|
||||
|
||||
async with self._cache._lock:
|
||||
# Apply deletes first
|
||||
for key in self._pending_deletes:
|
||||
await self._cache._delete_internal(key)
|
||||
# Apply puts
|
||||
for key, (val, ttl) in self._pending_puts.items():
|
||||
await self._cache._put_internal(key, val, ttl)
|
||||
|
||||
self._committed = True
|
||||
|
||||
async def rollback(self):
|
||||
"""Discards all pending changes."""
|
||||
self._pending_puts.clear()
|
||||
self._pending_deletes.clear()
|
||||
self._committed = True
|
||||
|
||||
|
||||
# ==========================================
|
||||
# Unit Tests
|
||||
# ==========================================
|
||||
|
||||
async def main():
|
||||
print("Starting LFU Cache Comprehensive Test Suite...")
|
||||
|
||||
# 1. Test O(1) LFU Eviction Order
|
||||
print("\nTesting LFU Eviction Order...", end=" ")
|
||||
cache = LFUCache(capacity=2)
|
||||
await cache.put("a", 1) # freq 1
|
||||
await cache.put("b", 2) # freq 1
|
||||
await cache.get("a") # freq 2
|
||||
await cache.put("c", 3) # "b" should be evicted as it's the LFU (freq 1)
|
||||
assert await cache.get("b") is None, "Key 'b' should have been evicted"
|
||||
assert await cache.get("a") == 1, "Key 'a' should still exist"
|
||||
assert await cache.get("c") == 3, "Key 'c' should still exist"
|
||||
print("PASSED")
|
||||
|
||||
# 2. Test TTL Eviction (Lazy & Background)
|
||||
print("Testing Dual-Layer TTL...", end=" ")
|
||||
cache = LFUCache(capacity=10)
|
||||
await cache.start_evictor(interval=0.1)
|
||||
|
||||
await cache.put("lazy", "val", ttl_seconds=0.1)
|
||||
await asyncio.sleep(0.2)
|
||||
# Lazy check: get() should trigger deletion
|
||||
assert await cache.get("lazy") is None, "Lazy eviction failed"
|
||||
|
||||
await cache.put("bg", "val", ttl_seconds=0.1)
|
||||
await asyncio.sleep(0.3)
|
||||
# Background check: key should be gone from internal map without calling get()
|
||||
async with cache._lock:
|
||||
assert "bg" not in cache.cache, "Background eviction failed"
|
||||
await cache.stop_evictor()
|
||||
print("PASSED")
|
||||
|
||||
# 3. Test Atomic Transactions (Commit vs Rollback)
|
||||
print("Testing Transaction Isolation...", end=" ")
|
||||
cache = LFUCache(capacity=10)
|
||||
await cache.put("shared", "original")
|
||||
|
||||
tx = cache.begin_transaction()
|
||||
await tx.put("shared", "modified")
|
||||
await tx.put("new", "added")
|
||||
|
||||
# Verify Isolation: Global reader shouldn't see changes
|
||||
assert await cache.get("shared") == "original"
|
||||
assert await cache.get("new") is None
|
||||
|
||||
# Verify RYOW: Transaction should see its own writes
|
||||
assert await tx.get("shared") == "modified"
|
||||
assert await tx.get("new") == "added"
|
||||
|
||||
await tx.commit()
|
||||
assert await cache.get("shared") == "modified", "Commit failed to update global state"
|
||||
assert await cache.get("new") == "added", "Commit failed to add new key"
|
||||
|
||||
tx2 = cache.begin_transaction()
|
||||
await tx2.put("shared", "rollback_this")
|
||||
assert await tx2.get("shared") == "rollback_this"
|
||||
await tx2.rollback()
|
||||
assert await cache.get("shared") == "modified", "Rollback mutated global state"
|
||||
print("PASSED")
|
||||
|
||||
# 4. Stress Test: Concurrent Access
|
||||
print("Testing High Concurrency (50 tasks)...", end=" ")
|
||||
cache = LFUCache(capacity=100)
|
||||
|
||||
async def worker(worker_id):
|
||||
for i in range(100):
|
||||
key = f"key_{i % 20}" # Create contention on keys
|
||||
if i % 3 == 0:
|
||||
await cache.put(key, f"val_{worker_id}_{i}", ttl_seconds=0.5)
|
||||
elif i % 3 == 1:
|
||||
await cache.get(key)
|
||||
else:
|
||||
tx = cache.begin_transaction()
|
||||
await tx.put(f"tx_{worker_id}", i)
|
||||
await tx.commit()
|
||||
|
||||
tasks = [worker(i) for i in range(50)]
|
||||
await asyncio.gather(*tasks)
|
||||
print("PASSED")
|
||||
|
||||
print("\nAll tests passed successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
||||
Reference in New Issue
Block a user