Gemma 4 26B-A4B head-to-head vs Qwen (4 prompts, via grade_run.py API)
Ran via tools/grade_run.py against LM Studio (no clipboard). Results: TTS: 80 Minor Flaws PASSES (real N-worker concurrency) <- Qwen 49, didn't parse Rust: 72 Minor Flaws COMPILES CLEAN (0 errs w/ deps) <- Qwen 50, 7 real errors Webhook: 55 Critical uses forbidden aiohttp (won't run) <- Qwen 75, passed Automation: 48 Critical SyntaxError (global-after-assign) <- first run for both DECISIVE head-to-head: Gemma generalizes where Qwen fails (TTS, Rust), but Qwen beats it on stdlib-discipline prompts (webhook). The two are COMPLEMENTARY local offloads, not redundant. Fixed: grade_run.py extractor (markdown/prose wrapping, multi-fence lang selection), TTFT-null handling in generator. TTFT capture from LM Studio API still needs the right stats key (left null + noted). Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,317 @@
|
||||
use rand::Rng;
|
||||
use serde::Serialize;
|
||||
use std::{collections::HashMap, sync::Arc, time::Duration};
|
||||
use thiserror::Error;
|
||||
use tokio::sync::{mpsc, RwLock};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
// --- Types and Errors ---
|
||||
|
||||
#[derive(Debug, Error, Clone)]
|
||||
pub enum WatcherError {
|
||||
#[error("Failed to fetch data from source")]
|
||||
FetchError,
|
||||
#[error("Channel closed unexpectedly")]
|
||||
ChannelClosed,
|
||||
#[error("Watcher exceeded max consecutive failures")]
|
||||
Unhealthy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WatchItem {
|
||||
pub watcher_id: u32,
|
||||
pub data: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AggregatedReport {
|
||||
pub total_items: usize,
|
||||
pub items: Vec<WatchItem>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct WatcherId(u32);
|
||||
|
||||
// --- Mock Source ---
|
||||
|
||||
/// Simulates an external API/Source.
|
||||
/// Returns 0-3 items or fails ~15% of the time.
|
||||
async fn mock_fetch(id: u32, force_fail: bool) -> Result<Vec<String>, WatcherError> {
|
||||
// Simulate network latency
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
let mut rng = rand::thread_rng();
|
||||
if force_fail || rng.gen_bool(0.15) {
|
||||
return Err(WatcherError::FetchError);
|
||||
}
|
||||
|
||||
let count = rng.gen_range(0..=3);
|
||||
Ok((0..count).map(|i| format!("val-{}-{}", id, i)).collect())
|
||||
}
|
||||
|
||||
// --- Watcher Logic ---
|
||||
|
||||
struct Watcher {
|
||||
id: WatcherId,
|
||||
tx: mpsc::Sender<WatchItem>,
|
||||
token: CancellationToken,
|
||||
force_fail: bool,
|
||||
}
|
||||
|
||||
impl Watcher {
|
||||
async fn run(self) -> Result<(), WatcherError> {
|
||||
let mut consecutive_failures = 0;
|
||||
const MAX_FAILURES: u32 = 5;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
// 1. Handle Shutdown Signal
|
||||
_ = self.token.cancelled() => {
|
||||
println!("[Watcher {:?}] Shutting down gracefully", self.id);
|
||||
return Ok(());
|
||||
}
|
||||
// 2. Periodic Polling
|
||||
_ = tokio::time::sleep(Duration::from_millis(200)) => {
|
||||
match mock_fetch(self.id.0, self.force_fail).await {
|
||||
Ok(items) => {
|
||||
consecutive_failures = 0; // Reset on success
|
||||
for item in items {
|
||||
let msg = WatchItem { watcher_id: self.id.0, data: item };
|
||||
// Backpressure: .send() awaits if the channel is full.
|
||||
// This prevents a fast watcher from overwhelming the consumer.
|
||||
if self.tx.send(msg).await.is_err() {
|
||||
return Err(WatcherError::ChannelClosed);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
consecutive_failures += 1;
|
||||
eprintln!("[Watcher {:?}] Error ({}): {}", self.id, consecutive_failures, e);
|
||||
|
||||
if consecutive_failures >= MAX_FAILURES {
|
||||
eprintln!("[Watcher {:?}] Marked UNHEALTHY. Terminating.", self.id);
|
||||
return Err(WatcherError::Unhealthy);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Manager Logic ---
|
||||
|
||||
pub struct WatcherManager {
|
||||
/// Map of active watcher handles. Protected by RwLock for concurrent add/remove.
|
||||
watchers: Arc<RwLock<HashMap<WatcherId, JoinHandle<()>>>>,
|
||||
/// Channel to send items to the consumer.
|
||||
tx: mpsc::Sender<WatchItem>,
|
||||
/// Token to signal all tasks to stop.
|
||||
shutdown_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl WatcherManager {
|
||||
pub fn new(buffer_size: usize) -> (Self, mpsc::Receiver<WatchItem>, CancellationToken) {
|
||||
let (tx, rx) = mpsc::channel(buffer_size);
|
||||
let token = CancellationToken::new();
|
||||
|
||||
let manager = Self {
|
||||
watchers: Arc::new(RwLock::new(HashMap::new())),
|
||||
tx,
|
||||
shutdown_token: token.clone(),
|
||||
};
|
||||
|
||||
(manager, rx, token)
|
||||
}
|
||||
|
||||
pub async fn add_watcher(&self, id: u32, force_fail: bool) {
|
||||
let watcher_id = WatcherId(id);
|
||||
let watcher = Watcher {
|
||||
id: watcher_id,
|
||||
tx: self.tx.clone(),
|
||||
token: self.shutdown_token.clone(),
|
||||
force_fail,
|
||||
};
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
if let Err(e) = watcher.run().await {
|
||||
eprintln!("[Manager] Watcher {:?} exited with error: {}", id, e);
|
||||
}
|
||||
});
|
||||
|
||||
let mut watchers = self.watchers.write().await;
|
||||
watchers.insert(watcher_id, handle);
|
||||
println!("[Manager] Added watcher {}", id);
|
||||
}
|
||||
|
||||
pub async fn remove_watcher(&self, id: u32) {
|
||||
let watcher_id = WatcherId(id);
|
||||
let mut watchers = self.watchers.write().await;
|
||||
if let Some(handle) = watchers.remove(&watcher_id) {
|
||||
// Abort the task immediately if it's still running
|
||||
handle.abort();
|
||||
println!("[Manager] Removed watcher {}", id);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_active_count(&self) -> usize {
|
||||
self.watchers.read().await.len()
|
||||
}
|
||||
|
||||
pub async fn shutdown(self) {
|
||||
println!("[Manager] Initiating shutdown...");
|
||||
self.shutdown_token.cancel();
|
||||
|
||||
// Wait for all handles to finish
|
||||
let mut watchers = self.watchers.write().await;
|
||||
for (id, handle) in watchers.drain() {
|
||||
let _ = handle.await;
|
||||
println!("[Manager] Joined watcher {:?}", id.0);
|
||||
}
|
||||
println!("[Manager] All watchers stopped.");
|
||||
}
|
||||
}
|
||||
|
||||
// --- Consumer Logic ---
|
||||
|
||||
async fn run_consumer(mut rx: mpsc::Receiver<WatchItem>, token: CancellationToken) -> AggregatedReport {
|
||||
let mut items = Vec::new();
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = token.cancelled() => {
|
||||
println!("[Consumer] Shutdown signal received.");
|
||||
break;
|
||||
}
|
||||
msg = rx.recv() => {
|
||||
match msg {
|
||||
Some(item) => items.push(item),
|
||||
None => break, // Channel closed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
AggregatedReport {
|
||||
total_items: items.len(),
|
||||
items,
|
||||
}
|
||||
}
|
||||
|
||||
// --- Tests ---
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_lifecycle_and_aggregation() {
|
||||
let (manager, rx, token) = WatcherManager::new(100);
|
||||
let manager = Arc::new(manager);
|
||||
|
||||
// Spawn consumer
|
||||
let consumer_token = token.clone();
|
||||
let consumer_handle = tokio::spawn(async move {
|
||||
run_consumer(rx, consumer_token).await
|
||||
});
|
||||
|
||||
// Add 4 watchers
|
||||
for i in 1..=4 {
|
||||
manager.add_watcher(i, false).await;
|
||||
}
|
||||
|
||||
// Let them run for a bit
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
// Shutdown
|
||||
let m = Arc::try_unwrap(manager).ok().expect("Manager should have no other owners");
|
||||
m.shutdown().await;
|
||||
token.cancel(); // Ensure consumer stops
|
||||
|
||||
let report = consumer_handle.await.unwrap();
|
||||
|
||||
assert!(report.total_items > 0, "Consumer should have received items");
|
||||
println!("Test A Passed: Received {} items", report.total_items);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_unhealthy_watcher() {
|
||||
let (manager, rx, token) = WatcherManager::new(10);
|
||||
let manager = Arc::new(manager);
|
||||
|
||||
// Spawn consumer to prevent channel blockage
|
||||
let ct = token.clone();
|
||||
tokio::spawn(async move { run_consumer(rx, ct).await });
|
||||
|
||||
// Add a watcher that always fails
|
||||
manager.add_watcher(99, true).await;
|
||||
|
||||
// Wait for it to hit 5 failures (5 * 200ms + buffer)
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
// The watcher should have exited.
|
||||
// Note: In this implementation, the manager doesn't automatically remove
|
||||
// dead handles from the map unless we implement a reaper, but we check if it's still "active"
|
||||
// by observing that the task is no longer running.
|
||||
let count = manager.get_active_count().await;
|
||||
assert!(count >= 1); // It's still in the map, but the task is dead.
|
||||
// In a real system, we'd have a background reaper task to clean the HashMap.
|
||||
println!("Test B Passed: Unhealthy watcher handled.");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_concurrent_mutation() {
|
||||
let (manager, rx, token) = WatcherManager::new(10);
|
||||
let manager = Arc::new(manager);
|
||||
tokio::spawn(async move { run_consumer(rx, token).await });
|
||||
|
||||
let mut handles = vec![];
|
||||
|
||||
// Hammer the manager with concurrent adds and removes
|
||||
for i in 0..50 {
|
||||
let m = manager.clone();
|
||||
handles.push(tokio::spawn(async move {
|
||||
m.add_watcher(i, false).await;
|
||||
if i % 2 == 0 {
|
||||
m.remove_watcher(i).await;
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
for h in handles {
|
||||
h.await.unwrap();
|
||||
}
|
||||
|
||||
let final_count = manager.get_active_count().await;
|
||||
println!("Test C Passed: Concurrent mutation finished with {} watchers", final_count);
|
||||
assert!(final_count <= 50);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
println!("Starting Service...");
|
||||
let (manager, rx, token) = WatcherManager::new(100);
|
||||
|
||||
// Spawn consumer
|
||||
let ct = token.clone();
|
||||
let consumer_handle = tokio::spawn(async move {
|
||||
run_consumer(rx, ct).await
|
||||
});
|
||||
|
||||
// Add some watchers
|
||||
manager.add_watcher(1, false).await;
|
||||
manager.add_watcher(2, false).await;
|
||||
manager.add_watcher(3, true).await; // This one will fail
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
// Shutdown
|
||||
manager.shutdown().await;
|
||||
token.cancel(); // Signal consumer
|
||||
|
||||
let report = consumer_handle.await.unwrap();
|
||||
println!("Final Report: {:?}", report);
|
||||
}
|
||||
Reference in New Issue
Block a user