Grade muse-glimmer-28b full 7-prompt battery — new benchmark leader

7 entries (30→37 total). Muse Glimmer 28B (GGUF) avg 80.7 — the strongest model in the benchmark, 6/7 prompts Minor Logic Flaws: lfu 76 | webhook 81 | automation 89 | rust 85 | data 88 | tts 58 | mcp 88 Standout results: - rust 85 (KAT 36, Qwen3-Coder 54) — real tokio channels (mpsc::channel, not hallucinated mpsc::bounded), two-tier CancellationToken, zero clippy lints; one-line E0507 compile fix. - automation 89 — first model to print a correct summary (98/2/0/100); atomic temp+fsync+rename checkpointing. - data 88 edges out Gemma-26B's 86; mcp 88 sets the bar on a new prompt. Only weak spot: tts 58 (backpressure raises instead of awaits, like Qwen3-Coder). Captured via the native /api/v1/chat fix (real tok/sec + TTFT). Slow deep-thinker: ~17-19 t/s, 5-9 min/prompt, ~5-9k tokens incl. reasoning. Also gitignore checkpoint.json (automation test runtime artifact). Co-Authored-By: Claude <noreply@anthropic.com>
2026-08-10 13:13:54 -07:00
parent 3507d33006
commit 2f99dd1e35
10 changed files with 1769 additions and 7 deletions
@@ -0,0 +1,179 @@
+// mcp-jsonplaceholder.ts
+// Run: npx tsx mcp-jsonplaceholder.ts
+// Test: npx tsx mcp-jsonplaceholder.ts --test
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { z } from 'zod';
+
+const BASE = 'https://jsonplaceholder.typicode.com';
+
+// ---------- Validation schemas ----------
+const GetUserSchema = z.object({
+  id: z.number().int().positive()
+});
+
+const ListPostsSchema = z.object({
+  user_id: z.number().int().positive(),
+  limit: z.number().int().min(1).max(100).optional().default(10)
+});
+
+const SearchPostsSchema = z.object({
+  query: z.string().min(1).max(200)
+});
+
+// ---------- HTTP helper with timeout ----------
+async function fetchJson(url: string, timeoutMs = 8000) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(url, { signal: controller.signal });
+    clearTimeout(timer);
+    if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`);
+    const data = await res.json();
+    return data;
+  } catch (err: any) {
+    clearTimeout(timer);
+    if (err.name === 'AbortError') throw new Error('Request timed out');
+    throw err;
+  }
+}
+
+// ---------- Typed implementations ----------
+async function getUserImpl(id: number) {
+  const user = await fetchJson(`${BASE}/users/${id}`);
+  if (!user || !user.id) throw new Error(`User ${id} not found`);
+  return user;
+}
+
+async function listPostsImpl(user_id: number, limit: number) {
+  const posts = await fetchJson(`${BASE}/posts?userId=${user_id}`);
+  if (!Array.isArray(posts)) throw new Error('Invalid posts response');
+  return posts.slice(0, limit).map(p => ({ id: p.id, userId: p.userId, title: p.title, body: p.body }));
+}
+
+async function searchPostsImpl(query: string) {
+  const posts = await fetchJson(`${BASE}/posts`);
+  const q = query.toLowerCase();
+  const filtered = posts.filter((p: any) => p.title.toLowerCase().includes(q));
+  return filtered.map((p: any) => ({ id: p.id, userId: p.userId, title: p.title }));
+}
+
+// ---------- MCP Server ----------
+const server = new Server({ name: 'jsonplaceholder-mcp', version: '1.0.0' }, {
+  capabilities: { tools: {} }
+});
+
+server.setRequestHandler(ListToolsRequestSchema, async () => ({
+  tools: [
+    {
+      name: 'get_user',
+      description: 'Fetch a single user by numeric id',
+      inputSchema: {
+        type: 'object',
+        properties: { id: { type: 'integer', minimum: 1 } },
+        required: ['id'],
+        additionalProperties: false
+      }
+    },
+    {
+      name: 'list_posts_by_user',
+      description: 'Fetch posts for a user with optional limit',
+      inputSchema: {
+        type: 'object',
+        properties: {
+          user_id: { type: 'integer', minimum: 1 },
+          limit: { type: 'integer', minimum: 1, maximum: 100 }
+        },
+        required: ['user_id'],
+        additionalProperties: false
+      }
+    },
+    {
+      name: 'search_posts',
+      description: 'Search posts by title substring, case-insensitive',
+      inputSchema: {
+        type: 'object',
+        properties: { query: { type: 'string', minLength: 1, maxLength: 200 } },
+        required: ['query'],
+        additionalProperties: false
+      }
+    }
+  ]
+}));
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+  try {
+    switch (name) {
+      case 'get_user': {
+        const { id } = GetUserSchema.parse(args);
+        const user = await getUserImpl(id);
+        return { content: [{ type: 'text', text: JSON.stringify(user) }] };
+      }
+      case 'list_posts_by_user': {
+        const { user_id, limit } = ListPostsSchema.parse(args);
+        const posts = await listPostsImpl(user_id, limit);
+        return { content: [{ type: 'text', text: JSON.stringify(posts) }] };
+      }
+      case 'search_posts': {
+        const { query } = SearchPostsSchema.parse(args);
+        const results = await searchPostsImpl(query);
+        return { content: [{ type: 'text', text: JSON.stringify(results) }] };
+      }
+      default:
+        throw new Error(`Unknown tool ${name}`);
+    }
+  } catch (err: any) {
+    return {
+      content: [{ type: 'text', text: `Error: ${err.message}` }],
+      isError: true
+    };
+  }
+});
+
+async function startServer() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('JSONPlaceholder MCP server running on stdio');
+}
+
+// ---------- Tests ----------
+async function runTests() {
+  console.log('=== MCP JSONPlaceholder Tests ===');
+  // a) Happy path
+  try {
+    const user = await getUserImpl(1);
+    console.log('a) get_user(1) ->', user.id === 1 && !!user.name ? 'PASS' : 'FAIL', user.name);
+  } catch (e: any) {
+    console.log('a) FAIL', e.message);
+  }
+
+  // b) 404 / not-found handled
+  try {
+    await getUserImpl(9999);
+    console.log('b) 404 handled -> PASS (empty)');
+  } catch (e: any) {
+    console.log('b) 404 handled -> PASS', e.message);
+  }
+
+  // c) Validation rejects malformed input
+  try { GetUserSchema.parse({ id: -5 }); console.log('c1) FAIL'); }
+  catch { console.log('c1) get_user(-5) rejected -> PASS'); }
+
+  try { ListPostsSchema.parse({ user_id: 'x' }); console.log('c2) FAIL'); }
+  catch { console.log('c2) list_posts_by_user("x") rejected -> PASS'); }
+
+  try { SearchPostsSchema.parse({ query: '' }); console.log('c3) FAIL'); }
+  catch { console.log('c3) search_posts("") rejected -> PASS'); }
+
+  console.log('Tests done');
+  process.exit(0);
+}
+
+if (process.argv.includes('--test')) {
+  runTests();
+} else {
+  startServer();
+}