[ { "id": "agarwal-2026-switchcraft", "title": "Switchcraft: AI Model Router for Agentic Tool Calling", "authors": [ "Sharad Agarwal", "Pooria Namyar", "Alec Wolman", "Rahul Ambavat", "Ankur Gupta", "Qizheng Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07112", "code_url": null, "topic_clusters": [ "C1", "C3", "C13" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "DistilBERT classifier on tool-call context", "query text" ], "cost_reduction_reported": "82.9% accuracy, matching/exceeding best single model, with 84% lower inference cost (over $3,600 saved per million queries)", "benchmarks": [ "five function-calling benchmarks", "five function-calling benchmarks (incl. BFCL)" ], "models_studied": [ "multiple tool-calling LLMs" ], "production_evidence": false, "summary": "Switchcraft is an inline model router for agentic tool calls that picks the cheapest model predicted to produce a correct call, using a DistilBERT classifier under a latency budget. It matches the best model's accuracy at 84% lower cost and finds larger models do not reliably win on tool use.", "key_idea": "Per-tool-call routing to the cheapest correct model.", "relevance_score": 5, "relevance_rationale": "Tool-call-level routing inside agents.", "found_via": "web search: model switching mid-trajectory | web search: agentic routing benchmark multi-turn", "cites_or_cited_by": [ "gonuguntla-2026-replay-gap", "liu-2026-harness-native-routing" ] }, { "id": "agyemang-2026-local-splitter", "title": "Local-Splitter: A Measurement Study of Seven Tactics for Reducing Cloud LLM Token Usage on Coding-Agent Workloads", "authors": [ "Justice Owusu Agyemang", "Jerry John Kponyo", "Elliot Amponsah", "Godfred Manu Addo Boakye", "Kwame Opuni-Boachie Obour Agyekum" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.12301", "code_url": null, "topic_clusters": [ "C3", "C10", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "local triage model", "workload class" ], "cost_reduction_reported": "Local routing + prompt compression gives 45-79% cloud token savings on edit- and explanation-heavy coding-agent workloads; full tactic set incl. draft-review gives 51% on RAG-heavy", "benchmarks": [ "four coding-agent workload classes" ], "models_studied": [], "production_evidence": false, "summary": "A measurement study of seven tactics for cutting cloud-LLM tokens when a small local model triages requests in front of a frontier model in coding agents: local routing, compression, semantic caching, local draft with cloud review, minimal diffs, intent extraction and batching with prompt caching. The tactics are delivered via an MCP/OpenAI-compatible shim.", "key_idea": "Small local model as a triage layer in front of a frontier model for coding agents; the best tactic mix depends on the workload.", "relevance_score": 5, "relevance_rationale": "Practical big-small split for coding agents, with measured token and dollar savings and open-source tooling.", "found_via": "Semantic Scholar query \"per-turn model routing coding agent\"", "cites_or_cited_by": [] }, { "id": "alazraki-2026-sale-strategy-auctions", "title": "Scaling Small Agents Through Strategy Auctions", "authors": [ "Lisa Alazraki", "William F. Shen", "Yoram Bachrach", "Akhil Mathur" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.02751", "code_url": null, "topic_clusters": [ "C1", "C3", "C8" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "strategy bids", "cost-value scoring", "auction memory" ], "cost_reduction_reported": "Reduces reliance on the largest agent by 53% and overall cost by 35%", "benchmarks": [ "deep search", "coding tasks" ], "models_studied": [], "production_evidence": false, "summary": "Shows small agents lag on complex tasks and proposes SALE, where agents of different sizes bid short strategic plans scored by a cost-value mechanism, enabling per-task routing and self-improvement without a separate router.", "key_idea": "Auction among differently sized agents for per-task routing.", "relevance_score": 5, "relevance_rationale": "Direct cost reduction via routing among small and large agents.", "found_via": "hf-papers search: enterprise agent fine-tuning small model", "cites_or_cited_by": [] }, { "id": "anthropic-2026-advisor-strategy", "title": "The advisor strategy: Give Sonnet an intelligence boost with Opus", "authors": [ "Anthropic" ], "year": 2026, "venue_or_source": "Claude blog (Anthropic)", "source_type": "blog", "url": "https://claude.com/blog/the-advisor-strategy", "code_url": null, "topic_clusters": [ "C2", "C3", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "executor-decided consultation" ], "cost_reduction_reported": "Sonnet executor + Opus advisor: +2.7 points on SWE-bench Multilingual over Sonnet alone with 11.9% lower cost per agentic task; Haiku + Opus advisor: 41.2% vs 19.7% on BrowseComp, trailing Sonnet solo by 29% but 85% cheaper per task", "benchmarks": [ "SWE-bench Multilingual", "BrowseComp", "Terminal-Bench 2.0" ], "models_studied": [ "Claude Opus (advisor)", "Claude Sonnet (executor)", "Claude Haiku (executor)" ], "production_evidence": true, "summary": "Anthropic's advisor tool lets a cheaper executor model run an agentic task end to end and consult a stronger advisor model mid-task for a plan or correction, all inside one API request, with advisor tokens billed at advisor rates. Vendor benchmarks show quality gains with lower or similar cost, especially for weaker executors.", "key_idea": "Cheap executor runs the loop and calls an expensive advisor only at hard decision points.", "relevance_score": 5, "relevance_rationale": "Productized step-level escalation pattern with vendor-reported cost and quality numbers.", "found_via": "web search: Anthropic advisor tool", "cites_or_cited_by": [] }, { "id": "anthropic-2026-optimizing-cost-and-intelligence", "title": "Optimizing for cost and intelligence", "authors": [ "Anthropic" ], "year": 2026, "venue_or_source": "Claude Platform documentation", "source_type": "docs", "url": "https://platform.claude.com/docs/en/about-claude/models/optimizing-for-cost-and-intelligence", "code_url": null, "topic_clusters": [ "C3", "C9", "C10", "C12", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "effort level", "advisor consultation", "task budget" ], "cost_reduction_reported": "Examples reported: prompt caching cut DeepResearch Bench II cost 81% (Fable 5.1) and 62.5% (Sonnet 5); on SWE-bench Pro, Opus 5.5 low effort with failures re-run at high reached 97% at ~$0.17/task vs 95.3% at $0.29 all-high; Fable 5 coordinator + Sonnet 5 worker cost ~50% of solo Fable 5 on an easy BrowseComp slice; on full BrowseComp, solo Fable 5 at lower effort was 22-30% cheaper than the orchestrator at same accuracy", "benchmarks": [ "SWE-bench Pro", "DeepResearch Bench II", "BrowseComp", "GPQA Diamond", "Terminal-Bench 3", "internal benchmarks" ], "models_studied": [ "Claude Opus 5.5", "Claude Fable 5.1", "Claude Sonnet 5", "Claude Haiku 4.5" ], "production_evidence": false, "summary": "Anthropic documentation with measured results on cost levers: caching, trimming tools and context, effort tuning, retrying failures at higher effort, task budgets, advisor pairing and orchestrator-worker delegation. It reports that advisor gains shrink as the executor approaches the advisor's capability and that cheaper-worker delegation helps on easy slices but can lose to a solo model at lower effort on the full set.", "key_idea": "Vendor-measured playbook of per-step and architecture cost levers, including when delegation to cheaper workers loses.", "relevance_score": 5, "relevance_rationale": "Rare quantitative vendor evidence comparing escalation, effort control and orchestrator-worker model splits on agentic benchmarks.", "found_via": "snowball: linked from anthropic-2026-advisor-strategy docs", "cites_or_cited_by": [] }, { "id": "bai-2026-agents-spend-your-money", "title": "How Do AI Agents Spend Your Money? Analyzing and Predicting Token Consumption in Agentic Coding Tasks", "authors": [ "Longju Bai", "Zhemin Huang", "Xingyao Wang", "Jiao Sun", "Rada Mihalcea", "Erik Brynjolfsson", "Alex Pentland", "Jiaxin Pei" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.22750", "code_url": null, "topic_clusters": [ "C9", "C10", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "token usage" ], "cost_reduction_reported": "Agentic coding uses ~1000x more tokens than code chat; runs on the same task vary up to 30x; Kimi-K2 and Claude-Sonnet-4.5 use over 1.5M more tokens than GPT-5 on average", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "eight frontier LLMs incl. GPT-5", "Claude-Sonnet-4.5", "Kimi-K2", "eight frontier LLMs" ], "production_evidence": false, "summary": "The first systematic token-consumption study of coding agents across eight frontier models on SWE-bench Verified. Input tokens dominate cost, usage is highly stochastic, accuracy saturates at intermediate cost, human difficulty ratings poorly predict tokens, and models underestimate their own token use.", "key_idea": "Where coding-agent tokens go and why they are hard to predict.", "relevance_score": 5, "relevance_rationale": "Seed-type token-consumption analysis of coding agents.", "found_via": "web search: token consumption coding agents | web search: coding agents spend tokens", "cites_or_cited_by": [ "hu-2026-cost-inefficient-coding-agents", "lin-2026-bagen-budget-aware-agents", "liu-2026-cost-utility-alignment-trajectories", "smekal-2026-task-spec-token-spend", "wang-2025-efficient-agents" ] }, { "id": "bao-2026-orcarouter", "title": "OrcaRouter: A Production-Oriented LLM Router with Hybrid Offline-Online Learning", "authors": [ "Zhenghua Bao", "Tian Fengya", "Chris Zhang", "Zhenjun Chen", "Xile Ma", "Yi SHI" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.30736", "code_url": null, "topic_clusters": [ "C1", "C5", "C14" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "lexical features", "sentence embedding", "bandit reward" ], "cost_reduction_reported": "75.54% accuracy at USD 1.00 per 1,000 queries; ranked second on RouterArena (arena score 72.08)", "benchmarks": [ "RouterArena" ], "models_studied": [ "candidate LLM pool" ], "production_evidence": false, "summary": "OrcaRouter fits per-arm ridge regressors offline from a full-information reward matrix on curated prompts, then deploys as a LinUCB contextual bandit that can keep learning from bandit feedback.", "key_idea": "Offline warm-start plus online LinUCB updates for a production-oriented router.", "relevance_score": 5, "relevance_rationale": "Practical hybrid offline-online router design with public leaderboard results.", "found_via": "OpenAlex title: online routing LLM", "cites_or_cited_by": [ "li-2010-linucb", "panda-2025-pilot" ] }, { "id": "bechard-2026-terminal-agents-suffice", "title": "Terminal Agents Suffice for Enterprise Automation", "authors": [ "Patrice Bechard", "Orlando Marquez Ayala", "Emily Chen", "Jordan Skelton", "Sagar Davasam", "Srinivas Sunkara", "Vikas Yadav", "Sai Rajeswar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.00073", "code_url": null, "topic_clusters": [ "C12", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Terminal agents match or outperform more complex agent architectures at a fraction of the cost", "benchmarks": [ "diverse real-world enterprise systems" ], "models_studied": [], "production_evidence": false, "summary": "ServiceNow researchers argue a coding agent with only a terminal and filesystem, calling platform APIs directly, solves many enterprise automation tasks better and far more cheaply than MCP tool agents or GUI web agents. Suggests simple programmatic interfaces plus strong models as the enterprise backbone.", "key_idea": "A plain terminal coding agent beats heavier enterprise agent stacks on cost.", "relevance_score": 5, "relevance_rationale": "Direct enterprise evidence on architecture-level cost reduction.", "found_via": "web search: single vs multi-agent cost", "cites_or_cited_by": [ "hajimiri-2026-skill-memory-worth-tokens" ] }, { "id": "bhola-2026-scrouting-superscout", "title": "Scrouting: Cost-Aware Routing of Coding Agents by Scouting the Repository First", "authors": [ "Ishaan Bhola", "Adithyan Krishnan", "Mukunda NS" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.04804", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "7B scout hidden states", "verified repository handoff" ], "cost_reduction_reported": "Matches best single model's solve rate on SWE-bench Pro Python slice (159 vs 158 of 266) at about a fifth of total cost per solve; the handoff, not routing, carries the result", "benchmarks": [ "SWE-bench Pro (Python, 266 tasks)" ], "models_studied": [ "SuperScout-7B", "four frontier fixers" ], "production_evidence": false, "summary": "A 7B searcher first scouts the repository and produces a sandbox-verified handoff, and a router uses its hidden states to dispatch the issue to one of four frontier fixers. Cost per solve drops to about a fifth, and an ablation shows the cheap-model handoff itself, lifting cheaper fixers, drives the gain.", "key_idea": "Small scout explores first; its verified handoff lets cheaper fixers succeed.", "relevance_score": 5, "relevance_rationale": "Big-little division in coding agents with honest ablation.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "son-2026-swe-router" ] }, { "id": "borthwick-2026-robophd", "title": "Competing at Every Price Point with Agentic Evolution over a Menu of LLMs", "authors": [ "Andrew Borthwick" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.16207", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "priced model menu", "cost target", "task score" ], "cost_reduction_reported": "Evolved agents hold every public Pareto-frontier slot but one on DS-1000 and PaperFindingBench leaderboards", "benchmarks": [ "DS-1000", "PaperFindingBench" ], "models_studied": [], "production_evidence": false, "summary": "An evolutionary meta-agent (RoboPhD) evolves complete multi-LLM agent programs from at most 100 training examples, given a priced menu of nine LLM endpoints and a per-problem cost target, to dominate competitors at each price point.", "key_idea": "Evolve multi-LLM agent programs to a cost target over a priced model menu.", "relevance_score": 5, "relevance_rationale": "Explicit cost-targeted multi-LLM agent evolution.", "found_via": "snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [] }, { "id": "cai-2026-think-big-search-small", "title": "Think Big, Search Small: Where Capacity Matters in Hierarchical Search Agents?", "authors": [ "Qinnan Cai", "Yibo Zhao", "Xiang Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.07548", "code_url": null, "topic_clusters": [ "C3", "C8", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "role capacity sweep" ], "cost_reduction_reported": "Scaling delegation backbone improves EM ~11 points vs ~2.6 for execution; a 1.7B distilled executor matches a frontier sub-agent with 37% fewer sub-agent tokens", "benchmarks": [ "five multi-hop QA benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "The authors factor hierarchical search agents into delegation, execution and answer-generation roles and sweep model capacity per role. Decomposition is the capacity bottleneck, while execution can be downsized; a 1.7B executor trained by quality-filtered trajectory distillation matches a frontier executor.", "key_idea": "Put capacity in the planner/delegator and downsize the executor.", "relevance_score": 5, "relevance_rationale": "A controlled planner-big/executor-small study with measured numbers (sub-theme b).", "found_via": "arXiv query ti:\"big-little\" OR ti:\"big and small\" AND abs:agent", "cites_or_cited_by": [] }, { "id": "chen-2026-agentslimming", "title": "AgentSlimming: Towards Efficient and Cost-Aware Multi-Agent Systems", "authors": [ "Yulang Chen", "Haoxuan Peng", "Jinyan Liu", "Zichen Wen", "Dongrui Liu", "Linfeng Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.08813", "code_url": "https://github.com/CitrusYL/AgentSlimming", "topic_clusters": [ "C3", "C4", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "agent importance score", "baseline-anchored acceptance", "agent importance scores" ], "cost_reduction_reported": "Reduces average token cost by up to 78.9% with negligible performance degradation", "benchmarks": [ "eight benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "AgentSlimming compresses graph-structured multi-agent workflows by scoring agent importance, then removing redundant agents or replacing them with low-cost ones, accepting each change only if performance holds. It achieves large token savings with little or no accuracy loss.", "key_idea": "Prune agents or downgrade them to cheaper models with validated acceptance.", "relevance_score": 5, "relevance_rationale": "Explicit subagent downscaling as a compression step.", "found_via": "web search: agent topology pruning | search: multi-objective agentic workflow optimization cost accuracy Pareto arXiv", "cites_or_cited_by": [ "zhang-2024-aflow", "zhang-2024-agentprune" ] }, { "id": "cui-2026-uno-orchestra", "title": "Uno-Orchestra: Parsimonious Agent Routing via Selective Delegation", "authors": [ "Zhiqing Cui", "Haotong Xie", "Jiahao Yuan", "Cheng Yang", "Hanqing Wang", "Yuxin Wu", "Yifan Wu", "Siru Zhong", "Tao Yu", "Yifu Guo", "Siyu Zhang", "Xinlei Yu", "Qibing Ren", "Usman Naseem" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.05007", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "task", "worker interactions" ], "cost_reduction_reported": "77.0% macro pass@1, about 16% above strongest workflow baseline at roughly an order of magnitude lower per-query cost", "benchmarks": [ "13-benchmark suite: math, code, knowledge, long-context, agentic tool use" ], "models_studied": [], "production_evidence": false, "summary": "Uno-Orchestra learns one policy that decides whether to decompose a task and dispatches each subtask to a (model, primitive) pair, trained from RL trajectories grounded in real worker interactions. It beats 22 baselines at about a tenth of the per-query cost.", "key_idea": "Jointly learned selective decomposition and subtask-to-model dispatch.", "relevance_score": 5, "relevance_rationale": "Subtask-level model routing with large cost gains.", "found_via": "OpenAlex search: orchestrator worker model cost", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "su-2025-toolorchestra", "yue-2025-masrouter", "zhang-2026-evoroute" ] }, { "id": "dennis-2026-compiling-workflows-into-weights", "title": "Compiling Agentic Workflows into LLM Weights: Near-Frontier Quality at Two Orders of Magnitude Less Cost", "authors": [ "Simon Dennis", "Rivaan Patil", "Kevin Shabahang", "Hao Guo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.22502", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "procedural workflow data" ], "cost_reduction_reported": "Title claims near-frontier quality at two orders of magnitude less cost; evaluated on travel booking, Zoom support and insurance-claims workflows", "benchmarks": [ "travel booking and other procedural tasks" ], "models_studied": [], "production_evidence": false, "summary": "Argues external orchestration frameworks can be replaced by fine-tuning procedures into model weights, testing across procedural tasks such as travel booking against frontier models with procedures in the system prompt.", "key_idea": "Compile agent workflows into a small model's weights.", "relevance_score": 5, "relevance_rationale": "Directly a production cost lever for procedural agents.", "found_via": "hf-papers search: small model orchestrator", "cites_or_cited_by": [] }, { "id": "dylan-2026-equal-inference-cost-mas", "title": "At Equal Inference Cost, Multi-Agent Structure Does Not Beat a Single Frozen Agent", "authors": [ "David Dylan", "Aoife Brennan", "Cian Murphy", "Niamh O'Sullivan", "Conor Kelly", "Saoirse Walsh" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.04217", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM call budget" ], "cost_reduction_reported": "Team 0.769 vs single agent 0.754 on ALFWorld (p=0.80) despite 1.8x more evaluation calls", "benchmarks": [ "ALFWorld", "WebShop" ], "models_studied": [ "frozen 7B backbone" ], "production_evidence": false, "summary": "Fixes the total number of LLM calls and compares evolving a planner-executor-critic team against evolving a single agent on a frozen 7B model. The team is not significantly better, its value comes entirely from the executor, and with equal cost multi-agent structure adds cost without clear benefit.", "key_idea": "Under equal call budgets, multi-agent structure does not beat a single agent.", "relevance_score": 5, "relevance_rationale": "Controlled evidence on MAS cost-effectiveness.", "found_via": "arXiv search: single vs multi-agent", "cites_or_cited_by": [ "yue-2025-masrouter", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "fang-2026-critique-and-routing-controller", "title": "Iterative Critique-and-Routing Controller for Multi-Agent Systems with Heterogeneous LLMs", "authors": [ "Wenzhi Fang", "Liangqi Yuan", "Guangchen Lan", "Dong-Jun Han", "Christopher G. Brinton" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.08686", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "draft critique", "agent-utilization constraints" ], "cost_reduction_reported": "Narrows the gap to the strongest agent while using it for fewer than 25% of calls", "benchmarks": [ "seven reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "A controller evaluates the current draft each turn, decides whether to stop and otherwise picks the next heterogeneous agent to refine it, formulated as a constrained finite-horizon MDP optimized with Lagrangian policy gradients. It approaches the strongest agent's quality while calling it under 25% of the time.", "key_idea": "Iterative critique-then-route controller with utilization constraints.", "relevance_score": 5, "relevance_rationale": "Multi-turn routing with explicit budget constraints.", "found_via": "arXiv title search: routing multi-agent system LLM", "cites_or_cited_by": [ "jin-2025-corl-centralized-budget" ] }, { "id": "farzaneh-2026-tsds-edge-deferral", "title": "Think Short, Defer Smart, Act, and Repeat: Calibrated Reasoning and Uncertainty-Aware Deferral for Edge LLM Agents", "authors": [ "Amirmohammad Farzaneh", "Osvaldo Simeone" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.26865", "code_url": null, "topic_clusters": [ "C2", "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "convergence probe", "perplexity", "Learn-Then-Test calibration" ], "cost_reduction_reported": "Per-episode thinking compute reduced 43%-65% vs deferral-only baselines on HotpotQA, MBPP and household-robot tasks, with certified reward and cloud-call-rate guarantees", "benchmarks": [ "GSM8K", "HotpotQA", "MBPP", "household robot planning" ], "models_studied": [], "production_evidence": false, "summary": "TSDS targets ReAct agents running on edge devices. A convergence probe stops on-device reasoning once the intended action stabilizes, and a perplexity rule defers uncertain actions to a cloud model; both thresholds are jointly calibrated on episode trajectories with Learn-Then-Test for finite-sample guarantees on reward and cloud-call rate.", "key_idea": "Per-action deferral from a small local model to a cloud model inside ReAct loops, with statistical guarantees.", "relevance_score": 5, "relevance_rationale": "Per-step small-to-large deferral inside a ReAct loop with certified cost control.", "found_via": "arXiv query ti:\"deferral\" AND abs:\"LLM agent\"", "cites_or_cited_by": [] }, { "id": "feng-2026-graphplanner", "title": "GraphPlanner: Graph Memory-Augmented Agentic Routing for Multi-Agent LLMs", "authors": [ "Tao Feng", "Haozhen Zhang", "Zijie Lei", "Peixuan Han", "Jiaxuan You" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.23626", "code_url": "https://github.com/ulab-uiuc/GraphPlanner", "topic_clusters": [ "C1", "C3" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "heterogeneous graph memory of queries/agents/responses" ], "cost_reduction_reported": "Up to 9.3% higher accuracy while reducing GPU cost from 186.26 GiB to 1.04 GiB", "benchmarks": [ "14 LLM tasks" ], "models_studied": [], "production_evidence": false, "summary": "GraphPlanner generates routing workflows per query as an MDP, choosing at each step both an LLM backbone and an agent role (planner, executor, summarizer), with graph memory of past interactions and RL optimizing accuracy and efficiency. It beats single- and multi-round routers and generalizes to unseen tasks and models.", "key_idea": "RL over per-step (role, backbone) choices with graph memory.", "relevance_score": 5, "relevance_rationale": "Step-level role and model routing.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [ "xu-2026-learning-what-to-skip" ] }, { "id": "fireworks-2026-firerouter-with-opus", "title": "Introducing FireRouter with Opus", "authors": [ "Fireworks AI" ], "year": 2026, "venue_or_source": "Fireworks AI blog", "source_type": "product", "url": "https://fireworks.ai/blog/introducing-firerouter-with-opus", "code_url": null, "topic_clusters": [ "C1", "C3", "C5", "C10", "C14" ], "routing_granularity": "step", "learning_regime": null, "signals_used": [ "per-turn task suitability estimate", "estimated cost per model", "prompt-cache loss cost" ], "cost_reduction_reported": "Internal A/B test (>1 month, sessions randomised to router vs Opus-only): cost per session $15.36 -> $6.63 (57% +/- 19 ppts lower); graded-turn accuracy 78.7% vs 80.2% (98.1% of Opus-only); cache hit rate 94.2% vs 97.8% (vendor-reported)", "benchmarks": [ "internal coding traffic A/B test" ], "models_studied": [ "Claude Opus 5.5", "GLM 5.3", "GLM 5.3 Flash" ], "production_evidence": true, "summary": "Fireworks releases a per-turn, cache-aware router for coding agents that routes each user turn among Claude Opus 5.5, GLM 5.3 and GLM 5.3 Flash, trading off estimated suitability against cost including the prompt-cache value lost by switching. It reports a randomised A/B test on internal coding traffic, scoring accuracy by task completion, correctness and whether the user corrected the next turn.", "key_idea": "Turn-level routing in coding agents with explicit pricing of cache loss, validated by a randomised live-traffic A/B test.", "relevance_score": 5, "relevance_rationale": "Step-level cost-aware routing in a production agent with a controlled live-traffic experiment (vendor-run).", "found_via": "direct: fireworks.ai/blog index", "cites_or_cited_by": [ "fireworks-2026-frontier-is-a-router", "notdiamond-2026-interactive-benchmarks-routing" ] }, { "id": "fireworks-2026-frontier-is-a-router", "title": "The frontier isn't a model. It's a router.", "authors": [ "Fireworks AI" ], "year": 2026, "venue_or_source": "Fireworks AI blog", "source_type": "blog", "url": "https://fireworks.ai/blog/the-frontier-isnt-a-model-its-a-router", "code_url": null, "topic_clusters": [ "C1", "C3", "C13", "C14" ], "routing_granularity": "query", "learning_regime": null, "signals_used": [ "task" ], "cost_reduction_reported": "DeepSWE v1.1 oracle over 18 models: 97.6% pass@1 at $1.88/task vs best single model GPT-6 Astra 74.1% at $6.52; open-models-only oracle 90.3% at $1.45; best pair +13.1 pts, best trio 91.2%. Production: FireRouter sessions $7.42 vs $15.81 for Opus 5 alone (53% reduction, 2,334 sessions over four weeks of internal coding traffic) (vendor-reported)", "benchmarks": [ "DeepSWE v1.1" ], "models_studied": [ "GPT-6 Astra", "Claude Opus 5", "GPT-5.6 Sol", "Claude Fable 5", "DeepSeek V4 Flash/Pro", "GLM-5.3", "GLM-5.3 Flash", "Kimi K3", "Qwen3.8 Max" ], "production_evidence": true, "summary": "Runs 18 models on every DeepSWE task to measure the per-task oracle routing ceiling, showing most of the gain comes from two or three complementary models and most oracle picks are cheap models. It stresses that oracle upside is upward-biased and that real routers fail on model recall (citing LLMRouterBench), then reports four weeks of internal production coding traffic through task-level, cache-aware FireRouter.", "key_idea": "Complementary models give a large oracle headroom for task-level routing in coding agents, but capturing it requires predicting the winner before execution.", "relevance_score": 5, "relevance_rationale": "Agentic-task routing ceiling analysis plus production cost numbers.", "found_via": "direct: fireworks.ai/blog index", "cites_or_cited_by": [ "fireworks-2026-firerouter-with-opus", "together-2026-deepswe-cascade-deepseek-gpt" ] }, { "id": "furman-2026-replit-free-the-models", "title": "Free the models: Harness design at the frontier", "authors": [ "Daniel Furman", "Jacky Zhao", "Vaibhav Kumar", "Ed Sioufi", "Michele Catasta" ], "year": 2026, "venue_or_source": "Replit blog", "source_type": "blog", "url": "https://replit.com/blog/free-the-models", "code_url": null, "topic_clusters": [ "C3", "C9", "C12", "C14" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "main-agent (LLM) judgement of subtask difficulty", "effort level", "subagent tier" ], "cost_reduction_reported": "DeepSWE v1.1: Replit Agent 72% at $2.11/task vs GPT-6 Astra (mini-swe-agent) 67% at $1.60 (low effort) and 74% at $4.43 (xhigh); sidekick architecture 61% at $1.34. Terminal-Bench 4.0: 49% at $2.53 vs Astra 42% at $2.25 (low) and 60% at $5.86 (xhigh); sidekick 33% at $1.84 (means of 4 runs; vendor-reported). Production delegation rates: 21-36% of turns dispatch a subagent", "benchmarks": [ "DeepSWE v1.1", "Terminal-Bench 4.0", "Replit Agent production traffic (delegation statistics)" ], "models_studied": [ "GPT-6 Astra", "Claude Fable 5", "Claude Fable 5.1" ], "production_evidence": true, "summary": "Replit argues that an external router is always less capable than the model it routes for, and instead lets the main agent choose at each step its own effort (changed mid-turn without cache loss on GPT-6), whether to delegate, and which subagent tier (small/standard/large), effort and specialist to use, including reusing already-briefed subagents. It reports production delegation statistics per model and a Pareto comparison against single-model baselines and a one-worker sidekick design.", "key_idea": "Let the frontier orchestrator itself assign tiers and effort to subagents rather than routing externally.", "relevance_score": 5, "relevance_rationale": "Directly addresses step/subagent-level downscaling in production agents with cost-quality results; useful counter-position to external routers.", "found_via": "direct: replit.com/blog index", "cites_or_cited_by": [ "amp-2026-subagent-model-assignment", "furman-2026-replit-closing-the-loop-evals", "lin-2026-cursor-agent-swarm-economics", "okeefe-2026-cursor-router" ] }, { "id": "gandhi-2026-steer-dont-solve-critic", "title": "Steer, Don't Solve: Training Small Critic Models for Large Code Agents", "authors": [ "Shubham Gandhi", "Yiqing Xie", "Atharva Naik", "Ruichen Zhu", "Carolyn Rose" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.21811", "code_url": null, "topic_clusters": [ "C3", "C7", "C8" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "small critic model" ], "cost_reduction_reported": "30-92x lower critic cost than a strong teacher; on Qwen3-Next-80B-A3B both more accurate (25.2% vs 20.8%) and cheaper ($0.04 vs $0.11)", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "CWM-32B", "Qwen agents", "Qwen3-Next-80B-A3B" ], "production_evidence": false, "summary": "Freezes the code agent and trains a small critic on agent trajectories to steer strategy-level reasoning; the critic transfers to unseen agents and improves accuracy at lower cost.", "key_idea": "A small trained critic steers a large agent cheaply.", "relevance_score": 5, "relevance_rationale": "Cheap signal plus specialization improving cost-quality.", "found_via": "hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [] }, { "id": "ganz-2026-handoff-tax", "title": "The Handoff Tax: Continuing Non-Native Trajectories in LLM Agents", "authors": [ "Roy Ganz", "Mor Shpigel Nacson", "Adi Kalyanpur", "Ron Litman" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.24358", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Full-trajectory escalation recovers less than half of the cheap-to-strong quality gap at a substantial cost premium; downshifting offers a favorable cost-quality point", "benchmarks": [ "coding agent tasks" ], "models_studied": [ "Claude and GPT low- and high-capability model pairs" ], "production_evidence": false, "summary": "Studies mid-trajectory model handoffs in coding agents, varying direction, timing and how much of the prior trajectory the receiving model sees. Escalating a cheap model's trajectory to a strong one recovers under half the gap at high cost (the handoff tax), while downshifting after hard reasoning is favorable; less inherited context helps escalation.", "key_idea": "Mid-trajectory model switches carry a handoff tax; downshifting works better than escalating.", "relevance_score": 5, "relevance_rationale": "Key empirical evidence for in-trajectory model switching.", "found_via": "web search: model switching mid-trajectory", "cites_or_cited_by": [ "hemadri-2026-r2v-agent", "son-2026-swe-router" ] }, { "id": "gao-2026-agentcollab", "title": "AgentCollab: A Self-Evaluation-Driven Collaboration Paradigm for Efficient LLM Agents", "authors": [ "Wenbo Gao", "Renxi Liu", "Xian Wang", "Fang Guo", "Shuai Yang", "Xi Chen", "Hui-Ling Zhen", "Hanting Chen", "Weizhe Lin", "Xiaosong Li", "Yaoyuan Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.26034", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "self-reflection progress signal", "recent failure signals" ], "cost_reduction_reported": "Consistently improves the accuracy-efficiency Pareto frontier (no single number in abstract)", "benchmarks": [ "diverse multi-step agent benchmarks" ], "models_studied": [ "two-level small-large model setting" ], "production_evidence": false, "summary": "AgentCollab runs a cheap model by default and escalates control to a stronger model only when the agent's own self-reflection indicates it is not making progress, with a difficulty-aware cumulative escalation that allocates more budget after recent failures. The search summary notes that putting the large model on execution rather than planning gave higher accuracy.", "key_idea": "Self-evaluated progress triggers escalation to a stronger model mid-trajectory.", "relevance_score": 5, "relevance_rationale": "Step-level escalation without an external router.", "found_via": "web search: planner executor", "cites_or_cited_by": [ "zhang-2026-budget-aware-agentic-routing" ] }, { "id": "gonuguntla-2026-replay-gap", "title": "The Replay Gap: Static Evaluation of Model Switching in LLM Agents Scores the Wrong World", "authors": [ "Ashritha Gonuguntla" ], "year": 2026, "venue_or_source": "COLM 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2608.08239", "code_url": null, "topic_clusters": [ "C3", "C6", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "branching rollouts" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-bench" ], "models_studied": [], "production_evidence": false, "summary": "Tests the common practice of evaluating in-agent model switching by replaying logged trajectories, using live forked SWE-bench rollouts. Swaps rewrite 61-94% of subsequent actions and replay evaluation mispredicts every success-relevant outcome, so static replay scores the wrong world.", "key_idea": "Offline replay badly misestimates the effect of mid-trajectory model swaps.", "relevance_score": 5, "relevance_rationale": "Crucial caveat for learning step-level routing from logs.", "found_via": "web search: step-level routing", "cites_or_cited_by": [ "agarwal-2026-switchcraft", "wei-2026-step-level-cua-cascade", "yang-2026-twinrouterbench" ] }, { "id": "gravara-2026-atlas", "title": "Atlas: Optimizing Deployment of Compound AI Workflows on Heterogeneous Clusters", "authors": [ "Milos Gravara", "Andrija Stanisic", "Stefan Nastic" ], "year": 2026, "venue_or_source": "IEEE/ACM SEC 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2609.04513", "code_url": null, "topic_clusters": [ "C4", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "Markovian accuracy predictor", "SLOs", "MILP" ], "cost_reduction_reported": "Selects plans within 0.03 of oracle accuracy while reducing deployment cost by up to 42% via heterogeneous placement; up to 2.6x lower profiling cost", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Atlas picks models per workflow stage and places them on a heterogeneous cluster via MILP, using a Markovian accuracy predictor that composes stage-to-stage accuracy transitions instead of full end-to-end profiling.", "key_idea": "Stage-wise model choice and placement via MILP with composed accuracy estimates.", "relevance_score": 5, "relevance_rationale": "Per-stage model selection with cost/SLOs.", "found_via": "snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [ "gravara-2026-compass" ] }, { "id": "guo-2026-routenlp", "title": "RouteNLP: Closed-Loop LLM Routing with Conformal Cascading and Distillation Co-Optimization", "authors": [ "Dongxin Guo", "Jikun Wu", "Siu Ming Yiu" ], "year": 2026, "venue_or_source": "ACL 2026 Industry Track", "source_type": "paper", "url": "https://arxiv.org/abs/2604.23577", "code_url": null, "topic_clusters": [ "C1", "C2", "C8", "C14" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "difficulty-aware router", "conformal confidence thresholds", "quality signals", "escalation failure clusters" ], "cost_reduction_reported": "8-week enterprise pilot (~5K queries/day): 58% lower inference cost at 91% response acceptance, p99 latency 1,847 ms to 387 ms; 40-85% cost reduction retaining 96-100% quality on a six-task benchmark", "benchmarks": [ "six-task finance/customer-service/legal benchmark" ], "models_studied": [], "production_evidence": true, "summary": "Closed-loop framework combining a difficulty-aware router, conformal cascading and a loop that clusters escalation failures, distils cheaper models on them and retrains the router. Reports an 8-week enterprise customer-service pilot.", "key_idea": "Route, cascade, and distil on escalation failures in a closed loop.", "relevance_score": 5, "relevance_rationale": "Production evidence and a closed-loop design that learns from traffic; directly on-premise for enterprise systems.", "found_via": "arXiv title search: cascade LLM", "cites_or_cited_by": [] }, { "id": "hamri-2026-zebra-budget-allocation", "title": "ZEBRA: Zero-shot Budgeted Resource Allocation for LLM Orchestration", "authors": [ "May Hamri", "Inbal Talgam-Cohen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.20485", "code_url": null, "topic_clusters": [ "C3", "C4", "C9" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "LLM-estimated per-phase utility curves" ], "cost_reduction_reported": "At 0.5 of unconstrained spend, recovers 94.4% of unconstrained quality vs 88.1% for LLM-direct on APPS; beats LLM-direct by 14.3pp on 3-phase HotpotQA pipeline", "benchmarks": [ "APPS", "HotpotQA" ], "models_studied": [], "production_evidence": false, "summary": "ZEBRA splits a fixed monetary budget across phases of a multi-agent pipeline by having an LLM estimate per-phase utility curves and solving a continuous knapsack with water-filling. It beats letting an LLM allocate directly and adapts splits to pipeline structure.", "key_idea": "Knapsack-based budget split across pipeline phases.", "relevance_score": 5, "relevance_rationale": "Per-phase budget allocation in MAS.", "found_via": "arXiv search: single vs multi-agent | search: BAMAS budget-aware multi-agent systems LLM provisioning ILP arXiv", "cites_or_cited_by": [ "jin-2025-corl-centralized-budget", "su-2025-daao" ] }, { "id": "hemadri-2026-r2v-agent", "title": "R2V Agent: Teaching SLMs When to Ask for Help", "authors": [ "Raghu Vamshi Hemadri", "Humaira Firdowse Mohammed", "Rishabh Maheshwary", "Srivatsava Daruru", "Sagar Davasam", "Vikas Yadav", "Srinivas Sunkara", "Sai Rajeswar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.16604", "code_url": null, "topic_clusters": [ "C2", "C3", "C7", "C8" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process verifier scores", "Brier-calibrated failure risk", "CVaR objective" ], "cost_reduction_reported": "94.3% HumanEval+ success with 0.60% LLM escalation; TextWorld 64.6% to 98.2% at 41.7% escalation; 93.3% TerminalBench at 33.9% LLM calls, about half heuristic-router cost", "benchmarks": [ "HumanEval+", "TextWorld", "TerminalBench" ], "models_studied": [ "four SLM backbones", "teacher LLM" ], "production_evidence": false, "summary": "R2V-Agent distills an SLM policy from a teacher LLM (behavior cloning then verifier-guided DPO), then trains a calibrated step-level router on the SLM's residual failures with a CVaR-constrained objective. The router escalates to the LLM only when failure risk warrants, improving the reliability-cost frontier.", "key_idea": "Calibrated per-step risk router decides when a distilled SLM should hand off to an LLM.", "relevance_score": 5, "relevance_rationale": "Directly addresses step-level escalation with distillation.", "found_via": "web search: step-level routing", "cites_or_cited_by": [ "ganz-2026-handoff-tax" ] }, { "id": "hong-2026-dats-difficulty-aware-topology", "title": "Learning How Much to Collaborate: Difficulty-Aware Topology Selection for Multi-Agent Code Generation", "authors": [ "Yunsong Hong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.13890", "code_url": null, "topic_clusters": [ "C1", "C3", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "39 interpretable difficulty features", "cost-penalty scalar" ], "cost_reduction_reported": "At 40% of always-hierarchical cost, 77.7% pass@1 vs 73.6% always-hierarchical and 74.3% strongest learned competitor; hierarchical advantage grows from 2.4 to 21.1 points with difficulty at ~10x tokens", "benchmarks": [ "APPS", "HumanEval+", "LiveCodeBench", "400 math problems" ], "models_studied": [ "four backbones" ], "production_evidence": false, "summary": "Shows the benefit of hierarchical multi-agent collaboration over a single agent grows strongly with problem difficulty while costing ~10x tokens, and trains a selector that picks the topology maximizing predicted success minus cost. Under budget-matched evaluation it beats always-hierarchical at 40% of its cost.", "key_idea": "Choose single agent vs MAS topology per problem by predicted difficulty and cost.", "relevance_score": 5, "relevance_rationale": "Direct per-query decision of when MAS is worth it.", "found_via": "arXiv API search", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer" ] }, { "id": "hua-2026-agentopt", "title": "AgentOpt v0.1 Technical Report: Client-Side Optimization for LLM-Based Agent", "authors": [ "Wenyue Hua", "Sripad Karne", "Qian Xie", "Armaan Agrawal", "Nikos Pagonas", "Kostis Kaffes", "Tianyi Peng" ], "year": 2026, "venue_or_source": "arXiv (technical report)", "source_type": "paper", "url": "https://arxiv.org/abs/2604.06296", "code_url": "https://agentoptimizer.github.io/agentopt/", "topic_clusters": [ "C3", "C4", "C5" ], "routing_granularity": "subagent", "learning_regime": "bandit-online", "signals_used": [ "small evaluation set", "accuracy", "cost" ], "cost_reduction_reported": "At matched accuracy, best vs worst model combinations differ 13-32x in cost; UCB-E recovers near-optimal accuracy with 62-76% less evaluation budget than brute force", "benchmarks": [ "four benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "AgentOpt is a framework-agnostic client-side package that searches for the most cost-effective assignment of models to pipeline roles from a small evaluation set, using bandit and BO search algorithms. It shows huge cost spreads across model combinations and finds near-optimal assignments cheaply.", "key_idea": "Bandit search over per-role model assignments for agent pipelines.", "relevance_score": 5, "relevance_rationale": "Directly optimizes role-level model assignment with measured cost spreads.", "found_via": "web search: step-level routing", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "huang-2026-scope", "title": "SCOPE: Cost-Efficient Model Selection for Compound AI Systems under Quality Constraints", "authors": [ "Yiqian Huang", "Shiqi Zhang", "Tianyuan Jin", "Xiaokui Xiao" ], "year": 2026, "venue_or_source": "KDD 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2606.00774", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "per-query cost and quality", "confidence bounds" ], "cost_reduction_reported": "Under same search budget and quality constraint, up to 20x lower cost than best competitor during search and up to 6x lower final cost in returned solution (three data processing tasks, 7 baselines)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formalizes constrained LLM selection for compound AI systems: choose an LLM per module to minimize average cost subject to a quality threshold. SCOPE uses per-query results and confidence bounds to guide search, with guarantees on meeting the threshold and near-optimal cost.", "key_idea": "Quality-constrained cost minimization over per-module LLM assignment with confidence-bound search.", "relevance_score": 5, "relevance_rationale": "Exactly the per-module model-choice-with-cost problem, with theoretical guarantees.", "found_via": "search: joint optimization model selection prompt compound AI system cost quality arXiv", "cites_or_cited_by": [ "chen-2025-llmselector", "he-2025-cognify" ] }, { "id": "jali-2026-turn-adaptive-budgets", "title": "Not All Turns Are Equally Hard: Adaptive Thinking Budgets For Efficient Multi-Turn Reasoning in Agents", "authors": [ "Neharika Jali", "Anupam Nayak", "Gauri Joshi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.05164", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "conversation history", "global token constraint", "global token budget" ], "cost_reduction_reported": "Saves up to 35% tokens and reduces latency up to 30% while maintaining accuracy; plan-aware variant saves up to 40% tokens over baselines", "benchmarks": [ "diverse agentic benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Treats multi-turn reasoning as sequential budget allocation (multi-objective MDP) and trains a GRPO policy that gives easy turns small budgets and saves tokens for hard turns.", "key_idea": "Learned per-turn thinking budgets under a global cap.", "relevance_score": 5, "relevance_rationale": "Step-level reasoning budget allocation inside agents.", "found_via": "arXiv title search: thinking budget | web search: adaptive compute per step", "cites_or_cited_by": [ "zhang-2026-budget-aware-agentic-routing" ] }, { "id": "jiang-2026-agentcard", "title": "Specialize Roles, Mix Deployments: Pushing the Cost-Accuracy Frontier of LLM Agent Teams", "authors": [ "Yinsicheng Jiang", "Liang Cheng", "Yeqi Huang", "Yufan Zhao", "Zhan Lu", "Li Dong", "Wenda Li", "Edoardo Ponti", "Luo Mai" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.20629", "code_url": "https://github.com/Auto-CAP/AgentCAP", "topic_clusters": [ "C3", "C4", "C11", "C13" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "role-level Shapley diagnostics", "unified API/self-hosted cost model", "role-level evaluation", "Shapley role diagnostics", "API/self-hosted cost model" ], "cost_reduction_reported": "Heterogeneous teams improve accuracy by up to 44% over cost-equivalent homogeneous teams, or match the strongest homogeneous team at up to 12x lower per-task cost via hybrid deployment", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AgentCARD is a role-aware benchmark suite that evaluates agent teams across which model fills each role (planner, executor, verifier) and where it is hosted (API, self-hosted, hybrid), with Pareto analysis and Shapley diagnostics of role bottlenecks. Heterogeneous teams dominate the cost-accuracy frontier, and whether the planner or executor is the bottleneck depends on the domain.", "key_idea": "Benchmark role-by-role model assignment and deployment mode for cost-accuracy.", "relevance_score": 5, "relevance_rationale": "Directly measures role-level model assignment trade-offs.", "found_via": "web search: heterogeneous MAS roles cost | search: heterogeneous LLM assignment multi-agent system roles optimization cost arXiv", "cites_or_cited_by": [ "kapoor-2025-hal-holistic-agent-leaderboard" ] }, { "id": "karmakar-2026-agentfloor", "title": "AgentFloor: How Far Up the tool use Ladder Can Small Open-Weight Models Go?", "authors": [ "Ranit Karmakar", "Jayita Chatterjee" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.00334", "code_url": null, "topic_clusters": [ "C3", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Strongest open-weight model matches GPT-5 on the benchmark while being substantially cheaper and faster", "benchmarks": [ "AgentFloor (30 tasks, six-tier ladder)" ], "models_studied": [ "16 open-weight models 0.27B-32B", "GPT-5" ], "production_evidence": false, "summary": "AgentFloor is a 30-task ladder from instruction following to long-horizon planning, run with 16 open models and GPT-5 over 16,542 scored runs. Small and mid-size open models handle most short, structured tool-use steps, while frontier models keep an edge on long-horizon planning.", "key_idea": "Map which agent step types need frontier models.", "relevance_score": 5, "relevance_rationale": "Empirical basis for per-step downscaling decisions.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "ke-2026-mas-orchestra", "title": "MAS-Orchestra: Understanding and Improving Multi-Agent Reasoning Through Holistic Orchestration and Controlled Benchmarks", "authors": [ "Zixuan Ke", "Yifei Ming", "Austin Xu", "Ryan Chin", "Xuan-Phi Nguyen", "Prathyusha Jwalapuram", "Jiayu Wang", "Semih Yavuz", "Caiming Xiong", "Shafiq Joty" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.14652", "code_url": null, "topic_clusters": [ "C3", "C12", "C13" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "task structure axes (depth, horizon, breadth, parallel, robustness)" ], "cost_reduction_reported": "More than 10x efficiency over strong baselines while improving accuracy", "benchmarks": [ "MASBENCH", "math reasoning", "multi-hop QA", "search-based QA" ], "models_studied": [], "production_evidence": false, "summary": "MAS-Orchestra trains an orchestrator with function-calling RL to generate an entire MAS at once, treating sub-agents as callable functions. Its controlled MASBENCH shows MAS gains depend on task depth, horizon, breadth, parallelism and robustness and on orchestrator and sub-agent capability, not universally.", "key_idea": "Holistic RL orchestration plus a benchmark of when MAS beats single agents.", "relevance_score": 5, "relevance_rationale": "Both a cost-efficient orchestrator and evidence on when MAS helps.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "gao-2025-single-or-multi-agent-why-not-both", "jiang-2026-cera-moa", "jin-2025-corl-centralized-budget", "kapoor-2024-ai-agents-that-matter", "liu-2026-e3-orch", "liu-2026-treecredit", "tran-2026-single-vs-multi-agent-equal-budget", "zhang-2025-maas-agentic-supernet" ] }, { "id": "khosravi-2026-which-llm-which-work", "title": "Which LLM for Which Work? Budgeted Model Allocation under Uncertain Evaluation", "authors": [ "Hamed Khosravi", "Xiao-Ming Huo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.29560", "code_url": null, "topic_clusters": [ "C1", "C6", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "proxy quality scores", "production log" ], "cost_reduction_reported": "On paid software tasks, better information about model quality yields more savings than further optimization of the assignment", "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Frames a company assigning LLMs to recurring workloads under a fixed budget as a knapsack whose quality table is uncertain because models are rarely compared on the same work and scores are proxies. Gives a two-solve certificate for when an assignment is optimal across all tables consistent with evidence and CASE, which targets evaluation at the pairs that matter, tested on a production log.", "key_idea": "Certify a budgeted model-to-workload assignment under uncertain, proxy-based evaluation and spend evaluation where it changes the decision.", "relevance_score": 5, "relevance_rationale": "Directly about choosing models per workload from logged, confounded evaluation data under a budget.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "kon-2026-swe-protege", "title": "SWE-Prot\\'eg\\'e: Learning to Selectively Collaborate With an Expert Unlocks Small Language Models as Software Engineering Agents", "authors": [ "Patrick Tser Jern Kon", "Archana Pradeep", "Ang Chen", "Alexander P. Ellis", "Warren Hunt", "Zijian Wang", "John Yang", "Samuel Thompson" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.22124", "code_url": null, "topic_clusters": [ "C2", "C3", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "stall detection (learned)", "selective expert calls" ], "cost_reduction_reported": "Qwen2.5-Coder-7B reaches 42.4% Pass@1 on SWE-bench Verified (+25.4% over prior SLM SOTA) using ~4 expert calls per task and 11% of tokens", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "Qwen2.5-Coder-7B-Instruct", "strong expert model", "Qwen2.5-Coder-7B" ], "production_evidence": false, "summary": "SWE-Protege post-trains a 7B coding model to remain the decision-maker while learning when to ask a strong expert model for help and how to follow its advice, using SFT on expert-augmented trajectories and RL that penalizes looping and unhelpful consultations. It reaches 42.4% on SWE-bench Verified with sparse expert use.", "key_idea": "Train the small agent to decide when to consult a large expert.", "relevance_score": 5, "relevance_rationale": "Learned step-level escalation in long-horizon coding agents.", "found_via": "arXiv title search: small language models agentic | hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "kumar-2026-task-session-routing-eval", "title": "Task- and Session-Level Model Routing: A Common-Interface Hybrid Evaluation of Four Open-Source Routers Across Four Benchmarks", "authors": [ "Kiran N. Kumar", "Santhosh K. Saminathan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.14641", "code_url": null, "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "router tier outputs", "router tier assignments" ], "cost_reduction_reported": "No evidence of task-specific advantage over content-blind allocation for tested open-source routers", "benchmarks": [ "RouterBench", "BFCL v4", "tau2-bench", "WebArena" ], "models_studied": [ "vLLM Semantic Router", "Aurelio", "Aurelio semantic router", "two others" ], "production_evidence": false, "summary": "Evaluates four open-source routers under a common protocol on agentic benchmarks and finds most emit near-constant tiers; gains track tier composition rather than task-specific targeting, so fixed-tier baselines are essential controls.", "key_idea": "Routers often do no better than share-matched fixed tiers.", "relevance_score": 5, "relevance_rationale": "Evaluation hygiene for agent routers.", "found_via": "snowball: cited-by ong-2024-routellm | web search: routing among agents | web search: SPROUT cost-aware routing benchmark", "cites_or_cited_by": [ "barres-2025-tau2-bench", "hu-2024-routerbench", "ong-2024-routellm", "vllm-2025-semantic-router", "zhang-2026-evoroute", "zhou-2023-webarena" ] }, { "id": "kwartler-2026-harness-tokenomics", "title": "Harness Tokenomics: A Router for the Enterprise Agentic Control Plane", "authors": [ "Ted Kwartler", "Alan Aqrawi", "Arian Abbasi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.28919", "code_url": null, "topic_clusters": [ "C1", "C3", "C10", "C12", "C14" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "calibrated prompt classifier", "prompt-cache state", "price sheet", "calibrated prompt classifier over request taxonomy" ], "cost_reduction_reported": "Recovers 13-21% of model spend ($3.3M-$5.1M/year) in an emulated 10,000-seat enterprise at Anthropic list prices; finds highest-priced model can cost less than the next tier on long tool-heavy sessions", "benchmarks": [ "~10,000 real sessions from public datasets" ], "models_studied": [ "Claude models", "Claude Code", "OpenAI Codex harnesses" ], "production_evidence": false, "summary": "Builds a cache-aware router for enterprise coding-agent harnesses that routes at session start, side lanes and subagent launch while avoiding cache rebuilds, and derives when mid-task model switches pay back.", "key_idea": "Prompt-cache-aware routing inside agent harnesses.", "relevance_score": 5, "relevance_rationale": "Directly about enterprise agent cost, caching and subagent routing.", "found_via": "snowball: cited-by ong-2024-routellm | arXiv API search", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "lei-2026-tacit-switch", "title": "TACIT-Switch: Cost-Aware Model Escalation for LLM Agents from Censored Supervision", "authors": [ "Ji'an Lei", "Jian Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.27911", "code_url": null, "topic_clusters": [ "C2", "C3", "C6" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "trajectory risk", "task features", "teacher-annotated censored intervention times", "cumulative trajectory risk" ], "cost_reduction_reported": "+7.4-11.1 pp success over task-level, step-level and fixed-prefix routing at comparable cost (simulation); best held-out success on ALFWorld (48.5% with 4B cheap, 45.5% with 9B cheap) and DABench (73.1%)", "benchmarks": [ "ALFWorld", "DABench" ], "models_studied": [ "4B and 9B cheap backbones" ], "production_evidence": false, "summary": "Learns when to permanently hand off an agent trajectory from a cheap to a strong backbone, using interval-censored teacher annotations of when intervention was needed. A mixture-cure threshold model estimates handoff timing; no teacher is needed at deployment.", "key_idea": "Survival-style model of when to switch an agent to a stronger backbone.", "relevance_score": 5, "relevance_rationale": "Mid-trajectory model escalation learned from offline data.", "found_via": "arXiv title search: escalation LLM | web search: model switching mid-trajectory", "cites_or_cited_by": [ "piatrashyn-2026-redact", "son-2026-swe-router", "zhang-2026-evoroute" ] }, { "id": "leins-2026-when-orchestration-pays-off", "title": "When Does LLM Orchestration Pay Off? A Controlled Evaluation of Accuracy, Cost, and Task Difficulty", "authors": [ "Nicolas Leins", "Nico Pelleriti", "Jana Gonnermann-Müller", "Sebastian Pokutta" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.00685", "code_url": null, "topic_clusters": [ "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Largest gain 4.6 points over optimized CoT while needing roughly 2-4x the tokens of task-only inference", "benchmarks": [ "competitive programming", "chess puzzles", "mathematics" ], "models_studied": [ "five LLM backbones" ], "production_evidence": false, "summary": "A controlled comparison of Self-Refine, Best-of-N and Debate against optimized single-call baselines, with equal GEPA optimization budgets. Orchestration gives modest benchmark-dependent gains at 2-4x tokens, does not help more on harder items, and depends strongly on the backbone.", "key_idea": "Orchestration gains are modest, model-specific and costly.", "relevance_score": 5, "relevance_rationale": "Controlled evidence on when multi-call orchestration is worth it.", "found_via": "snowball: cited-by kapoor-2024-ai-agents-that-matter", "cites_or_cited_by": [ "su-2025-daao", "tran-2026-single-vs-multi-agent-equal-budget", "zhang-2025-maas-agentic-supernet" ] }, { "id": "li-2026-flowcompile", "title": "FlowCompile: An Optimizing Compiler for Structured LLM Workflows", "authors": [ "Junyan Li", "Zhang-Wei Hong", "Maohao Shen", "Yang Zhang", "Chuang Gan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.13647", "code_url": null, "topic_clusters": [ "C4", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "per-sub-agent profiles", "structure-aware proxy" ], "cost_reduction_reported": "Up to 6.4x speedup vs heuristically optimized configurations and routing-based baselines", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "FlowCompile compiles structured LLM workflows by profiling each sub-agent under diverse configurations and composing measurements with a structure-aware proxy to find a reusable accuracy-latency trade-off set in one compile-time pass.", "key_idea": "Compile-time per-sub-agent config profiling composed into workflow-level trade-offs.", "relevance_score": 5, "relevance_rationale": "Per-sub-agent configuration optimization with reusable Pareto set.", "found_via": "search: FlowBank query-adaptive agentic workflows precompute reuse", "cites_or_cited_by": [] }, { "id": "li-2026-jev-as-a-judge", "title": "JEV-as-a-Judge: Accept When Confident, Escalate When Unsure", "authors": [ "Yubo Li", "Yidi Miao", "Ramayya Krishnan", "Rema Padman" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.26550", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "decision-only judge confidence" ], "cost_reduction_reported": "Within 3 points of a SOTA LLM judge at 0.36% of its fee; cascade retains 99% of comparator accuracy at lower cost", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A decision-only judge provides an economical first-pass evaluation and escalates uncertain cases to a stronger judge; compared against 16 judges with blinded human adjudication.", "key_idea": "Accept confident cheap verdicts, escalate uncertain ones.", "relevance_score": 5, "relevance_rationale": "Direct judge-cascade cost reduction.", "found_via": "hf-papers search: LLM judge cost efficient", "cites_or_cited_by": [] }, { "id": "li-2026-llmrouterbench", "title": "LLMRouterBench: A Massive Benchmark and Unified Framework for LLM Routing", "authors": [ "Hao Li", "Yiqun Zhang", "Zhaoyan Guo", "Chenxu Wang", "Shengji Tang", "Qiaosheng Zhang", "Yang Chen", "Biqing Qi", "Peng Ye", "Lei Bai", "Zhen Wang", "Shuyue Hu" ], "year": 2026, "venue_or_source": "ACL 2026", "source_type": "benchmark", "url": "https://arxiv.org/abs/2601.07206", "code_url": "https://github.com/ynulihao/LLMRouterBench", "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "precomputed outcomes", "cost", "latency", "query embedding" ], "cost_reduction_reported": null, "benchmarks": [ "LLMRouterBench (21 datasets)", "21 datasets (400K instances)" ], "models_studied": [ "33 models", "10 routing baselines incl. commercial routers" ], "production_evidence": false, "summary": "LLMRouterBench unifies 400K+ instances from 21 datasets and 33 models with 10 baselines. Under unified evaluation, many routers perform similarly and several recent and commercial routers fail to beat a simple baseline, while a large oracle gap remains due to model-recall failures.", "key_idea": "Large unified re-evaluation showing most routers are not reliably better than simple baselines.", "relevance_score": 5, "relevance_rationale": "Important sober benchmark for choosing a router.", "found_via": "snowball: cited-by ong-2024-routellm | web search: RouterEval benchmark routing LLMs", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "hu-2024-routerbench", "lee-2026-routing-gap-task-type", "ong-2024-routellm" ] }, { "id": "li-2026-progrouter", "title": "ProgRouter: Online Progress-Guided Orchestration for Multi-Agent LLM Workflows under Quality-Cost Tradeoffs", "authors": [ "Songyuan Li", "Ahmed M. Abdelmoniem", "Shiqiang Wang" ], "year": 2026, "venue_or_source": "EMNLP 2026 Findings", "source_type": "paper", "url": "https://arxiv.org/abs/2608.25992", "code_url": null, "topic_clusters": [ "C3", "C5" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "multi-view task progress score", "progress-gain prediction", "time and cost budgets" ], "cost_reduction_reported": "Reduces operating cost relative to key baselines while preserving quality (details truncated in abstract view)", "benchmarks": [ "HumanEval Plus", "MBPP", "MATH-500", "ASQA" ], "models_studied": [], "production_evidence": false, "summary": "ProgRouter routes each step of a multi-agent workflow online to a candidate LLM agent by predicting its progress gain from a multi-view progress scorer and meta-gating, balancing progress, time budget and long-run cost.", "key_idea": "Online per-step routing driven by predicted task progress.", "relevance_score": 5, "relevance_rationale": "Step-level routing in MAS workflows under budgets.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [] }, { "id": "li-2026-rsi-router", "title": "RSI-Router: Evolving Subtask-Level LLM Routing and Skills for Cost-Efficient Agents", "authors": [ "Hao Li", "Hangfan Zhang", "Zhiyao Cui", "Chunjiang Mu", "Yiqun Zhang", "Bo Zhang", "Danyang Jia", "Shuyue Hu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.34712", "code_url": null, "topic_clusters": [ "C3", "C4", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "mined subtask definitions", "trajectory comparisons" ], "cost_reduction_reported": "Surpasses DeepSeek-only baseline at about 48.3% of the cost across five agentic benchmarks; 74.7-82.2% cost cuts with better performance on ALFWorld, ScienceWorld, WebShop; +16.7% relative on Terminal-Bench 2.0 at 18.0% lower cost", "benchmarks": [ "ALFWorld", "ScienceWorld", "WebShop", "Terminal-Bench 2.0" ], "models_studied": [ "DeepSeek-V4.1-Flash", "Qwen3.5-9B" ], "production_evidence": false, "summary": "RSI-Router mines subtask definitions from trajectories, evolves subtask-to-model assignments, develops model-specific skills by comparing routed and large-only runs, and keeps a Pareto population of routers across iterations. Routing between a large and a 9B model halves cost while improving performance.", "key_idea": "Recursive self-improvement of subtask-level routing and skills.", "relevance_score": 5, "relevance_rationale": "Subtask-level large-small collaboration with strong savings.", "found_via": "web search: step-level routing", "cites_or_cited_by": [ "rajib-2026-gated-memory-routing", "son-2026-swe-router", "zhang-2026-budget-aware-agentic-routing" ] }, { "id": "li-2026-wiserouter", "title": "WISERouter: LLM Routing with Workload Budget Constraint", "authors": [ "Yifei Li", "Zihui Gao", "Laks V. S. Lakshmanan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.23765", "code_url": null, "topic_clusters": [ "C1", "C5", "C6" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "query embedding", "workload budget" ], "cost_reduction_reported": null, "benchmarks": [ "RouterBench", "SWE-Bench" ], "models_studied": [], "production_evidence": false, "summary": "WISERouter treats routing as a constrained contextual bandit with a workload-level (not per-query) budget, with an offline variant learning from historical interactions and an online variant with O(sqrt T) regret. Tested on RouterBench and SWE-Bench.", "key_idea": "Workload-level budgeted routing learned from logs or online exploration.", "relevance_score": 5, "relevance_rationale": "Bridges offline logs and online exploration under budget; includes agentic SWE-Bench.", "found_via": "OpenAlex title: LLM routing budget", "cites_or_cited_by": [ "panda-2025-pilot" ] }, { "id": "lin-2026-cursor-agent-swarm-economics", "title": "Agent swarms and the new model economics", "authors": [ "Wilson Lin" ], "year": 2026, "venue_or_source": "Cursor blog", "source_type": "blog", "url": "https://cursor.com/blog/agent-swarm-model-economics", "code_url": "https://github.com/cursor/minisqlite", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "role (planner vs worker)" ], "cost_reduction_reported": "Building SQLite in Rust from docs: model mixes reached similar quality but cost ranged from $1,339 (Opus 4.8 planner + Composer 2.5 workers) to $10,565 (GPT-5.5 for all roles); workers carried >=69% (usually >90%) of tokens; worker fleet cost $411 vs $9,373 (vendor-reported)", "benchmarks": [ "held-out SQL test suite (SQLite-from-scratch task)" ], "models_studied": [ "Claude Opus 4.8", "Composer 2.5", "GPT-5.5", "Claude Fable 5", "Grok 4.5" ], "production_evidence": false, "summary": "Cursor's planner/worker swarm recursively decomposes a goal, with frontier planners and cheaper, faster workers. On a long-running build-SQLite task, the redesigned swarm beat the old one in every model configuration, and mixing a frontier planner with a cheap worker matched single-frontier quality at a fraction of the cost because workers dominate tokens. A more expensive planner that used fewer tokens still raised total cost by inducing more worker tokens.", "key_idea": "Role-level model assignment (frontier planner, cheap workers) captures most savings in long-horizon multi-agent runs.", "relevance_score": 5, "relevance_rationale": "Direct evidence on subagent-level model assignment economics in a long-horizon multi-agent system.", "found_via": "direct: cursor.com/blog/topic/research", "cites_or_cited_by": [ "furman-2026-replit-free-the-models", "okeefe-2026-cursor-router" ] }, { "id": "liskowski-2026-compositional-online-learning", "title": "Compositional Online Learning for Semantic Data Processing Systems", "authors": [ "Paweł Liskowski", "Fuheng Zhao", "Benjamin Han", "Anupam Datta", "Dimitris Tsirogiannis" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.27244", "code_url": null, "topic_clusters": [ "C2", "C5", "C10", "C14" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "per-call outcomes", "filter selectivity", "cascade confidence" ], "cost_reduction_reported": "Learning components compose to an 11.4x upper bound on per-row LLM cost reduction for a representative conjunction-filter workload, reduced to a realistic ~8x after self-selection, sample-budget shrinkage and selectivity drift", "benchmarks": [ "Cortex AISQL production case study" ], "models_studied": [], "production_evidence": true, "summary": "Argues that LLM calls in semantic data processing are slow enough to hide online learner updates inside each round-trip, and builds compositional online learning: memoization, an online filter-ordering learner and an online per-batch cascade-routing learner. A production case study in Snowflake Cortex AISQL decomposes per-row LLM cost by component.", "key_idea": "Compose online learners (caching, ordering, cascade routing) at the LLM call boundary in production.", "relevance_score": 5, "relevance_rationale": "Production online cascade learning with explicit cost decomposition, including drift effects.", "found_via": "query: cascade threshold online adaptation", "cites_or_cited_by": [] }, { "id": "liu-2026-adaptive-vlm-routing-cua", "title": "Adaptive Vision-Language Model Routing for Computer Use Agents", "authors": [ "Xunzhuo Liu", "Bowei He", "Xue Liu", "Andy Luo", "Haichen Zhang", "Huamin Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.12823", "code_url": "https://github.com/vllm-project/semantic-router", "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "multimodal embedding difficulty", "small-VLM confidence probe", "retrieved memory" ], "cost_reduction_reported": "Projected inference cost reductions up to 78% while staying within 2 points of an all-large-model baseline", "benchmarks": [ "ScreenSpot-Pro", "OpenClaw agent routing benchmark" ], "models_studied": [], "production_evidence": false, "summary": "Adaptive VLM Routing inserts a routing layer between a computer-use agent orchestrator and a pool of VLMs: for each tool call it estimates difficulty, probes a small VLM's confidence and sends the action to the cheapest model meeting a reliability threshold, escalating risky actions. Part of the vLLM Semantic Router project.", "key_idea": "Per-action VLM routing with confidence probing in computer-use agents.", "relevance_score": 5, "relevance_rationale": "Per-step model routing in GUI agents.", "found_via": "arXiv title search: model routing agentic", "cites_or_cited_by": [ "chen-2026-workload-router-pool", "wei-2026-routing-least-learnable", "zhou-2026-agent-as-a-router" ] }, { "id": "liu-2026-adwm-ope-llm-agents", "title": "Autoregressive Diffusion World Models for Off-Policy Evaluation of LLM Agents", "authors": [ "Kai Liu", "Guo-Jun Xiong", "Weinan Zhang", "Sheng Tang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.05558", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "pre-collected trajectories", "policy-conditioned score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Estimates the value of a new LLM agent policy in multi-turn environments using only pre-collected trajectories, by learning a latent diffusion world model whose per-step denoising is guided by the evaluated agent. Alternating agent and world-model steps avoids the compounding errors of autoregressive world models.", "key_idea": "Off-policy evaluation of LLM agents with an agent-guided diffusion world model.", "relevance_score": 5, "relevance_rationale": "Direct OPE for agent policies from logs, the core method needed to vet agent configuration changes offline.", "found_via": "query: off-policy evaluation LLM routing", "cites_or_cited_by": [] }, { "id": "liu-2026-caster", "title": "CASTER: Breaking the Cost-Performance Barrier in Multi-Agent Orchestration via Context-Aware Strategy for Task Efficient Routing", "authors": [ "Shanyv Liu", "Xuyang Yuan", "Tao Chen", "Zijun Zhan", "Zhu Han", "Danyang Zheng", "Weishan Zhang", "Shaohua Cao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.19793", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "semantic embedding", "structural meta-features", "on-policy negative feedback" ], "cost_reduction_reported": "Up to 72.4% lower inference cost than strong-model baselines at matched success rates", "benchmarks": [ "software engineering", "data analysis", "scientific discovery", "cybersecurity tasks" ], "models_studied": [], "production_evidence": false, "summary": "CASTER is a lightweight router for graph-based MAS that estimates each node's task difficulty from semantic and structural features and picks a model, self-improving from its own routing failures. It cuts cost substantially versus using strong models everywhere.", "key_idea": "Per-node difficulty router for graph MAS, trained from its own failures.", "relevance_score": 5, "relevance_rationale": "Node-level model routing in MAS.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "zhang-2025-maas-agentic-supernet" ] }, { "id": "liu-2026-harness-native-routing", "title": "Agentic Routing: The Harness-Native Data Flywheel", "authors": [ "Xinchen Liu", "Hang Zhou", "Yingjie Zong", "Yuchuan Tian", "Liuyang Song", "Shuo Zhang", "Yulong Li", "Wei He", "Mengyu Zheng", "Runke Liu", "Siyang Cheng", "Xiang Kuang", "Hailin Hu", "Kai Han", "Yunhe Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.11399", "code_url": null, "topic_clusters": [ "C1", "C3", "C5", "C6", "C13", "C14" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "full harness state", "structured routing records", "execution trace", "outcome", "cost", "harness state", "execution state", "harness signals" ], "cost_reduction_reported": "LightGBM router preserves task quality while cutting realized cost ~90% on PinchBench and ~43% on DRACO vs fixed strong model", "benchmarks": [ "DRACO", "PinchBench" ], "models_studied": [ "frontier and open models" ], "production_evidence": false, "summary": "Proposes step-level agentic routing conditioned on harness state that picks one model or an ensemble per step, with each decision generating a data record that feeds a data flywheel.", "key_idea": "Harness-native step routing with data flywheel.", "relevance_score": 5, "relevance_rationale": "Maps to learning from agent logs.", "found_via": "snowball: cited-by ong-2024-routellm | web search: step-level routing | snowball: cited-by panda-2025-pilot | web search: agentic routing benchmark multi-turn", "cites_or_cited_by": [ "agarwal-2026-switchcraft", "chen-2026-hm-router", "deng-2026-cfrc-residual-completion-handoffs", "ding-2024-hybrid-llm", "ong-2024-routellm", "yang-2026-twinrouterbench", "yue-2025-masrouter" ] }, { "id": "liu-2026-vllm-sr-session-aware-agentic-routing", "title": "Session-Aware Agentic Routing: Continuity-Aware Model Selection for Long-Horizon LLM Agents", "authors": [ "Xunzhuo Liu", "Bowei He", "Huamin Chen", "Haichen Zhang", "Andy Luo", "vLLM Semantic Router Team" ], "year": 2026, "venue_or_source": "vLLM blog", "source_type": "blog", "url": "https://blog.vllm.ai/blog/2026-06-02-session-aware-agentic-routing", "code_url": "https://github.com/vllm-project/semantic-router", "topic_clusters": [ "C1", "C3", "C10", "C11", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "semantic signals", "session memory", "tool-loop state", "provider-state portability", "task drift", "prefix-cache warmth / switch cost" ], "cost_reduction_reported": "Across 21,600 deterministic turns: 79.29% fewer model switches, 3,836 unsafe switches eliminated, 78.71% estimated physical-model cost reduction vs single-turn routing (quality delta -0.0453); sticky sessions 98.65% cost reduction but -0.1433 quality. 2,896 live AMD ROCm requests with 0 continuity violations (project-reported)", "benchmarks": [ "deterministic replay of 21,600 agent turns", "live AMD ROCm requests" ], "models_studied": [], "production_evidence": false, "summary": "Introduces SAAR in vLLM Semantic Router: per-turn semantic routing augmented with router-owned session memory, hard locks during tool loops and around non-portable provider state, safe reset boundaries (drift, idle), and prefix-cache-aware pricing of model switches. Ablations show each component's contribution to switch reduction, unsafe-switch elimination and cost.", "key_idea": "Turn-level routing in agent sessions must price switches (lost cache, broken tool state), not just pick the best model per prompt.", "relevance_score": 5, "relevance_rationale": "Directly addresses step-level cost-aware routing inside long-horizon agents with ablations and cost numbers.", "found_via": "direct: blog.vllm.ai index", "cites_or_cited_by": [ "notdiamond-2026-interactive-benchmarks-routing", "vllm-2026-agentx-agentic-serving" ] }, { "id": "lugoloobi-2026-llms-encode-failures", "title": "LLMs Encode Their Failures: Predicting Success from Pre-Generation Activations", "authors": [ "William Lugoloobi", "Thomas Foster", "William Bankes", "Chris Russell" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.09924", "code_url": "https://github.com/KabakaWilliam/llms_know_difficulty", "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "pre-generation probe" ], "cost_reduction_reported": "Routing with probes exceeds the best single model while reducing inference cost by up to 70% on MATH", "benchmarks": [ "MATH", "E2H-AMC", "coding" ], "models_studied": [], "production_evidence": false, "summary": "Probes on pre-generation activations predict policy-specific success on math and code; models encode a model-specific difficulty distinct from human difficulty, and probe-based routing across a model pool beats the best model at up to 70% lower cost.", "key_idea": "Pre-generation probes drive cost-aware routing.", "relevance_score": 5, "relevance_rationale": "Direct cost reduction from a cheap internal signal.", "found_via": "WebSearch: probes hidden states LLM correctness prediction routing escalation", "cites_or_cited_by": [ "cencerrado-2025-no-answer-needed" ] }, { "id": "lyu-2026-differential-reasoning-router", "title": "The Differential Reasoning Router: Operationalizing Cost-Aware LLM Annotation in E-commerce", "authors": [ "Cheng Lyu", "Jingyue Zhang", "Vinny DeGenova", "Mengwei Li", "Yuanli Pei" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.30224", "code_url": null, "topic_clusters": [ "C1", "C2", "C9", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "per-sample and per-rule success probabilities of direct vs reasoning model" ], "cost_reduction_reported": "In a production e-commerce workflow, reaches accuracy parity with the strongest confidence-based router with more than 60% reasoning-token cost savings", "benchmarks": [ "production e-commerce annotation workflow" ], "models_studied": [], "production_evidence": true, "summary": "For cold-start LLM annotation, estimates separate success probabilities for a direct and a reasoning model and routes easy items directly, uses reasoning where it helps, and escalates likely double failures to humans.", "key_idea": "Route between direct model, reasoning model and humans.", "relevance_score": 5, "relevance_rationale": "Production evidence for reasoning vs non-reasoning routing with escalation.", "found_via": "arXiv title search: reasoning router", "cites_or_cited_by": [] }, { "id": "notdiamond-2026-interactive-benchmarks-routing", "title": "Interactive benchmarks: a new methodology for evaluating model routing", "authors": [ "Not Diamond" ], "year": 2026, "venue_or_source": "Not Diamond blog", "source_type": "blog", "url": "https://www.notdiamond.ai/blog/interactive-benchmarks-a-new-methodology-for-evaluating-model-routing", "code_url": null, "topic_clusters": [ "C1", "C3", "C10", "C13", "C14" ], "routing_granularity": "step", "learning_regime": null, "signals_used": [ "session state", "message and token counts", "task complexity", "KV-cache state", "intermediate and final rewards", "reasoning effort" ], "cost_reduction_reported": "Approximates or exceeds Opus 5 xhigh quality at 20-80% cost reduction; SWE-Interact 50.7% at $4.28/attempt, SWE-Together 67.6% at $4.16, Dialogue SWE-Bench-Pro 57.6% at $3.00; adding GLM 5.2 and DeepSeek V4 Flash raised accuracy 3.6% and savings from 39% to 66% vs Opus xhigh. Companion 'Not Diamond Code' post: 39% (Poly-SWE-Bench Verified) and 61% (LongCodeQA) cost reduction vs Opus 4.8 xhigh, >20% savings in enterprise early access (all vendor-reported)", "benchmarks": [ "SWE-Interact", "SWE-Together", "Dialogue SWE-Bench-Pro", "Poly-SWE-Bench Verified", "LongCodeQA" ], "models_studied": [ "Claude Opus 5", "Claude Sonnet 5", "Claude Haiku 4.5", "GLM 5.2", "DeepSeek V4 Flash" ], "production_evidence": false, "summary": "Argues that single-turn routing breaks the KV cache and first-turn routing is brittle to mid-session shifts, and describes Not Diamond Code, which predicts future reward and cost for each (model, reasoning effort) pair at every agent step in a cache-aware way. Because static benchmarks miss long sessions and cache expiry (a 10-20x input-cost jump), it evaluates on interactive benchmarks with simulated users built on the Harbor framework and releases a benchmark repo so clients can reproduce comparisons.", "key_idea": "Evaluate agent routers on interactive, simulated-user sessions where cache dynamics and session length drive cost.", "relevance_score": 5, "relevance_rationale": "Step-level (model, effort) routing inside coding agents plus an evaluation methodology tailored to agent cost.", "found_via": "direct: notdiamond.ai/blog index", "cites_or_cited_by": [ "fireworks-2026-firerouter-with-opus", "liu-2026-vllm-sr-session-aware-agentic-routing" ] }, { "id": "okeefe-2026-cursor-router", "title": "How Cursor Router chooses the right model for the task", "authors": [ "Connor O'Keefe", "Yuri Volkov" ], "year": 2026, "venue_or_source": "Cursor blog", "source_type": "blog", "url": "https://cursor.com/blog/how-cursor-router-works", "code_url": null, "topic_clusters": [ "C1", "C3", "C5", "C6", "C7", "C14" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "task category", "recent tool calls", "conversation state", "Compass complexity score (predicted user satisfaction)", "learned task/domain/modifier taxonomy", "implicit user feedback (moves on vs corrects)", "per-turn cost incl. cache misses" ], "cost_reduction_reported": "Auto Intelligence: above Fable-level user satisfaction at 68% lower cost; Auto Balance: outperforms Opus 4.8 at 41% lower cost with +3% satisfaction (relative to frontier models, measured on live traffic; vendor-reported). Compass top-rated turns got positive signal 96% of the time vs 71% for lowest-rated", "benchmarks": [ "live Cursor traffic", "held-out offline split of production turns" ], "models_studied": [ "Grok", "GPT-5.6 Sol", "Claude Opus 4.8", "Claude Opus 5", "Claude Fable" ], "production_evidence": true, "summary": "Cursor's per-turn router is trained on hundreds of thousands of production agent turns labelled with implicit satisfaction (user moves on vs corrects the agent) and actual cost including switch-induced cache misses. A complexity predictor (Compass) thresholds turns to a cheap model or to a taxonomy router, which admits a frontier model for a task label only if it beats the cheap model with 75% one-sided confidence, then chooses a traffic-weighted mix within a per-mode cost budget. Policies are tuned by cross-validation, checked on held-out logs, and validated on live traffic before launch.", "key_idea": "Learn step-level routing from implicit feedback in production logs, with offline-to-online validation and budgeted confidence-gated upgrades.", "relevance_score": 5, "relevance_rationale": "Production step-level routing learned from customer interaction logs with offline and online evaluation: close to the premise.", "found_via": "direct: cursor.com/blog index", "cites_or_cited_by": [ "furman-2026-replit-free-the-models", "katz-2026-cursor-token-efficiency", "lin-2026-cursor-agent-swarm-economics" ] }, { "id": "paul-2026-agentrouter-step-tiers", "title": "AgentRouter: Heterogeneous Model Routing for Cost-Optimal Multi-Step Agentic Workflows", "authors": [ "Rudrendu Kumar Paul", "Sourav Nandy" ], "year": 2026, "venue_or_source": "ICML 2026 Workshop (AgenticUQ)", "source_type": "paper", "url": "https://arxiv.org/abs/2609.22951", "code_url": null, "topic_clusters": [ "C3", "C4", "C6", "C14" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "five routing-time step features", "five step-level features", "annotated trajectory steps" ], "cost_reduction_reported": "72% cost reduction vs frontier-only while retaining 97.3% of frontier quality; RouteLLM and FrugalGPT applied per-step achieve 31% and 44%", "benchmarks": [ "planning, coding, research and data-analysis agent trajectories (50,000 annotated steps)", "50,000 annotated agent trajectory steps" ], "models_studied": [ "four model tiers" ], "production_evidence": false, "summary": "AgentRouter is a 12M-parameter classifier (<5ms per step) that maps each trajectory step to one of four model tiers from five features, trained on 50,000 annotated steps. It cuts cost 72% versus frontier-only with under 3% quality loss and beats per-step RouteLLM and FrugalGPT.", "key_idea": "Lightweight per-step tier classifier for agent trajectories.", "relevance_score": 5, "relevance_rationale": "Direct step-level routing with strong reported savings.", "found_via": "web search: step-level routing | snowball: cited-by chen-2025-llmselector | snowball: cited-by panda-2025-pilot", "cites_or_cited_by": [ "jin-2025-corl-centralized-budget", "ong-2024-routellm", "su-2025-daao", "zhang-2026-budget-aware-agentic-routing" ] }, { "id": "piatrashyn-2026-redact", "title": "ReDAct: Uncertainty-Aware Deferral for LLM Agents", "authors": [ "Dzianis Piatrashyn", "Nikita Kotelevskii", "Kirill Grishchenkov", "Nikita Glazkov", "Ivan Nasonov", "Ilya Makarov", "Timothy Baldwin", "Preslav Nakov", "Roman Vashurin", "Maxim Panov" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.07036", "code_url": null, "topic_clusters": [ "C2", "C3", "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "small-model predictive uncertainty", "calibrated threshold", "token-level predictive uncertainty" ], "cost_reduction_reported": "Deferring about 15% of decisions to the large model matches exclusive large-model quality at significantly lower cost; search summary reports 64% savings vs GPT-5.2 alone", "benchmarks": [ "ALFWorld", "MiniGrid" ], "models_studied": [ "small cheap LLM", "large LLM (e.g., GPT-5.2)" ], "production_evidence": false, "summary": "An agent runs a small model by default and defers individual decisions to a large model when the small model's uncertainty exceeds a calibrated threshold. Evaluated in text-based embodied environments.", "key_idea": "Per-step uncertainty-based deferral inside an agent loop.", "relevance_score": 5, "relevance_rationale": "Step-level cascade for agents: exactly lever (b).", "found_via": "arXiv title search: deferral LLM | web search: uncertainty-triggered escalation", "cites_or_cited_by": [ "lei-2026-tacit-switch" ] }, { "id": "qian-2026-dyserve", "title": "Serving Agentic Workflows with a Physical-Plan Compiler and Adaptive Runtime", "authors": [ "Jiayi Qian", "Yichong Zhang", "Hanchen Yang", "Chun Tao", "Souvik Kundu", "Zishen Wan", "Tushar Krishna" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.02942", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "load pressure", "workflow variants" ], "cost_reduction_reported": "accuracy improved 3-9 points with 1.1-6.8x mean-latency speedups over the highest-accuracy baseline; on a burst trace, correct on-time completions rose from 18.1% to 67.2%", "benchmarks": [ "four agentic workloads" ], "models_studied": [], "production_evidence": false, "summary": "Compiles agentic workflows into physical plans (model/variant assignments) and an adaptive runtime switches variants under load without re-running the optimizer.", "key_idea": "Physical-plan compilation plus runtime variant switching for agent workflows.", "relevance_score": 5, "relevance_rationale": "Load-adaptive per-step model/variant assignment in agentic serving.", "found_via": "Aragog just-in-time model routing agentic workflows serving", "cites_or_cited_by": [ "dai-2025-aragog" ] }, { "id": "rajib-2026-gated-memory-routing", "title": "Learning What to Retain: Gated-Memory Routing for Efficient Collaboration in Multi-Agent LLM Systems", "authors": [ "Rakibul Hasan Rajib", "Mengxin Zheng", "Qian Lou" ], "year": 2026, "venue_or_source": "EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2609.00237", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "learned execution memory", "write/retrieval gates", "halting controller" ], "cost_reduction_reported": "+2.44 points average accuracy over strongest baseline with 31.9% lower HumanEval inference cost", "benchmarks": [ "five reasoning and code-generation benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Gated-Memory Routing chooses the next role and backbone at each step from the query plus a compact learned memory, with gates that write only non-redundant steps and retrieve relevant subsets, and a halting controller that stops once evidence suffices.", "key_idea": "Per-step role/backbone routing on a compact learned memory with adaptive halting.", "relevance_score": 5, "relevance_rationale": "Step-level MAS routing with context efficiency.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "li-2026-rsi-router" ] }, { "id": "ray-2026-adaptevolve", "title": "AdaptEvolve: Improving Efficiency of Evolutionary AI Agents through Adaptive Model Selection", "authors": [ "Pretam Ray", "Pratik Prabhanjan Brahma", "Zicheng Liu", "Emad Barsoum" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.11931", "code_url": "https://github.com/raypretam/adaptive_llm_selection", "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "intrinsic generation confidence" ], "cost_reduction_reported": "Average 37.9% lower total inference cost while retaining 97.5% of upper-bound accuracy of static large-model baselines", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AdaptEvolve picks, at each generation step of an evolutionary refinement agent, which model in a pool is capable enough. It uses the model's own generation confidence as a real-time solvability estimate, acting as a confidence-driven cascade.", "key_idea": "Confidence-driven per-step model selection inside evolutionary agent loops.", "relevance_score": 5, "relevance_rationale": "Step-level model selection in an agent loop with explicit cost-accuracy trade-off.", "found_via": "arXiv query ti:\"model selection\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "ruan-2026-early-abort-agent-episodes", "title": "Doomed from the Start: Early Abort of LLM Agent Episodes via a Recall-Controlled Probe Cascade", "authors": [ "Kai Ruan", "Zihe Huang", "Ziqi Zhou", "Qianshan Wei", "Jinghao Lin", "Xuan Wang", "Hao Sun" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.06503", "code_url": null, "topic_clusters": [ "C2", "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state failure probes", "calibrated per-round thresholds" ], "cost_reduction_reported": "Saves 1.5-8.8x more compute than best single-gate baseline at 90% recall; up to 60.2% (TextCraft) and 54.9% (WebShop) fewer generated tokens at 90% recall", "benchmarks": [ "TextCraft", "WebShop" ], "models_studied": [ "Qwen-2.5-7B", "Llama-3.2-3B", "Qwen3-1.7B" ], "production_evidence": false, "summary": "Linear probes on activations predict agent task failure from the first round; a recall-controlled cascade of per-round gates aborts doomed episodes with a certified recall guarantee.", "key_idea": "Abort doomed agent trajectories early with guaranteed recall.", "relevance_score": 5, "relevance_rationale": "Direct compute saving inside long-horizon agents.", "found_via": "arXiv title search: cascade LLM", "cites_or_cited_by": [] }, { "id": "sedghani-2026-agentic-budget-constraints", "title": "Optimizing Agentic AI Applications Under Budget Constraints", "authors": [ "Hamta Sedghani", "Federica Filippini", "Z. Seyedi", "Blerina Spahiu", "Michele Ciavotta", "D. Ardagna" ], "year": 2026, "venue_or_source": "ICPE 2026 Companion", "source_type": "paper", "url": "https://doi.org/10.1145/3777911.3800631", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "per-config accuracy/latency/energy profiles" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formulates configuring a multi-step agentic pipeline as choosing one (agent, inference method, parameter) triple per task from a heterogeneous pool of LLMs and SLMs. The objective maximizes end-to-end quality under global time and energy budgets and is solved as a MILP plus complementary methods, accounting for error propagation across steps.", "key_idea": "MILP-based per-step model/config selection under global budgets.", "relevance_score": 5, "relevance_rationale": "Per-step heterogeneous model assignment under explicit budgets (sub-themes b and e).", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [] }, { "id": "singh-2026-planner-as-router", "title": "Planner-as-Router: Joint Plan-Time Model Routing for Cost-Efficient Multi-Agent Workflows", "authors": [ "Vivek Kumar Singh", "Preeti Priyam", "Gautam Bhowmick" ], "year": 2026, "venue_or_source": "AIxSET 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2609.32917", "code_url": null, "topic_clusters": [ "C3", "C14" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "plan structure", "subtask dependencies" ], "cost_reduction_reported": "Cuts cost 44% vs all-frontier routing while giving up 2.9 accuracy points; matches frontier-on-sink heuristic and beats FrugalGPT cascade on cost", "benchmarks": [ "EntBench (54 enterprise tasks with live SQL/MongoDB grading)" ], "models_studied": [], "production_evidence": false, "summary": "Planner-as-Router has the planner assign a model tier (small, mid, frontier) to each subtask as it decomposes the query, so the whole workflow's dependencies inform tier choice without a separate router or training data. On an enterprise benchmark graded against live databases it stays on the cost-accuracy frontier.", "key_idea": "Let the planner pick each subtask's model tier at plan time.", "relevance_score": 5, "relevance_rationale": "Training-free subtask-level model assignment in enterprise workflows.", "found_via": "web search: planner executor", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "son-2026-swe-router", "title": "SWE-Router: Routing in Multi-turn Agentic Software Engineering Tasks", "authors": [ "Seongho Son", "Sangwoong Yoon", "Jiahua Tang", "Shuhan Wang", "Lorenz Wolf", "Ilija Bogunovic" ], "year": 2026, "venue_or_source": "ICML 2026 Workshop (DL4Code)", "source_type": "paper", "url": "https://arxiv.org/abs/2607.00053", "code_url": null, "topic_clusters": [ "C1", "C2", "C3", "C6" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "partial trajectory of cheap model", "value-based estimate", "value estimate" ], "cost_reduction_reported": "Greatly improves cost efficiency while maintaining most of the strong model's performance (no single number in abstract)", "benchmarks": [ "SWE-bench", "SWE tasks" ], "models_studied": [ "weak/strong LLM pairs across the cost-capability frontier" ], "production_evidence": false, "summary": "SWE-Router lets a cheap model run a few exploratory turns on a software engineering task, then reads the partial trajectory to decide whether to continue cheaply or escalate. A Bayes-optimality theorem shows conditioning on the trajectory never hurts.", "key_idea": "Route agentic tasks after a short cheap exploration.", "relevance_score": 5, "relevance_rationale": "Directly about cost-aware routing in long-horizon agents.", "found_via": "snowball: cited-by ong-2024-routellm | OpenAlex search: trajectory routing LLM agents cost", "cites_or_cited_by": [ "bhola-2026-scrouting-superscout", "chen-2026-hm-router", "deng-2026-cfrc-residual-completion-handoffs", "ganz-2026-handoff-tax", "lei-2026-tacit-switch", "li-2026-rsi-router", "ong-2024-routellm", "raj-2026-trace-router" ] }, { "id": "sridhar-2026-correlated-bandits-routing", "title": "Correlation-Aware Contextual Bandits with Surrogate Rewards for LLM Routing", "authors": [ "Ajay Narayanan Sridhar", "Ronak Singh", "Mehrdad Mahdavi", "Vijaykrishnan Narayanan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.09015", "code_url": null, "topic_clusters": [ "C1", "C5", "C7", "C15" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "surrogate reward model", "inter-arm correlation", "bandit feedback" ], "cost_reduction_reported": null, "benchmarks": [ "LLM routing benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Contextual bandit algorithms for routing that exploit correlated arms and noisy surrogate rewards (e.g. judge predictions), with regret guarantees robust to surrogate misspecification.", "key_idea": "Bandits with surrogate (judge) rewards for routing.", "relevance_score": 5, "relevance_rationale": "Online routing with proxy quality signals.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "ong-2024-routellm", "wei-2025-barp" ] }, { "id": "taberner-miller-2026-paretobandit", "title": "ParetoBandit: Budget-Paced Adaptive Routing for Non-Stationary LLM Serving", "authors": [ "Annette Taberner-Miller" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.00136", "code_url": "https://github.com/ParetoBandit/ParetoBandit", "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "contextual bandit", "primal-dual budget pacer", "geometric forgetting", "prompt embedding", "cost", "quality reward" ], "cost_reduction_reported": "budget compliance within 0.4%; up to +0.071 quality lift under price/quality shifts (1,824 prompts, three-model portfolio)", "benchmarks": [ "1,824 benchmark prompts" ], "models_studied": [ "three-model portfolio" ], "production_evidence": false, "summary": "ParetoBandit is an open-source cost-aware contextual-bandit router with an online primal-dual pacer enforcing a dollar cost ceiling and forgetting to track price/quality drift, plus hot-swappable models.", "key_idea": "Budget-paced non-stationary bandit routing.", "relevance_score": 5, "relevance_rationale": "Addresses drift and budgets in live traffic.", "found_via": "snowball: cited-by ong-2024-routellm | OpenAlex title: routing non-stationary LLM", "cites_or_cited_by": [ "chen-2023-chatgpt-behavior-over-time", "chu-2026-lqm-contextroute", "ding-2024-hybrid-llm", "lee-2026-drift-aware-routing", "liu-2022-nonstationary-bwk", "ma-2023-prompt-getting-worse", "ong-2024-routellm", "panda-2025-pilot", "wu-2026-assign-tasks-capacity" ] }, { "id": "tran-2026-single-vs-multi-agent-equal-budget", "title": "Single-Agent LLMs Outperform Multi-Agent Systems on Multi-Hop Reasoning Under Equal Thinking Token Budgets", "authors": [ "Dat Tran", "Douwe Kiela" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.02460", "code_url": null, "topic_clusters": [ "C9", "C12", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "thinking-token budget", "reasoning-token budget" ], "cost_reduction_reported": null, "benchmarks": [ "multi-hop reasoning tasks", "multi-hop reasoning benchmarks" ], "models_studied": [ "Qwen3", "DeepSeek-R1-Distill-Llama", "Gemini 2.5" ], "production_evidence": false, "summary": "Information-theoretic and empirical argument that under equal reasoning-token budgets single agents match or beat multi-agent systems on multi-hop reasoning; flags API budget-control artefacts (notably Gemini 2.5).", "key_idea": "Many multi-agent gains are unaccounted compute.", "relevance_score": 5, "relevance_rationale": "Budget-matched evaluation of when multi-agent is worth it.", "found_via": "arXiv title search: token budget reasoning | web search: Anthropic multi-agent tokens", "cites_or_cited_by": [ "gao-2025-single-or-multi-agent-why-not-both", "ke-2026-mas-orchestra", "kim-2025-science-of-scaling-agent-systems", "leins-2026-when-orchestration-pays-off", "yu-2026-when-mas-help-information-bottleneck", "zeng-2026-mixture-of-roles-single-agent" ] }, { "id": "wang-2026-boundaryrouter", "title": "Learning Agent Routing From Early Experience", "authors": [ "Yimin Wang", "Jiahao Qiu", "Xuan Qi", "Xinzhe Juan", "Jingzhe Shi", "Zelin Zhao", "Hongru Wang", "Shilong Liu", "Mengdi Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07180", "code_url": null, "topic_clusters": [ "C1", "C2", "C3", "C6", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "experience memory of seed runs", "rubric-guided reasoning", "early experience memory" ], "cost_reduction_reported": "reduces inference time by 60.6% vs agent while improving performance by 28.6% over direct LLM inference", "benchmarks": [ "RouteBench", "RouteBench (new)" ], "models_studied": [], "production_evidence": false, "summary": "BoundaryRouter decides per query whether direct LLM inference suffices or a full agent is needed, using a memory built by running both on a seed set and rubric-guided reasoning over retrieved cases. It introduces RouteBench with in-domain, paraphrased and out-of-domain splits.", "key_idea": "Route between plain LLM and full agent using early experience.", "relevance_score": 5, "relevance_rationale": "Routing between architectures (LLM vs agent) for cost.", "found_via": "web search: routing among agents | snowball: cited-by panda-2025-pilot", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "wang-2026-dropbox-dash-dspy", "title": "How we optimized Dash's relevance judge with DSPy", "authors": [ "Eric Wang", "Dmitriy Meyerzon" ], "year": 2026, "venue_or_source": "Dropbox Tech blog", "source_type": "blog", "url": "https://dropbox.tech/machine-learning/optimizing-dropbox-dash-relevance-judge-with-dspy", "code_url": null, "topic_clusters": [ "C4", "C7", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "human-annotated relevance labels", "NMSE" ], "cost_reduction_reported": "Adapting the production relevance judge from o3 to gpt-oss-120b with GEPA cut NMSE 45% (8.83 to 4.86); enabled 10-100x more labeling at the same cost; on gemma-3-12b (MIPROv2) NMSE 46.88 to 17.26 and malformed JSON down >97%", "benchmarks": [ "Dropbox Dash relevance judgments (internal)" ], "models_studied": [ "OpenAI o3", "gpt-oss-120b", "gemma-3-12b" ], "production_evidence": true, "summary": "Dropbox used DSPy optimizers (GEPA, MIPROv2) to migrate an LLM relevance judge used across Dash production pipelines from an expensive model to cheaper open models, measured against human labels. Model adaptation time fell from 1-2 weeks to 1-2 days.", "key_idea": "Treat model swaps as re-optimization problems so prompts transfer to cheaper models.", "relevance_score": 5, "relevance_rationale": "Production case study of cost reduction by downscaling a pipeline component with automated prompt optimization.", "found_via": "search: dropbox.tech DSPy relevance judge Dash prompt optimization", "cites_or_cited_by": [ "agrawal-2025-gepa", "opsahlong-2024-mipro" ] }, { "id": "wang-2026-oi-mas", "title": "Orchestrating Intelligence: Confidence-Aware Routing for Efficient Multi-Agent Collaboration across Multi-Scale Models", "authors": [ "Jingbo Wang", "Sendong Zhao", "Jiatong Liu", "Haochun Wang", "Wanting Li", "Bing Qin", "Ting Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.04861", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "state", "confidence", "task complexity" ], "cost_reduction_reported": "Up to 12.88% higher accuracy and up to 79.78% lower cost vs baseline MAS", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "OI-MAS routes, at each reasoning stage, to an agent role and a model scale from a heterogeneous pool of multi-scale LLMs, using a confidence-aware mechanism to avoid unnecessary use of large models. It raises accuracy and cuts cost versus uniform-model MAS.", "key_idea": "State-dependent routing of roles and model sizes with confidence gating.", "relevance_score": 5, "relevance_rationale": "Direct step/role-level model-scale routing in MAS.", "found_via": "arXiv title search: model routing agentic", "cites_or_cited_by": [ "jiang-2026-cera-moa", "liu-2026-treecredit", "ye-2025-x-mas", "zhang-2025-maas-agentic-supernet" ] }, { "id": "wei-2026-step-level-cua-cascade", "title": "Step-level Optimization for Efficient Computer-use Agents", "authors": [ "Jinbiao Wei", "Kangqi Ni", "Yilun Zhao", "Guo Gan", "Arman Cohan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.27151", "code_url": null, "topic_clusters": [ "C2", "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "Stuck Monitor (progress stall)", "Milestone Monitor (semantic drift checkpoints)" ], "cost_reduction_reported": null, "benchmarks": [ "computer-use benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Observes that computer-use trajectories are heterogeneous, with errors concentrated at a few high-risk moments, and proposes an event-driven step-level cascade that runs a small policy by default and escalates when learned monitors detect stalls or semantic drift. The design layers on existing agents without retraining the large model.", "key_idea": "Learned stuck and milestone monitors trigger per-step escalation in GUI agents.", "relevance_score": 5, "relevance_rationale": "Direct step-level cascading in long-horizon agents.", "found_via": "web search: step-level routing", "cites_or_cited_by": [ "deng-2026-cfrc-residual-completion-handoffs", "gonuguntla-2026-replay-gap", "yue-2025-masrouter" ] }, { "id": "woisetschlager-2026-slarouter", "title": "Cost-Optimal LLM Routing with Limited User Feedback under User Satisfaction Guarantees", "authors": [ "Herbert Woisetschläger", "Arastun Mammadli", "Ryan Zhang", "Shiqiang Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.19376", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "sparse one-sided user feedback", "cost" ], "cost_reduction_reported": "reduces operating cost by up to 2.2x over existing baselines while meeting SLA constraints", "benchmarks": [ "wide range of LLM benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "SLARouter learns a cost-optimal routing policy online from the sparse, one-sided feedback available in production (e.g. complaints), with guarantees on cost optimality and SLA compliance, without per-benchmark tuning.", "key_idea": "Online routing from sparse production feedback with SLA guarantees.", "relevance_score": 5, "relevance_rationale": "Matches the production reality of limited feedback; cost-focused.", "found_via": "OpenAlex title: user feedback LLM routing", "cites_or_cited_by": [ "he-2026-market-aware-routing", "wei-2025-barp" ] }, { "id": "wu-2026-rlm-cascade", "title": "RLM-Cascade: Response-Level Speculative Decoding for Cost-Efficient LLM API Serving", "authors": [ "Haifeng Wu", "Srinivasan Manoharan", "Fangbo Tu", "Junhua Zhao", "Jian Wan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.22840", "code_url": null, "topic_clusters": [ "C2", "C3", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "rule-based complexity router", "verify model accept/enhance" ], "cost_reduction_reported": "On a Claude Code agentic workload: 88.8% draft-use over 125 production requests, 45.8% lower API cost vs direct Opus, p50 latency 2,026 ms vs 3,698 ms; 100% vs 95% pass on a 20-task benchmark", "benchmarks": [ "125 production Claude Code requests", "20-task Code/Math/Instruct set" ], "models_studied": [ "DeepSeek", "Claude Opus" ], "production_evidence": true, "summary": "A proxy layer where a cheap draft model writes each response and a strong model accepts, enhances or is skipped depending on a complexity router; schema-critical tool-selection turns bypass the pipeline. Deployed as an enterprise component on coding-agent traffic (small sample).", "key_idea": "Response-level draft-and-verify proxy for agent turns.", "relevance_score": 5, "relevance_rationale": "Turn-level cascading in a real agentic coding workload; small evidence base.", "found_via": "arXiv title search: cascade LLM", "cites_or_cited_by": [] }, { "id": "xiang-2026-otrope", "title": "OTROPE: Optimal Transport-based Robust Off-policy Evaluation for Large Language Models", "authors": [ "Liner Xiang", "Wenbo Zhang", "Hengrui Cai" ], "year": 2026, "venue_or_source": "NeurIPS 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2609.36264", "code_url": "https://github.com/LinerXiang/OTROPE", "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "human labels from behavior model", "proxy evaluator predictions", "semantic embeddings" ], "cost_reduction_reported": null, "benchmarks": [ "synthetic and real LLM evaluation tasks" ], "models_studied": [], "production_evidence": false, "summary": "OTROPE evaluates a new target LLM using scarce human labels collected on a behavior LLM, aligning samples in semantic space via optimal transport and combining corrected residuals with proxy predictors in a doubly-robust style, without needing response likelihoods.", "key_idea": "Optimal-transport doubly-robust OPE for black-box LLMs.", "relevance_score": 5, "relevance_rationale": "OPE for swapping models when likelihoods are unavailable (API models).", "found_via": "snowball: cited-by bhargava-2024-ope-logged-human-feedback", "cites_or_cited_by": [ "angelopoulos-2023-ppi", "angelopoulos-2023-ppi-plus-plus", "bhargava-2024-ope-logged-human-feedback", "boyeau-2024-autoeval-done-right", "dudik-2011-doubly-robust", "fisch-2024-stratified-ppi" ] }, { "id": "xu-2026-seqroute", "title": "SeqRoute: Global Budget-Aware Sequential LLM Routing via Offline Reinforcement Learning", "authors": [ "Zhongling Xu", "Shunan Zheng", "Wei Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.25424", "code_url": null, "topic_clusters": [ "C1", "C3", "C5", "C6", "C15" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "remaining session budget", "Conservative Q-Learning", "hindsight budget relabeling", "session history", "remaining budget" ], "cost_reduction_reported": "Reduces operational costs by 6.0-73.5% while maintaining or improving quality; bankruptcy under 1% (abstract)", "benchmarks": [ "10,000 raw sessions expanded to 2.38M transitions" ], "models_studied": [], "production_evidence": false, "summary": "SeqRoute formulates multi-turn session routing under a global budget as a finite-horizon MDP solved with offline RL (CQL), using hindsight budget relabeling to expand 10,000 sessions into 2.38M transitions.", "key_idea": "MDP/offline-RL routing across a session with a global budget.", "relevance_score": 5, "relevance_rationale": "MDP formulation over sequential calls; transferable to agent trajectories.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: cited-by yue-2025-masrouter | OpenAlex title: offline reinforcement learning routing LLM", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm", "zhang-2024-treacle", "zhang-2026-budget-aware-agentic-routing" ] }, { "id": "yang-2026-ares-reasoning-effort", "title": "Ares: Adaptive Reasoning Effort Selection for Efficient LLM Agents", "authors": [ "Jingbo Yang", "Bairu Hou", "Wei Wei", "Yujia Bao", "Shiyu Chang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.07915", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "interaction history", "minimum-sufficient effort labels" ], "cost_reduction_reported": "Reduces reasoning token usage by up to 52.7% vs fixed high-effort reasoning with minimal degradation in task success (TAU-Bench, BrowseComp-Plus, WebArena)", "benchmarks": [ "TAU-Bench", "BrowseComp-Plus", "WebArena" ], "models_studied": [], "production_evidence": false, "summary": "A lightweight router predicts the lowest reasoning-effort level (low/medium/high) sufficient for each agent step, trained on labels of the minimum effort that still succeeds.", "key_idea": "Per-step reasoning-effort routing for agents.", "relevance_score": 5, "relevance_rationale": "Exactly step-level effort downscaling in agents.", "found_via": "arXiv title search: reasoning effort", "cites_or_cited_by": [] }, { "id": "yang-2026-harness-adaptation-slm", "title": "Better Harnesses, Smaller Models: Building 90% Cheaper Agents via Automated Harness Adaptation", "authors": [ "Chenyang Yang", "Xinran Zhao", "Tongshuang Wu", "Christian Kästner" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.08938", "code_url": null, "topic_clusters": [ "C3", "C4", "C8", "C10", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "failure trajectories", "automated harness adaptation" ], "cost_reduction_reported": "best small-model agent recovered 89.7% of LLM performance at 4% of the cost; optimized harnesses improved 16 of 21 task-SLM pairs, closing the gap in seven", "benchmarks": [ "seven business-oriented agentic tasks", "routine business tasks (21 task-SLM pairs)" ], "models_studied": [ "three SLM families" ], "production_evidence": false, "summary": "Shows that SLM agents can match frontier agents on many routine business tasks at about 90% lower cost if the harness (instructions, tools, orchestration loops) is adapted, and builds a meta-agent that discovers adaptations from failure trajectories. Works best on repetitive workflows and capable-enough SLMs.", "key_idea": "Lift task difficulty into the harness so small models suffice.", "relevance_score": 5, "relevance_rationale": "Downscaling agents via automated harness adaptation.", "found_via": "web search: SLM tool calling | snowball: cited-by kang-2025-acon", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai", "kim-2025-science-of-scaling-agent-systems", "zhang-2025-maas-agentic-supernet" ] }, { "id": "yang-2026-twinrouterbench", "title": "TwinRouterBench: Fast Static and Live Dynamic Evaluation for Realistic Agentic LLM Routing", "authors": [ "Pei Yang", "Wanyi Chen", "Tongyun Yang", "Pengbin Feng", "Jiarong Xing", "Wentao Guo", "Yuhang Yao", "Yuhang Han", "Hanchen Li", "Xu Wang", "Zeyu Wang", "Jie Xiao", "Anjie Yang", "Liang Tian", "Lynn Ai", "Eric Yang", "Tianyu Shi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "benchmark", "url": "https://arxiv.org/abs/2605.18859", "code_url": "https://github.com/CommonstackAI/TwinRouterBench", "topic_clusters": [ "C1", "C3", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "router-visible prefix", "execution-verified tier labels", "execution-verified tier", "token cost" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-bench", "BFCL", "mtRAG", "QMSum", "PinchBench", "SWE-bench Verified", "SWE-bench Verified (dynamic)" ], "models_studied": [ "locked model pool" ], "production_evidence": false, "summary": "TwinRouterBench evaluates step-level routing in agents with a static track of 970 router-visible prefixes with verified target tiers and a dynamic track that runs routers inside a SWE-bench harness, avoiding online judges.", "key_idea": "Step-level agentic routing benchmark.", "relevance_score": 5, "relevance_rationale": "Benchmark for per-call routing in agents.", "found_via": "snowball: cited-by ong-2024-routellm | web search: step-level routing | web search: agentic routing benchmark multi-turn", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "gonuguntla-2026-replay-gap", "hu-2024-routerbench", "liu-2026-harness-native-routing", "ong-2024-routellm" ] }, { "id": "yao-2026-mera", "title": "MERA: Model Evolution and Routing with Skill Adaptation for Agentic Systems at Scale", "authors": [ "Yuhang Yao", "Zeyu Wang", "Wanyi Chen", "Tongyun Yang", "Yuhang Han", "Jie Xiao", "Chengke Bao", "Tianyi Zhao", "Lynn Ai", "Eric Yang", "Tianyu Shi" ], "year": 2026, "venue_or_source": "arXiv (preliminary version in CAIS RL-Eval)", "source_type": "paper", "url": "https://arxiv.org/abs/2608.10333", "code_url": null, "topic_clusters": [ "C3", "C6", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "replayed failed invocations", "verifier-backed fallback", "cost-calibrated router" ], "cost_reduction_reported": "Deployed policy retains 88.3% pass at 60.8% of always-teacher cost on HumanEval+MBPP; Qwen2.5-Coder-1.5B improves 28.7% to 49.7% after four cycles", "benchmarks": [ "HumanEval+", "MBPP", "TAU-2" ], "models_studied": [ "Qwen2.5-Coder-1.5B", "Qwen3.5-2B" ], "production_evidence": false, "summary": "MERA improves the small model behind a step-level router rather than only routing: it replays failed student invocations to get verified teacher demonstrations, distills a SkillBook, fine-tunes a LoRA student, and serves it behind a cost-calibrated router with verifier fallback, admitting changes only when replay preserves quality.", "key_idea": "Continually distill the cheap model from its routed failures.", "relevance_score": 5, "relevance_rationale": "Combines step routing with log-driven distillation.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [] }, { "id": "yuan-2026-coca-capability-allocation", "title": "Learning What to Activate: Combinatorial Capability Allocation for Long-Horizon Multimodal Agents", "authors": [ "Wen-Hao Yuan", "Chen-Chen Lin", "Jian Chen", "Jin-Feng Xu", "Shuo Yang", "Edith C. H. Ngai" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.27869", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "teacher conditional comparisons", "activation cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CoCA learns to activate a cost-sensitive subset of specialized capabilities (perception, retrieval, reasoning, verification, execution) at each stage of a long-horizon multimodal agent. A stronger teacher compares marginal capability values on student-visited states; the policy is distilled on-policy and refined with trajectory RL on success and activation cost.", "key_idea": "Per-stage combinatorial selection of which specialist modules to run, trading success against activation cost.", "relevance_score": 5, "relevance_rationale": "Stage-level allocation of expensive components learned with cost-aware RL.", "found_via": "snowball: cited-by zhang-2026-mtrouter", "cites_or_cited_by": [ "zhang-2026-mtrouter" ] }, { "id": "yuan-2026-flowbank", "title": "FlowBank: Query-Adaptive Agentic Workflows Optimization through Precompute-and-Reuse", "authors": [ "Lingzhi Yuan", "Chenghao Deng", "Fangxu Yu", "Souradip Chakraborty", "Mohammad Rostami", "Furong Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.11290", "code_url": null, "topic_clusters": [ "C1", "C4" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query-workflow edge-value prediction", "predicted query-workflow utility" ], "cost_reduction_reported": "Improves over strongest automated and handcrafted baselines by 4.26% and 14.92% relative while remaining cost-competitive", "benchmarks": [ "five benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "FlowBank builds a compact portfolio of complementary workflows offline and routes each query to the best one by predicting utility on a query-workflow bipartite graph, avoiding per-query workflow generation costs.", "key_idea": "Precompute a workflow portfolio and route queries among workflows.", "relevance_score": 5, "relevance_rationale": "Routing among agent configurations rather than models.", "found_via": "web search: MAS generation cost | search: FlowBank query-adaptive agentic workflows precompute reuse", "cites_or_cited_by": [ "gao-2025-flowreasoner", "yue-2025-masrouter", "zhang-2025-maas-agentic-supernet" ] }, { "id": "zhang-2026-budget-aware-agentic-routing", "title": "Budget-Aware Agentic Routing via Boundary-Guided Training", "authors": [ "Caiqi Zhang", "Menglin Xia", "Xuchao Zhang", "Daniel Madrigal", "Ankur Mallick", "Samuel Kessler", "Victor Ruehle", "Saravan Rajmohan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.21227", "code_url": null, "topic_clusters": [ "C1", "C2", "C3" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "boundary policies (always-small/always-large)", "per-task budget", "RL rewards", "boundary policies (always-small vs always-large)" ], "cost_reduction_reported": "Matches strong routing baselines at substantially lower cost (no single number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Budget-Aware Agentic Routing (Microsoft) chooses cheap vs expensive model at each agent step under strict per-task budgets, using boundary-guided SFT data and Boundary-Guided Policy Optimization to handle sparse rewards.", "key_idea": "Per-step cheap/expensive routing trained with boundary-guided RL.", "relevance_score": 5, "relevance_rationale": "Directly addresses step-level cost in agents.", "found_via": "snowball: cited-by ong-2024-routellm | OpenAlex search: per-step model routing", "cites_or_cited_by": [ "chen-2026-workload-router-pool", "ding-2024-hybrid-llm", "gao-2026-agentcollab", "jali-2026-turn-adaptive-budgets", "li-2026-rsi-router", "ong-2024-routellm", "paul-2026-agentrouter-step-tiers", "wang-2026-mcpp-budget-deadline-allocation", "xu-2026-seqroute" ] }, { "id": "zhang-2026-evoroute", "title": "EvoRoute: Experience-Driven Self-Routing LLM Agent Systems", "authors": [ "Guibin Zhang", "Haiyang Yu", "Kaiming Yang", "Bingli Wu", "Fei Huang", "Yongbin Li", "Shuicheng Yan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.02695", "code_url": null, "topic_clusters": [ "C3", "C4", "C5" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "experience knowledge base", "environment feedback" ], "cost_reduction_reported": "Reduces execution cost by up to 80% and latency by over 70% while sustaining or improving performance on GAIA and BrowseComp+", "benchmarks": [ "GAIA", "BrowseComp+" ], "models_studied": [], "production_evidence": false, "summary": "EvoRoute frames the Agent System Trilemma of performance, cost and latency and selects a Pareto-optimal LLM backbone at every step using a growing knowledge base of past experience, refining its policy from environment feedback. Plugged into off-the-shelf agent systems it cuts cost by up to 80%.", "key_idea": "Experience-driven per-step backbone selection that self-evolves.", "relevance_score": 5, "relevance_rationale": "Per-step model routing inside agents with large reported savings.", "found_via": "OpenAlex search: MAS backbone assignment | snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [ "chen-2026-workload-router-pool", "cui-2026-uno-orchestra", "jiang-2026-cera-moa", "kumar-2026-task-session-routing-eval", "lei-2026-tacit-switch", "wang-2026-mcpp-budget-deadline-allocation", "zhang-2025-maas-agentic-supernet" ] }, { "id": "zhang-2026-hera-device-cloud", "title": "Hera: Learning Long-Horizon Coordination for Device-Cloud Collaborative LLM Agents", "authors": [ "Yuxin Zhang", "Mengxue Hu", "Zheng Lin", "Xiaoyi Fan", "Fan Xie", "Zihan Fang", "Jing Yang", "Wenjun Zhu", "Zhiwen Chen", "Chengfei Lv", "Zhe Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.24598", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "device-cloud action agreement labels", "expected return", "future cloud calls" ], "cost_reduction_reported": "92.5% of cloud-only success with cloud use in only 46.3% of steps", "benchmarks": [ "ALFWorld", "WebShop", "AppWorld" ], "models_studied": [], "production_evidence": false, "summary": "Hera coordinates device and cloud LLM agents step by step, first learning by imitation from labels based on whether the device agent's action agrees with the cloud's, then with cost-aware RL that favors fewer future cloud calls. It keeps most cloud-only success while halving cloud usage.", "key_idea": "Per-step device/cloud routing trained by imitation then cost-aware RL.", "relevance_score": 5, "relevance_rationale": "Step-level big-little routing in long-horizon agents.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [ "liu-2026-cost-utility-alignment-trajectories", "narayan-2025-minions" ] }, { "id": "zhang-2026-hetero-designer", "title": "Hetero-Designer: Automated Design of Multi-Agent Systems with Heterogeneous LLMs", "authors": [ "Zhiheng Zhang", "Yuanzhe Zhang", "Bo-Han Yu", "Daojian Zeng", "Kang Liu", "Jun Zhao" ], "year": 2026, "venue_or_source": "ACL 2026", "source_type": "paper", "url": "https://doi.org/10.18653/v1/2026.acl-long.1272", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "query-LLM-role dependency encoding" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Hetero-Designer introduces automated design of multi-agent systems built from heterogeneous LLMs, jointly choosing roles and which LLM fills each role per query, aiming for cost-effective systems. It encodes dependencies among queries, LLMs and roles to search this joint space efficiently.", "key_idea": "Automated joint design of topology, roles and per-role LLM choice for heterogeneous MAS.", "relevance_score": 5, "relevance_rationale": "Directly the role-level model-assignment problem; ACL 2026.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [] }, { "id": "zhang-2026-mtrouter", "title": "MTRouter: Cost-Aware Multi-Turn LLM Routing with History-Model Joint Embeddings", "authors": [ "Yiqun Zhang", "Hao Li", "Zihan Wang", "Shi Feng", "Xiaocui Yang", "Daling Wang", "Bo Zhang", "Lei Bai", "Shuyue Hu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.23530", "code_url": "https://github.com/ZhangYiqun018/MTRouter", "topic_clusters": [ "C1", "C3", "C6" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "history-model joint embedding", "logged trajectory outcomes", "cost budget" ], "cost_reduction_reported": "On ScienceWorld surpasses GPT-5 while reducing total cost by 58.7%; on HLE competitive accuracy with 43.4% lower cost than GPT-5", "benchmarks": [ "ScienceWorld", "Humanity's Last Exam" ], "models_studied": [ "GPT-5" ], "production_evidence": false, "summary": "MTRouter selects which model to call at every turn of a multi-turn, long-horizon task under a fixed budget. It embeds the interaction history jointly with candidate models and learns a turn-level utility estimator from logged trajectories; it makes fewer model switches than prior multi-turn routers and shows emergent model specialization.", "key_idea": "Per-turn model selection learned offline from logged agent trajectories via history-model joint embeddings.", "relevance_score": 5, "relevance_rationale": "Directly addresses per-turn routing inside agent loops learned from trajectories (sub-themes a and c), with cost-quality numbers vs a frontier model.", "found_via": "arXiv query ti:\"LLM routing\" AND abs:\"multi-turn\"", "cites_or_cited_by": [ "yuan-2026-coca-capability-allocation" ] }, { "id": "zhang-2026-rethinking-mas-efficiency-evaluation", "title": "Rethinking the Evaluation of Efficiency Methods for Multi-Agent Systems", "authors": [ "Jiamu Zhang", "Lingxi Zhang", "Pengjun Lu", "Qiyue Zhang", "Yu-Neng Chuang", "Zhengchen Li", "Shuai Xu", "Vipin Chaudhary", "Hanjie Chen" ], "year": 2026, "venue_or_source": "EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2609.05933", "code_url": null, "topic_clusters": [ "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "controlled MAS-demanding diagnostic benchmark" ], "models_studied": [], "production_evidence": false, "summary": "Argues that reported gains of MAS efficiency methods (agent pruning, edge removal, compact structure search) are often overestimated because of method-specific prompts, starting topologies and non-MAS-demanding tasks where random pruning already works. A controlled benchmark shows many gains come from structural collapse or disabled tools.", "key_idea": "Many MAS pruning gains vanish under controlled, MAS-demanding evaluation.", "relevance_score": 5, "relevance_rationale": "Critical check on the AgentPrune/AgentDropout line of work.", "found_via": "snowball: cited-by zhang-2024-agentprune", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "gao-2025-single-or-multi-agent-why-not-both", "zhang-2025-maas-agentic-supernet" ] }, { "id": "zhao-2026-sc-mas", "title": "SC-MAS: Constructing Cost-Efficient Multi-Agent Systems with Edge-Level Heterogeneous Collaboration", "authors": [ "Di Zhao", "Longhui Ma", "Siwei Wang", "Miao Wang", "Yi Kong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.09434", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "query", "role", "edge context" ], "cost_reduction_reported": "+3.35% accuracy on MMLU with 15.38% lower inference cost; +3.53% on MBPP with 12.13% cost reduction", "benchmarks": [ "MMLU", "MBPP" ], "models_studied": [], "production_evidence": false, "summary": "SC-MAS models a MAS as a directed graph with edge-level collaboration strategies and uses a controller to pick roles, per-edge strategies and a backbone LLM for each agent per query. It improves accuracy while reducing cost relative to MasRouter-style homogeneous-mode routing.", "key_idea": "Jointly route roles, per-edge collaboration styles and per-agent LLMs.", "relevance_score": 5, "relevance_rationale": "Extends MasRouter with heterogeneous collaboration and backbone allocation.", "found_via": "web search: heterogeneous MAS | search: heterogeneous LLM assignment multi-agent system roles optimization cost arXiv", "cites_or_cited_by": [] }, { "id": "zhou-2026-agent-as-a-router", "title": "Agent-as-a-Router: Agentic Model Routing for Coding Tasks", "authors": [ "Pengfei Zhou", "Zhiwei Tang", "Yixing Ma", "Jiasheng Tang", "Yizeng Han", "Zhenglin Wan", "Fanqing Meng", "Wei Wang", "Bohan Zhuang", "Wangbo Zhao", "Yang You" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.22902", "code_url": "https://github.com/LanceZPF/agent-as-a-router", "topic_clusters": [ "C1", "C3", "C5", "C13" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "task-dimension performance statistics", "verifier", "memory of execution feedback", "execution feedback", "memory" ], "cost_reduction_reported": "Augmenting a vanilla LLM router with dimension-level performance statistics yields 15.3% relative gain (abstract)", "benchmarks": [ "CodeRouterBench (~10K tasks)", "CodeRouterBench (~10K tasks, 8 frontier LLMs)" ], "models_studied": [ "8 frontier LLMs" ], "production_evidence": false, "summary": "Argues router bottlenecks come from information deficit and proposes a Context-Action-Feedback loop router (orchestrator, verifier, memory) that accumulates execution-grounded experience during deployment on coding tasks.", "key_idea": "Agentic router that learns from execution feedback.", "relevance_score": 5, "relevance_rationale": "Deployment-time learning for coding agents.", "found_via": "snowball: cited-by ong-2024-routellm | web search: routing among agents", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "liu-2026-adaptive-vlm-routing-cua", "ong-2024-routellm", "yue-2025-masrouter" ] }, { "id": "zywot-2026-small-agents-beat-large-llm", "title": "Can Small Agents Collaborate to Beat a Single Large Language Model?", "authors": [ "Agata Żywot", "Xinyi Chen", "Yifei Yuan", "Anders Søgaard", "Maarten de Rijke" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.11327", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "factual retrieval", "multi-hop reasoning", "scientific QA", "math" ], "models_studied": [], "production_evidence": false, "summary": "Controlled comparisons show a minimal MAS of small models (one orchestrator plus specialist sub-agents) can beat much larger single-agent models on tool-heavy tasks. Reasoning at the orchestrator gives the biggest gains, while sub-agent reasoning adds little; orchestrator capacity drives performance.", "key_idea": "Spend capacity on the orchestrator; sub-agents can be small.", "relevance_score": 5, "relevance_rationale": "Direct evidence on where to allocate model capacity across roles.", "found_via": "web search: big-little collaboration", "cites_or_cited_by": [] }, { "id": "atalar-2025-sequential-bandits", "title": "Neural Bandit Based Optimal LLM Selection for a Pipeline of Subtasks", "authors": [ "Baran Atalar", "Eddie Zhang", "Carlee Joe-Wong", "Carlee Joe‐Wong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.09958", "code_url": null, "topic_clusters": [ "C3", "C4", "C5" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "neural contextual bandit", "subtask success", "cost", "subtask context", "downstream success" ], "cost_reduction_reported": null, "benchmarks": [ "two real datasets (e.g. diagnosis extraction pipeline)" ], "models_studied": [ "multiple LLMs" ], "production_evidence": false, "summary": "Addresses LLM selection for each subtask in a pipeline where upstream outputs affect downstream cost and success; proposes a neural contextual bandit (Sequential Bandits) with sublinear regret that needs no historical performance data.", "key_idea": "Online neural-bandit selection of LLMs per pipeline subtask.", "relevance_score": 5, "relevance_rationale": "Online per-step model assignment with cost for pipelines.", "found_via": "search: LLM selection for each stage of multi-stage pipeline budget constraint optimization arXiv | OpenAlex title: neural bandit LLM", "cites_or_cited_by": [ "nguyen-2024-metallm" ] }, { "id": "barker-2025-mo-hpo-rag", "title": "Faster, Cheaper, Better: Multi-Objective Hyperparameter Optimization for LLM and RAG Systems", "authors": [ "Matthew Barker", "Andrew Bell", "Evan Thomas", "James Carr", "Thomas Andrews", "Umang Bhatt" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.18635", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "cost", "latency", "safety", "alignment" ], "cost_reduction_reported": "Bayesian optimization obtains a superior Pareto front over baselines on two new RAG benchmark tasks (no single headline cost number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Introduces multi-objective parameter optimization of cost, latency, safety and alignment over entire LLM and RAG systems, including choice of LLM, embedder and reranker. Bayesian optimization beats baselines in Pareto quality.", "key_idea": "Multi-objective BO over full LLM/RAG system configs including model choice.", "relevance_score": 5, "relevance_rationale": "Cost as an explicit objective with LLM choice as a variable in end-to-end system tuning.", "found_via": "search: RAG hyperparameter configuration tuning Bayesian optimization cost arXiv 2025", "cites_or_cited_by": [] }, { "id": "belcak-2025-slms-future-agentic-ai", "title": "Small Language Models are the Future of Agentic AI", "authors": [ "Peter Belcak", "Greg Heinrich", "Shizhe Diao", "Yonggan Fu", "Xin Dong", "Saurav Muralidharan", "Yingyan Celine Lin", "Pavlo Molchanov" ], "year": 2025, "venue_or_source": "arXiv (position paper)", "source_type": "paper", "url": "https://arxiv.org/abs/2506.02153", "code_url": null, "topic_clusters": [ "C3", "C8", "C12", "C15" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "NVIDIA's position paper argues that small language models are capable enough, better suited and far cheaper for most repetitive, narrow invocations inside agentic systems, and that heterogeneous agents mixing SLMs and LLMs are the natural design. Outlines an LLM-to-SLM conversion algorithm from agent usage logs.", "key_idea": "Most agent calls should go to SLMs; convert LLM agents to SLMs using logged usage.", "relevance_score": 5, "relevance_rationale": "Seed; central thesis for subagent downscaling with a log-driven conversion recipe.", "found_via": "web search: seed | seed", "cites_or_cited_by": [ "bijoy-2025-prost-slm-multi-agent", "bozoglan-2026-slm-database-agent", "chen-2026-da-grpo-cloud-offloading", "da-2026-routebalance", "davidson-2025-collaboration-gap", "elzemity-2026-orchestrated-slms-malware", "he-2025-information-theoretic-agentic-design", "hua-2026-agentopt", "karmakar-2026-agentfloor", "kim-2025-cost-of-dynamic-reasoning", "kon-2026-swe-protege", "kondapalli-2026-slms-as-multi-agent-routers", "rainone-2026-hybrid-cloud-device-mas-lessons", "su-2025-toolorchestra", "taheri-2026-iffc-function-calling", "tripathy-2025-swenergy", "wang-2025-slm-mux", "wu-2025-inter-cascade", "xu-2026-agent-skills-slm-industrial", "xu-2026-oneflow-single-agent-baseline", "yang-2026-fusion-moa-fleet", "yang-2026-harness-adaptation-slm", "zhang-2026-orchslm", "zhu-2025-planner-r1" ] }, { "id": "cai-2025-agentbalance", "title": "AgentBalance: Backbone-then-Topology Design for Cost-Effective Multi-Agent Systems under Budget Constraints", "authors": [ "Shuowei Cai", "Yansong Ning", "Hao Liu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.11426", "code_url": "https://github.com/usail-hkust/AgentBalance", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "token-cost budget", "latency budget", "agent representations" ], "cost_reduction_reported": "Up to 10% and 22% performance gains under matched token-cost and latency budgets respectively (14 candidate LLM backbones)", "benchmarks": [ "benchmarks with 14 candidate LLM backbones" ], "models_studied": [ "14 LLM backbones" ], "production_evidence": false, "summary": "AgentBalance first matches heterogeneous LLM backbones to roles under explicit token-cost and latency budgets, then generates a latency-aware communication topology. It outperforms topology-first methods at equal budgets and works as a plug-in for existing MAS.", "key_idea": "Backbone-then-topology MAS design under explicit budgets.", "relevance_score": 5, "relevance_rationale": "Role-level model assignment under deployment budgets.", "found_via": "web search: MAS generation cost | search: AgentBalance backbone-then-topology cost-effective multi-agent budget", "cites_or_cited_by": [ "chen-2026-lemon-counterfactual-orchestration", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "chaudhry-2025-murakkab", "title": "Murakkab: Resource-Efficient Agentic Workflow Orchestration in Cloud Platforms", "authors": [ "Gohar Irfan Chaudhry", "Esha Choukse", "Haoran Qiu", "Íñigo Goiri", "Rodrigo Fonseca", "Adam Belay", "Ricardo Bianchini" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.18298", "code_url": null, "topic_clusters": [ "C3", "C4", "C11", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "profiles", "SLOs", "workflow structure", "user SLOs" ], "cost_reduction_reported": "Reduces GPU usage by up to 2.8x, energy by 3.7x and cost by 4.3x while maintaining SLOs", "benchmarks": [ "diverse agentic workflows" ], "models_studied": [], "production_evidence": false, "summary": "Murakkab is a serving system that decouples workflow specification from execution configuration and uses a profile-guided optimizer to map workflow components to models and hardware under SLOs.", "key_idea": "Declarative workflows with profile-guided model/hardware mapping.", "relevance_score": 5, "relevance_rationale": "Joint model and hardware configuration for agentic workflows (Microsoft).", "found_via": "snowball: cited-by chen-2025-llmselector | snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "white-2026-scepsy" ] }, { "id": "chen-2025-llmselector", "title": "Optimizing Model Selection for Compound AI Systems", "authors": [ "Lingjiao Chen", "Jared Quincy Davis", "Boris Hanin", "Peter Bailis", "Matei Zaharia", "James Zou", "Ion Stoica" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.14815", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "LLM-estimated per-module performance", "end-to-end task score" ], "cost_reduction_reported": "5%-70% accuracy gains vs. using the same LLM for all modules (multi-agent debate, self-refine); API calls scale linearly with number of modules", "benchmarks": [], "models_studied": [ "GPT-4o", "Claude 3.5 Sonnet", "Gemini 1.5" ], "production_evidence": false, "summary": "Studies how to pick which LLM to use for each module of a compound system (e.g., self-refine, multi-agent debate). Exploits the observation that end-to-end quality is often monotone in per-module quality and that an LLM can estimate per-module quality, then greedily assigns the best model module by module.", "key_idea": "Per-module model allocation via iterative greedy search with LLM-estimated module-wise performance.", "relevance_score": 5, "relevance_rationale": "Canonical formulation of model choice per module as a decision variable in compound AI systems; direct template for per-subagent model assignment.", "found_via": "seed: Optimizing Model Selection for Compound AI Systems", "cites_or_cited_by": [ "huang-2026-scope", "wu-2025-optimas" ] }, { "id": "conway-2025-syftr", "title": "syftr: Pareto-Optimal Generative AI", "authors": [ "Alexander Conway", "Debadeepta Dey", "Stefan Hackmann", "Matthew Hausknecht", "Michael Schmidt", "Mark Steadman", "Nick Volynets" ], "year": 2025, "venue_or_source": "AutoML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2505.20266", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "accuracy", "cost", "early-stopping" ], "cost_reduction_reported": "Finds flows on average ~9x cheaper while preserving most of the accuracy of the most accurate Pareto-frontier flows, across multiple RAG benchmarks", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "syftr runs multi-objective Bayesian optimization over a very large space of agentic and non-agentic RAG flows (embedders, splitters, retrievers, synthesizing LLMs, verifiers, rerankers) to find Pareto-optimal accuracy-cost configurations, with early stopping to prune weak candidates.", "key_idea": "Multi-objective BO over RAG/agentic flow configs including LLM choice per component.", "relevance_score": 5, "relevance_rationale": "Directly optimizes cost jointly with quality and treats LLM per module as a decision variable.", "found_via": "search: syftr Pareto optimal RAG flows arXiv", "cites_or_cited_by": [] }, { "id": "cuadron-2025-danger-of-overthinking-agents", "title": "The Danger of Overthinking: Examining the Reasoning-Action Dilemma in Agentic Tasks", "authors": [ "Alejandro Cuadron", "Dacheng Li", "Wenjie Ma", "Xingyao Wang", "Yichuan Wang", "Siyuan Zhuang", "Shu Liu", "Luis Gaspar Schroeder", "Tian Xia", "Huanzhi Mao", "Nicholas Thumiger", "Aditya Desai", "Ion Stoica", "Ana Klimovic", "Graham Neubig", "et al." ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.08235", "code_url": null, "topic_clusters": [ "C3", "C9", "C13" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "overthinking score" ], "cost_reduction_reported": "Selecting the solution with lower overthinking score improves performance by almost 30% while reducing computational costs by 43% (SWE-Bench Verified)", "benchmarks": [ "SWE-Bench Verified (4018 trajectories)" ], "models_studied": [ "reasoning and non-reasoning models" ], "production_evidence": false, "summary": "Studies overthinking in agentic software engineering, where reasoning models prefer internal reasoning over interacting with the environment, with failure patterns like analysis paralysis and rogue actions. Overthinking correlates with lower success.", "key_idea": "Reasoning models overthink in agent loops and this costs success.", "relevance_score": 5, "relevance_rationale": "Direct evidence on reasoning cost in agentic tasks.", "found_via": "arXiv title search: overthinking agents", "cites_or_cited_by": [] }, { "id": "dai-2025-aragog", "title": "Aragog: Just-in-Time Model Routing for Scalable Serving of Agentic Workflows", "authors": [ "Yinwei Dai", "Zhuofu Chen", "Anand Iyer", "Ravi Netravali" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.20975", "code_url": null, "topic_clusters": [ "C3", "C4", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "system load", "accuracy-preserving config set", "accuracy-preserving configuration set", "runtime system load" ], "cost_reduction_reported": "maximum serving throughput increased 42.8-217.0% and median latency reduced 32.5-86.1% at peak load while keeping accuracy comparable to the most expensive configurations", "benchmarks": [ "diverse agentic workflows" ], "models_studied": [ "multiple model families" ], "production_evidence": false, "summary": "Aragog assigns workflow agents to LLMs progressively during execution: a one-time routing step finds all accuracy-preserving configurations, and a cheap per-stage scheduler picks among them based on current system load. It greatly improves serving throughput and latency.", "key_idea": "Just-in-time per-stage model reassignment driven by live load.", "relevance_score": 5, "relevance_rationale": "Per-stage model assignment in agent workflows with systems evidence.", "found_via": "arXiv title search: model routing agentic | Aragog just-in-time model routing agentic workflows serving", "cites_or_cited_by": [ "chen-2026-workload-router-pool", "luo-2025-autellix", "qian-2026-dyserve", "yue-2025-masrouter" ] }, { "id": "dang-2025-puppeteer-evolving-orchestration", "title": "Multi-Agent Collaboration via Evolving Orchestration", "authors": [ "Yufan Dang", "Chen Qian", "Xueheng Luo", "Jingru Fan", "Zihao Xie", "Ruijie Shi", "Weize Chen", "Cheng Yang", "Xiaoyin Che", "Ye Tian", "Xuantang Xiong", "Lei Han", "Zhiyuan Liu", "Maosong Sun" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2505.19591", "code_url": "https://github.com/OpenBMB/ChatDev/tree/puppeteer", "topic_clusters": [ "C3", "C4", "C12" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "task state", "reward" ], "cost_reduction_reported": "Superior performance with reduced computational cost; token use falls during training", "benchmarks": [ "closed- and open-domain scenarios" ], "models_studied": [], "production_evidence": false, "summary": "A centralized orchestrator (puppeteer) is trained with REINFORCE to choose which agent acts next as the task state evolves, learning to skip unhelpful agents and terminate early. It improves performance while lowering compute, converging to compact cyclic structures.", "key_idea": "RL orchestrator sequences agents step by step and prunes low-utility ones.", "relevance_score": 5, "relevance_rationale": "Step-level agent selection with cost-aware RL.", "found_via": "web search: big-little collaboration | snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [ "chen-2024-optima", "cui-2026-uno-orchestra", "fan-2026-science-of-collective-ai", "ghosh-2026-grade-gated-routing", "jiang-2026-cera-moa", "ke-2026-mas-orchestra", "kim-2025-science-of-scaling-agent-systems", "liu-2026-cost-utility-alignment-trajectories", "liu-2026-e3-orch", "nielsen-2025-conductor", "rajib-2026-gated-memory-routing", "wang-2025-strmac", "wang-2026-agentconductor", "yuan-2026-paramanager-small-orchestrator", "zhang-2026-rethinking-mas-efficiency-evaluation" ] }, { "id": "dong-2025-hybridflow", "title": "HybridFlow: Resource-Adaptive Subtask Routing for Efficient Edge-Cloud LLM Inference", "authors": [ "Jiangwen Dong", "Jiayu Li", "Tianhang Zheng", "Wanyu Lin" ], "year": 2025, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2512.22137", "code_url": "https://github.com/WanyuGroup/ICML2026_HybridFlow", "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "learned benefit-cost utility", "budget" ], "cost_reduction_reported": "Reduces latency and cloud API usage while maintaining competitive accuracy (no single number in abstract)", "benchmarks": [ "GPQA", "MMLU-Pro", "AIME24", "LiveBench-Reasoning" ], "models_studied": [ "edge SLM", "cloud LLM" ], "production_evidence": false, "summary": "HybridFlow decomposes a query into a subtask DAG, runs unlocked subtasks in parallel and routes each subtask online to an edge SLM or cloud LLM via a learned benefit-cost utility and a budget-aware rule. It improves the cost-accuracy trade-off over per-query routing.", "key_idea": "Subtask-level edge/cloud routing with a learned utility under budgets.", "relevance_score": 5, "relevance_rationale": "Subtask-level model routing with budgets.", "found_via": "web search: HybridFlow", "cites_or_cited_by": [] }, { "id": "fan-2025-imad", "title": "iMAD: Intelligent Multi-Agent Debate for Efficient and Accurate LLM Inference", "authors": [ "Wei Fan", "JinYi Yoon", "Bo Ji" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2511.11306", "code_url": null, "topic_clusters": [ "C2", "C7", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "41 self-critique hesitation features", "debate-decision classifier" ], "cost_reduction_reported": "Up to 92% token reduction with up to 13.5% accuracy improvement", "benchmarks": [ "six (visual) QA datasets" ], "models_studied": [], "production_evidence": false, "summary": "iMAD triggers multi-agent debate only when a lightweight classifier on self-critique features predicts debate will fix a wrong single-agent answer, avoiding debate costs and harmful overturns on easy queries.", "key_idea": "Escalate from single agent to debate only when it is likely to help.", "relevance_score": 5, "relevance_rationale": "Query-level escalation from single-agent to MAS with large savings.", "found_via": "arXiv API search", "cites_or_cited_by": [] }, { "id": "feng-2025-ipr", "title": "IPR: Intelligent Prompt Routing with User-Controlled Quality-Cost Trade-offs", "authors": [ "Aosong Feng", "Balasubramaniam Srinivasan", "Yun Zhou", "Zhichao Xu", "Kang Zhou", "Sheng Guan", "Yueyan Chen", "Xian Wu", "Ninad Kulkarni", "Yi Zhang", "Zhengyuan Shen", "Dmitriy Bespalov", "Soumya Smruti Mishra", "Yifei Teng", "Darren Yow-Bang Wang", "Haibo Ding", "Lin Lee Cheong" ], "year": 2025, "venue_or_source": "EMNLP 2025 (Industry)", "source_type": "paper", "url": "https://arxiv.org/abs/2509.06274", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "lightweight quality estimators", "user tolerance parameter" ], "cost_reduction_reported": "Deployed on a major cloud platform: 43.9% cost reduction at quality parity with the strongest Claude-family model, sub-150ms latency (abstract)", "benchmarks": [ "IPRBench (1.5M prompts, 11 LLMs)" ], "models_studied": [ "Claude family" ], "production_evidence": true, "summary": "IPR (AWS) trains quality estimators on 1.5M annotated prompts and routes to the cheapest model within a user-specified quality tolerance. Frozen encoders with model-specific adapters speed onboarding; the system is deployed in production.", "key_idea": "Tolerance-controlled quality-estimator routing in a production cloud service.", "relevance_score": 5, "relevance_rationale": "Rare production evidence for query routing with frontier models.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "gao-2025-single-or-multi-agent-why-not-both", "title": "Single-agent or Multi-agent Systems? Why Not Both?", "authors": [ "Mingyan Gao", "Yanzi Li", "Banruo Liu", "Yifan Yu", "Phillip Wang", "Ching-Yu Lin", "Fan Lai" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.18286", "code_url": null, "topic_clusters": [ "C2", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "error-prone agent detection" ], "cost_reduction_reported": "Hybrid cascading between MAS and single-agent improves accuracy 1.1-12% while reducing deployment cost up to 20%", "benchmarks": [ "popular agentic applications" ], "models_studied": [ "OpenAI-o3", "Gemini-2.5-Pro" ], "production_evidence": false, "summary": "An empirical comparison of multi-agent and single-agent systems across agentic applications finds that MAS advantages shrink as frontier LLMs improve. Proposes request cascading between single-agent and multi-agent systems, gaining accuracy while cutting cost up to 20%.", "key_idea": "Cascade requests between single-agent and multi-agent systems.", "relevance_score": 5, "relevance_rationale": "Direct evidence and a mechanism for when MAS is worth it.", "found_via": "web search: single vs multi-agent cost", "cites_or_cited_by": [ "ke-2026-mas-orchestra", "kim-2025-science-of-scaling-agent-systems", "ruan-2026-aorchestra", "saleh-2026-illusion-agentic-complexity-readme", "tran-2026-single-vs-multi-agent-equal-budget", "yu-2026-when-mas-help-information-bottleneck", "zhang-2026-rethinking-mas-efficiency-evaluation" ] }, { "id": "hadfield-2025-multi-agent-research-system", "title": "How we built our multi-agent research system", "authors": [ "Jeremy Hadfield", "Barry Zhang", "Kenneth Lien", "Florian Scholz", "Jeremy Fox", "Daniel Ford" ], "year": 2025, "venue_or_source": "Anthropic engineering blog", "source_type": "blog", "url": "https://www.anthropic.com/engineering/multi-agent-research-system", "code_url": null, "topic_clusters": [ "C3", "C12", "C14" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "token usage", "tool calls", "model choice" ], "cost_reduction_reported": "Not a cost reduction: Opus 4 lead + Sonnet 4 subagents outperformed single-agent Opus 4 by 90.2% on an internal research eval; agents use ~4x chat tokens and multi-agent systems ~15x; token usage explained 80% of BrowseComp performance variance; parallel tool calls cut research time up to 90%", "benchmarks": [ "internal research eval", "BrowseComp" ], "models_studied": [ "Claude Opus 4 (lead)", "Claude Sonnet 4 (subagents)", "Claude Opus 4", "Claude Sonnet 4" ], "production_evidence": true, "summary": "Describes the production Claude Research system, where an Opus lead agent plans and spawns parallel Sonnet subagents with their own context windows. The heterogeneous setup beat single-agent Opus by 90.2% on an internal eval but used about 15x the tokens of chat, so it pays off only for high-value, parallelizable, context-heavy tasks.", "key_idea": "Big-model lead with smaller-model parallel subagents buys quality with tokens; worth it only for high-value breadth tasks.", "relevance_score": 5, "relevance_rationale": "Seed; production evidence of role-level model assignment and multi-agent token overhead.", "found_via": "web search: seed | seed list: Anthropic engineering blog", "cites_or_cited_by": [ "wei-2025-browsecomp" ] }, { "id": "he-2025-cognify", "title": "Cognify: Supercharging Gen-AI Workflows With Hierarchical Autotuning", "authors": [ "Zijian He", "Reyna Abhyankar", "Vikranth Srivatsa", "Yiying Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.08056", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "user evaluator score", "execution cost", "latency" ], "cost_reduction_reported": "Improves generation quality by up to 2.8x, reduces execution monetary cost by up to 10x, and end-to-end latency by 2.7x across six workflow types", "benchmarks": [ "RAG QA", "text-to-SQL" ], "models_studied": [], "production_evidence": false, "summary": "Cognify autotunes gen-AI workflows (LangChain, DSPy, or its own model) with AdaSeek, a hierarchical search that splits a user search budget across structure, operator (including model choice) and prompt layers and reallocates budget to promising configurations. Supports quality, cost and latency objectives and returns multiple Pareto versions.", "key_idea": "Budget-adaptive hierarchical autotuning over workflow structure, per-step model and prompt settings.", "relevance_score": 5, "relevance_rationale": "Directly optimizes per-step model selection jointly with cost/latency objectives on real workflow frameworks.", "found_via": "seed: Cognify", "cites_or_cited_by": [ "huang-2026-scope" ] }, { "id": "he-2025-information-theoretic-agentic-design", "title": "An Information Theoretic Perspective on Agentic System Design", "authors": [ "Shizhe He", "A. Narayan", "Ishan S. Khare", "Scott W. Linderman", "Christopher Ré", "Dan Biderman" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.21720", "code_url": null, "topic_clusters": [ "C3", "C10", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "mutual information between context and compression" ], "cost_reduction_reported": "A 7B Qwen-2.5 compressor is 1.6x more accurate, 4.6x more concise and conveys 5.5x more MI per token than its smaller sibling (abstract)", "benchmarks": [ "five datasets" ], "models_studied": [ "Qwen-2.5" ], "production_evidence": false, "summary": "The paper models agentic systems in which small compressor LMs summarize raw context for larger predictor LMs as a noisy channel. It proposes a mutual-information estimator that predicts downstream performance independent of task, and finds that larger compressors are both more accurate and more token-efficient; this guides how to size each role.", "key_idea": "Choose compressor/predictor sizes in multi-LM agents by mutual information rather than task-specific sweeps.", "relevance_score": 5, "relevance_rationale": "A principled analysis of small-worker/large-predictor splits, the core pattern behind Deep Research and Claude Code style systems.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai / su-2025-toolorchestra", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai", "su-2025-toolorchestra" ] }, { "id": "jin-2025-corl-centralized-budget", "title": "Controlling Performance and Budget of a Centralized Multi-agent LLM System with Reinforcement Learning", "authors": [ "Bowen Jin", "TJ Collins", "Donghan Yu", "Mert Cemri", "Shenao Zhang", "Mengyu Li", "Jay Tang", "Tian Qin", "Zhiyang Xu", "Jiarui Lu", "Guoli Yin", "Jiawei Han", "Zirui Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.02755", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "budget condition", "task state" ], "cost_reduction_reported": "Single system surpasses best expert LLM under high budget while staying strong in low-budget modes", "benchmarks": [ "four benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "CoRL trains a controller LLM with RL to selectively call a pool of expert LLMs, maximizing performance while minimizing cost, and conditions on a budget so one system can operate at multiple cost levels. It beats the best single expert at high budget.", "key_idea": "Budget-conditioned RL controller that decides which expert LLMs to invoke.", "relevance_score": 5, "relevance_rationale": "Controllable cost in centralized multi-LLM agents.", "found_via": "web search: heterogeneous MAS", "cites_or_cited_by": [ "fang-2026-critique-and-routing-controller", "hamri-2026-zebra-budget-allocation", "ke-2026-mas-orchestra", "paul-2026-agentrouter-step-tiers" ] }, { "id": "kapoor-2025-hal-holistic-agent-leaderboard", "title": "Holistic Agent Leaderboard: The Missing Infrastructure for AI Agent Evaluation", "authors": [ "Sayash Kapoor", "Benedikt Stroebl", "Peter Kirgis", "Nitya Nadgir", "Zachary S Siegel", "Boyi Wei", "Tianci Xue", "Ziru Chen", "Felix Chen", "Saiteja Utpala", "Franck Ndzomga", "Dheeraj Oruganty", "Sophie Luskin", "Kangheng Liu", "Botao Yu", "Amit Arora", "Dongyoon Hahm", "Harsh Trivedi", "Huan Sun", "Juyong Lee", "Tengjun Jin", "Yifan Mai", "Yifei Zhou", "Yuxuan Zhu", "Rishi Bommasani", "Daniel Kang", "Dawn Song", "Peter Henderson", "Yu Su", "Percy Liang", "Arvind Narayanan" ], "year": 2025, "venue_or_source": "arXiv (reported as ICLR 2026 in secondary sources)", "source_type": "paper", "url": "https://arxiv.org/abs/2510.11977", "code_url": null, "topic_clusters": [ "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "cost", "accuracy", "log inspection", "LLM-aided log inspection" ], "cost_reduction_reported": "Reports ~$40,000 total cost for 21,730 rollouts across 9 models x 9 benchmarks; finds higher reasoning effort reduced accuracy in most runs", "benchmarks": [ "9 benchmarks in coding, web navigation, science, customer service", "9 benchmarks incl. coding, web navigation, science, customer service (e.g., tau-bench, GAIA, SWE-bench)" ], "models_studied": [ "9 models" ], "production_evidence": false, "summary": "HAL provides a standardized harness that runs agent evaluations in parallel on hundreds of VMs, and reports 21,730 rollouts over 9 models and 9 benchmarks (about $40,000) with cost-accuracy Pareto analysis. LLM-aided log inspection surfaces behaviors like searching for benchmark answers; all 2.5B tokens of logs are released.", "key_idea": "Standardized, cost-aware agent leaderboard with released logs.", "relevance_score": 5, "relevance_rationale": "Seed; key evaluation infrastructure and cost-accuracy evidence across scaffolds and models.", "found_via": "web search: seed | seed: HAL", "cites_or_cited_by": [ "chen-2026-hm-router", "ghosh-2026-eval-costs-bottleneck", "jiang-2026-agentcard", "kapoor-2024-ai-agents-that-matter", "ndzomga-2026-efficient-benchmarking-agents", "rabanser-2026-agent-reliability" ] }, { "id": "kim-2025-science-of-scaling-agent-systems", "title": "Towards a Science of Scaling Agent Systems", "authors": [ "Yubin Kim", "Ken Gu", "Chanwoo Park", "Chunjong Park", "Samuel Schmidgall", "A. Ali Heydari", "Yao Yan", "Zhihan Zhang", "Yuchen Zhuang", "Yun Liu", "Mark Malhotra", "Paul Pu Liang", "Hae Won Park", "Yuzhe Yang", "Xuhai Xu", "Yilun Du", "Shwetak Patel", "Tim Althoff", "Daniel McDuff", "Xin Liu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.08296", "code_url": null, "topic_clusters": [ "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "coordination metrics" ], "cost_reduction_reported": "Relative change vs single agent ranges from +80.8% (decomposable financial reasoning) to -70.0% (sequential planning)", "benchmarks": [ "six agentic benchmarks" ], "models_studied": [ "three LLM families" ], "production_evidence": false, "summary": "Google Research runs 260 controlled configurations across five architectures (single-agent and four multi-agent types) and six benchmarks, fitting a predictive model of when coordination helps. Multi-agent gains saturate as single-agent capability rises, tool-heavy tasks incur multi-agent overhead, and mismatched coordination hurts performance. A companion Google Research blog post ('Towards a science of scaling agent systems: When and why agent systems work') summarizes the findings.", "key_idea": "Predictive scaling principles for when multi-agent architectures help.", "relevance_score": 5, "relevance_rationale": "Core evidence for when multi-agent is worth it.", "found_via": "web search: scaling agents cost", "cites_or_cited_by": [ "bertalanic-2026-small-llm-team-scaling-decomposition", "dang-2025-puppeteer-evolving-orchestration", "dennis-2026-in-context-obsoletes-orchestration", "fan-2026-science-of-collective-ai", "gao-2025-single-or-multi-agent-why-not-both", "gharzeddine-2026-cyclic-subtask-graphs", "gillespie-2026-enterprise-agent-routing-scaling", "kapoor-2024-ai-agents-that-matter", "li-2026-single-agent-skills-vs-mas", "liu-2026-ems-majority-then-stopping", "liu-2026-hcp-mad", "mieczkowski-2026-latte-adaptive-task-graphs", "rainone-2026-hybrid-cloud-device-mas-lessons", "tran-2026-single-vs-multi-agent-equal-budget", "yang-2026-harness-adaptation-slm", "yu-2026-when-mas-help-information-bottleneck", "zeng-2026-mixture-of-roles-single-agent", "zhang-2025-maas-agentic-supernet" ] }, { "id": "kiyohara-2025-prompt-opt-logged-bandit", "title": "Prompt Optimization with Logged Bandit Data", "authors": [ "Haruka Kiyohara", "Daniel Yiming Cao", "Yuta Saito", "Thorsten Joachims" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.02646", "code_url": null, "topic_clusters": [ "C4", "C6" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "click feedback", "sentence similarity kernel" ], "cost_reduction_reported": null, "benchmarks": [ "new benchmark suite: personalized movie descriptions" ], "models_studied": [ "LLM pipeline with candidate prompts" ], "production_evidence": false, "summary": "Optimizes which prompt an LLM pipeline uses for personalized sentence generation from naturally logged user feedback (clicks), with a kernel-based off-policy gradient that exploits similarity between generated sentences to cut variance while limiting bias.", "key_idea": "Off-policy prompt-policy learning from logged bandit feedback.", "relevance_score": 5, "relevance_rationale": "Direct OPE/OPL applied to LLM configuration choice from production-style logs.", "found_via": "OpenAlex title search (seed: Kiyohara/Saito OPE for language generation)", "cites_or_cited_by": [ "kiyohara-2025-opl-sentence-personalization", "saito-2022-ope-large-action-embeddings", "swaminathan-2015-crm" ] }, { "id": "lee-2025-cope-llm-collaboration-planning", "title": "Efficient LLM Collaboration via Planning", "authors": [ "Byeongchan Lee", "Jonghoon Lee", "Dongyoung Kim", "Jaehyung Kim", "Kyungjoon Park", "Dongjun Lee", "Jinwoo Shin" ], "year": 2025, "venue_or_source": "TMLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2506.11578", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "plan quality / confidence in cascade" ], "cost_reduction_reported": "Comparable performance to large proprietary models with drastically reduced inference API cost", "benchmarks": [ "math reasoning", "code generation", "open-ended tasks", "agent tasks" ], "models_studied": [ "small open models", "large proprietary models" ], "production_evidence": false, "summary": "COPE lets small and large models alternate as planner and executor in a multi-stage cascade, with plans acting as a lightweight intermediate that guides the executor. It reaches large-model performance on math, code, open-ended and agent tasks at much lower API cost.", "key_idea": "Plans as a cheap intermediate in a small/large planner-executor cascade.", "relevance_score": 5, "relevance_rationale": "Planner-executor model split with escalation.", "found_via": "web search: planner executor", "cites_or_cited_by": [] }, { "id": "lee-2025-steer", "title": "Confidence-Guided Stepwise Model Routing for Cost-Efficient Reasoning", "authors": [ "Sangmook Lee", "Dohyung Kim", "Hyukhun Koh", "Nakyeong Yang", "Kyomin Jung" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.06190", "code_url": null, "topic_clusters": [ "C2", "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "small-model logit confidence before each reasoning step" ], "cost_reduction_reported": "Up to +20% accuracy with 48% fewer FLOPs vs using only the larger model on AIME", "benchmarks": [ "AIME", "multi-hop QA", "planning" ], "models_studied": [], "production_evidence": false, "summary": "Routes each reasoning step between a small and a large model based on the small model's logit confidence before it generates the step, with no trained router.", "key_idea": "Step-level SLM/LLM routing on internal confidence.", "relevance_score": 5, "relevance_rationale": "Training-free step-level model switching.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty", "si-2026-policy-guided-stepwise-routing", "wu-2026-differentiable-moa", "ye-2026-ecotab-stepwise-table-routing", "ye-2026-roro-process-reward-routing" ] }, { "id": "li-2025-llm-bandit", "title": "LLM Bandit: Cost-Efficient LLM Generation via Preference-Conditioned Dynamic Routing", "authors": [ "Yang Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.02743", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query embedding", "user preference vector", "model identity vectors", "query features", "user cost/performance preference", "model identity features" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "multiple LLM platforms" ], "production_evidence": false, "summary": "Formulates LLM selection as a multi-armed bandit with preference-conditioned dynamic routing so users can set accuracy-cost preferences at inference time. The policy is designed to generalize to unseen LLMs.", "key_idea": "Preference-conditioned bandit router generalizing to new models.", "relevance_score": 5, "relevance_rationale": "Tunable online router.", "found_via": "snowball: cited-by ong-2024-routellm | ti:\"LLM Bandit\"", "cites_or_cited_by": [ "nguyen-2024-metallm", "wei-2025-barp", "zeng-2026-metarouter", "zhang-2026-m-cmab" ] }, { "id": "li-2025-webrouter", "title": "WebRouter: Query-specific Router via Variational Information Bottleneck for Cost-sensitive Web Agent", "authors": [ "Tao Li", "Jinlong Hu", "Yang Wang", "Junfeng Liu", "Xuejun Liu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.11221", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "compressed prompt representation (VIB)", "expected operational cost" ], "cost_reduction_reported": "87.8% lower operational cost vs GPT-4o baseline with 3.8% accuracy drop on five WebVoyager websites", "benchmarks": [ "WebVoyager" ], "models_studied": [ "GPT-4o" ], "production_evidence": false, "summary": "WebRouter is a router for LLM-driven web agents whose prompts combine goals, action histories and page state. It is trained with a cost-aware variational information bottleneck that compresses the prompt while penalizing expected cost, and it chooses a model per agent query.", "key_idea": "Cost-aware information-bottleneck router for web-agent calls.", "relevance_score": 5, "relevance_rationale": "Routing for web agents with a large measured cost reduction.", "found_via": "arXiv query ti:router AND ti:agent", "cites_or_cited_by": [] }, { "id": "narayan-2025-minions", "title": "Minions: Cost-efficient Collaboration Between On-device and Cloud Language Models", "authors": [ "Avanika Narayan", "Dan Biderman", "Sabri Eyuboglu", "Avner May", "Scott Linderman", "James Zou", "Christopher Re" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.15964", "code_url": "https://github.com/HazyResearch/minions", "topic_clusters": [ "C3", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "Naive Minion: 30.4x lower remote cost at 87% of frontier quality; MinionS: 5.7x lower cost at 97.9% of remote-only performance", "benchmarks": [ "financial, medical and scientific long-document reasoning tasks" ], "models_studied": [ "small on-device LM", "frontier cloud LM" ], "production_evidence": false, "summary": "Studies local-remote collaboration where a small on-device model reads long documents and a frontier cloud model coordinates. In MinionS the remote model decomposes the task into chunk-level subtasks executed in parallel by the local model, cutting cloud cost 5.7x while keeping about 98% of quality. Together AI published a companion blog post on Minions.", "key_idea": "Frontier model decomposes; small local model executes many cheap subtasks.", "relevance_score": 5, "relevance_rationale": "Planner-big/executor-small division of labor with measured cost savings.", "found_via": "web search: Minions", "cites_or_cited_by": [ "chen-2025-llm-slm-collab-survey", "zhang-2026-hera-device-cloud" ] }, { "id": "nie-2025-foundation-model-programs", "title": "Resource-efficient Inference with Foundation Model Programs", "authors": [ "Lunyiu Nie", "Zhimin Ding", "Kevin Yu", "Marco Cheung", "Chris Jermaine", "Swarat Chaudhuri" ], "year": 2025, "venue_or_source": "COLM 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2504.07247", "code_url": null, "topic_clusters": [ "C2", "C3", "C5" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "per-module backend choice", "ground-truth feedback stream" ], "cost_reduction_reported": "Up to 98% resource savings with minimal accuracy loss vs monolithic multimodal models", "benchmarks": [ "two streaming VQA tasks" ], "models_studied": [], "production_evidence": false, "summary": "Compiles a task into a program whose modules call foundation models, then learns an online policy that picks cheap or expensive backends per module per input.", "key_idea": "Per-module backend selection in model programs, learned online.", "relevance_score": 5, "relevance_rationale": "Step-level model assignment learned from feedback.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "openai-2025-gpt5-system-card", "title": "GPT-5 System Card", "authors": [ "OpenAI" ], "year": 2025, "venue_or_source": "OpenAI (system card)", "source_type": "docs", "url": "https://cdn.openai.com/gpt-5-system-card.pdf", "code_url": null, "topic_clusters": [ "C1", "C5", "C9", "C14" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "conversation type", "complexity", "tool needs", "explicit user intent", "user model switches", "preference rates", "measured correctness", "explicit intent" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "gpt-5-main", "gpt-5-thinking", "mini variants" ], "production_evidence": true, "summary": "Describes GPT-5 as a unified system with a fast model, a deeper reasoning model, and a real-time router that picks between them using conversation type, complexity, tool needs and explicit intent. The router is continuously trained on real signals such as user model switches, response preference rates and measured correctness, and mini models handle traffic once usage limits are hit.", "key_idea": "Production query-level router between fast and reasoning models trained on live user signals.", "relevance_score": 5, "relevance_rationale": "Largest publicly documented production deployment of query-level routing with online learning from traffic.", "found_via": "search: GPT-5 system card real-time router | seed: GPT-5 system card", "cites_or_cited_by": [] }, { "id": "panda-2025-pilot", "title": "Adaptive LLM Routing under Budget Constraints", "authors": [ "Pranoy Panda", "Raghav Magazine", "Chaitanya Devaguptapu", "Sho Takemori", "Vishal Sharma" ], "year": 2025, "venue_or_source": "EMNLP 2025 (Findings)", "source_type": "paper", "url": "https://arxiv.org/abs/2508.21141", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "shared query-LLM embedding", "offline human preference prior", "LinUCB bandit feedback", "budget knapsack", "query embedding", "LLM embedding", "human preference prior", "bandit reward", "budget" ], "cost_reduction_reported": null, "benchmarks": [ "RouterBench-style routing benchmarks" ], "models_studied": [ "multiple LLMs (pool incl. GPT-4 class)" ], "production_evidence": false, "summary": "PILOT treats routing as a contextual bandit, initializing a shared query-model embedding space from offline human preference data and refining it online with LinUCB. An online cost policy based on a multi-choice knapsack enforces user budgets.", "key_idea": "Preference-prior LinUCB with knapsack budget control.", "relevance_score": 5, "relevance_rationale": "Offline-to-online routing under budgets.", "found_via": "snowball: cited-by ong-2024-routellm | ti:\"Adaptive LLM Routing under Budget Constraints\"", "cites_or_cited_by": [ "badanidiyuru-2013-bwk", "bao-2026-orcarouter", "chiang-2025-dueling-routing", "ding-2024-hybrid-llm", "li-2010-linucb", "li-2026-wiserouter", "nguyen-2024-metallm", "ong-2024-routellm", "taberner-miller-2026-paretobandit", "wei-2025-barp" ] }, { "id": "qian-2025-xrouter", "title": "xRouter: Training Cost-Aware LLMs Orchestration System via Reinforcement Learning", "authors": [ "Cheng Qian", "Zuxin Liu", "Shirley Kokane", "Akshara Prabhakar", "Jielin Qiu", "Haolin Chen", "Zhiwei Liu", "Heng Ji", "Weiran Yao", "Shelby Heinecke", "Silvio Savarese", "Caiming Xiong", "Huan Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.08439", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "router LLM tool calls", "cost-aware reward" ], "cost_reduction_reported": "Substantial cost reductions at comparable task completion rates across diverse benchmarks (no single number in abstract)", "benchmarks": [ "diverse benchmarks" ], "models_studied": [ "small open router models", "external premium LLMs" ], "production_evidence": false, "summary": "xRouter (Salesforce) trains a tool-calling router LLM with RL and an explicit cost-aware reward; it can answer directly or invoke one or more external models. The paper reports substantial cost reductions at comparable completion rates and shares lessons about what helps learned routing.", "key_idea": "RL-trained orchestrator that decides whether and which external models to call.", "relevance_score": 5, "relevance_rationale": "Cost-aware orchestration trained end-to-end; transferable to agent subcalls.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: related work of zhang-2026-budget-aware-agentic-routing", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "russo-2025-abacus", "title": "Abacus: A Cost-Based Optimizer for Semantic Operator Systems", "authors": [ "Matthew Russo", "Chunwei Liu", "Sivaprasad Sudhir", "Gerardo Vitagliano", "Michael Cafarella", "Tim Kraska", "Samuel Madden" ], "year": 2025, "venue_or_source": "VLDB 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2505.14661", "code_url": "https://github.com/mitdbg/palimpzest", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "validation examples", "prior beliefs", "LLM judge" ], "cost_reduction_reported": "Abacus-optimized systems achieve 6.7%-39.4% better quality and are 10.8x cheaper and 3.4x faster than the next best system on average (BioDEX, CUAD, MMQA)", "benchmarks": [ "BioDEX", "CUAD", "MMQA" ], "models_studied": [], "production_evidence": false, "summary": "Abacus is a cost-based optimizer for semantic operator systems (in Palimpzest) that picks physical implementations, including model choice per operator, under constrained objectives, estimating operator performance from few validation examples, priors or an LLM judge and tracking Pareto frontiers of subplans.", "key_idea": "Pareto-Cascades cost-based query optimization over LLM operator implementations.", "relevance_score": 5, "relevance_rationale": "Database-style per-operator model/implementation selection with cost-quality constraints.", "found_via": "search: Abacus cost-based optimizer semantic operator systems Palimpzest arXiv", "cites_or_cited_by": [ "wei-2025-moar" ] }, { "id": "sabbatella-2025-malbo", "title": "MALBO: Optimizing LLM-Based Multi-Agent Teams via Multi-Objective Bayesian Optimization", "authors": [ "Antonio Sabbatella" ], "year": 2025, "venue_or_source": "arXiv (Master's thesis)", "source_type": "paper", "url": "https://arxiv.org/abs/2511.11788", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "Gaussian-process surrogates", "accuracy", "inference cost", "GP surrogate" ], "cost_reduction_reported": "BO phase reduced average configuration cost by over 45% vs random search at comparable performance; heterogeneous teams achieve up to 65.8% cost reduction vs homogeneous baselines at maximum performance", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MALBO frames assigning LLMs to agent roles as multi-objective Bayesian optimization over accuracy and cost, using GP surrogates over a continuous LLM feature space and expected hypervolume improvement. It finds heterogeneous teams that are much cheaper than homogeneous ones.", "key_idea": "Multi-objective BO over per-role LLM assignments.", "relevance_score": 5, "relevance_rationale": "Directly optimizes role-level model assignment for cost.", "found_via": "OpenAlex search: heterogeneous LLMs multi-agent | search: multi-objective agentic workflow optimization cost accuracy Pareto arXiv", "cites_or_cited_by": [] }, { "id": "shao-2025-r2-reasoner", "title": "Route-and-Reason: Scaling Large Language Model Reasoning with Reinforced Model Router", "authors": [ "Chenyang Shao", "Xinyang Liu", "Yutang Lin", "Fengli Xu", "Yong Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.05901", "code_url": null, "topic_clusters": [ "C1", "C3", "C9" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "subtask decomposition", "subtask allocator" ], "cost_reduction_reported": "Reduces API costs by 84.46% vs state-of-the-art baselines while maintaining competitive accuracy on six reasoning benchmarks", "benchmarks": [ "six reasoning benchmarks" ], "models_studied": [ "nine models from <1B to hundreds of billions of parameters" ], "production_evidence": false, "summary": "A decomposer splits a query into subtasks and a reinforced allocator assigns each subtask to one of nine models of very different sizes. Trained by alternating SFT and RL.", "key_idea": "Decompose, then route each subtask to a model.", "relevance_score": 5, "relevance_rationale": "Subtask-level model assignment with large reported savings.", "found_via": "web search: route between reasoning and non-reasoning models", "cites_or_cited_by": [] }, { "id": "si-2025-ccpo", "title": "Conformal Constrained Policy Optimization for Cost-Effective LLM Agents", "authors": [ "Wenwen Si", "Sooyong Jang", "Insup Lee", "Osbert Bastani" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.11828", "code_url": null, "topic_clusters": [ "C2", "C3", "C6" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "conformal threshold", "cost-aware score" ], "cost_reduction_reported": "Up to 30% cost reduction vs other cost-aware baselines and LLM-guided methods without compromising reliability on two multi-hop QA benchmarks", "benchmarks": [ "two multi-hop QA benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "CCPO trains an orchestration policy that runs models and tools of different cost in sequence, minimizing cost subject to a user-specified reliability level formalized with conformal prediction. It combines constrained policy optimization with off-policy RL and online conformal prediction.", "key_idea": "Cost-minimizing multi-model agent orchestration with conformal reliability constraints.", "relevance_score": 5, "relevance_rationale": "Sequential multi-model orchestration in agents with guarantees and cost numbers.", "found_via": "arXiv query ti:\"cost-effective\" AND ti:agents", "cites_or_cited_by": [ "si-2026-policy-guided-stepwise-routing" ] }, { "id": "somerstep-2025-carrot", "title": "CARROT: A Cost Aware Rate Optimal Router", "authors": [ "Seamus Somerstep", "Felipe Maia Polo", "Allysson Flavio Melo de Oliveira", "Prattyush Mangal", "Mírian Silva", "Onkar Bhardwaj", "Mikhail Yurochkin", "Subha Maity" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.03261", "code_url": null, "topic_clusters": [ "C1", "C6", "C13", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted accuracy", "predicted cost", "query embedding" ], "cost_reduction_reported": null, "benchmarks": [ "SPROUT", "RouterBench", "Open LLM Leaderboard v2", "open-LLM-leaderboard-v2", "SPROUT (GPQA, MuSR, MMLU-Pro, MATH, OpenHermes, RAGBench)" ], "models_studied": [ "o3-mini", "13 SOTA LLMs incl. GPT-4o, Claude-3.5-Sonnet, o3-mini (per SPROUT)" ], "production_evidence": false, "summary": "Provides a minimax analysis of routing and shows a router that separately estimates each model's cost and accuracy can be rate-optimal. Introduces CARROT based on this principle and releases the SPROUT dataset spanning recent frontier LLMs with price-aware routing labels.", "key_idea": "Estimate both cost and accuracy per model and plug into a linear utility: minimax-optimal routing.", "relevance_score": 5, "relevance_rationale": "Theory-backed simple router plus the SPROUT dataset (seed 'SPROUT' is released here).", "found_via": "seed | snowball: cited-by nguyen-2024-metallm | seed: SPROUT", "cites_or_cited_by": [ "hu-2024-routerbench", "polo-2024-tinybenchmarks" ] }, { "id": "su-2025-daao", "title": "Difficulty-Aware Agentic Orchestration for Query-Specific Multi-Agent Workflows", "authors": [ "Jinwei Su", "Qizhen Lan", "Yinghui Xia", "Lifan Sun", "Weiyou Tian", "Tianyu Shi", "Xinyuan Song", "Lewei He", "Yang Jingsong" ], "year": 2025, "venue_or_source": "WWW 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2509.11079", "code_url": null, "topic_clusters": [ "C1", "C3", "C4" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "VAE difficulty estimate", "workflow success feedback", "predicted query difficulty (VAE)", "cost-aware router" ], "cost_reduction_reported": "Up to 11.21% higher accuracy with up to 36% lower cost (per search summary of the paper); abstract reports gains in accuracy and efficiency on six benchmarks", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "DAAO predicts query difficulty with a VAE, allocates workflow depth and operators accordingly, and routes each operator to an LLM with a cost- and performance-aware router, updating difficulty estimates from workflow outcomes. Simple queries get lighter workflows and cheaper models.", "key_idea": "Difficulty-driven workflow depth plus per-operator LLM routing.", "relevance_score": 5, "relevance_rationale": "Per-node model routing within query-specific workflows.", "found_via": "web search: difficulty-aware orchestration | search: Difficulty-Aware Agentic Orchestration query-specific workflows DAAO arXiv", "cites_or_cited_by": [ "chen-2026-lemon-counterfactual-orchestration", "hamri-2026-zebra-budget-allocation", "leins-2026-when-orchestration-pays-off", "liu-2026-e3-orch", "liu-2026-treecredit", "paul-2026-agentrouter-step-tiers", "yue-2025-masrouter", "zhang-2025-maas-agentic-supernet" ] }, { "id": "su-2025-toolorchestra", "title": "ToolOrchestra: Elevating Intelligence via Efficient Model and Tool Orchestration", "authors": [ "Hongjin Su", "Shizhe Diao", "Ximing Lu", "Mingjie Liu", "Jiacheng Xu", "Xin Dong", "Yonggan Fu", "Peter Belcak", "Hanrong Ye", "Hongxu Yin", "Yi Dong", "Evelina Bakhturina", "Tao Yu", "Yejin Choi", "Jan Kautz", "Pavlo Molchanov" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.21689", "code_url": null, "topic_clusters": [ "C1", "C3", "C8" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "outcome, efficiency and user-preference rewards", "outcome, efficiency and preference rewards" ], "cost_reduction_reported": "On HLE 37.1% vs GPT-5 35.1% while 2.5x more efficient; on tau2-Bench and FRAMES surpasses GPT-5 using about 30% of the cost", "benchmarks": [ "Humanity's Last Exam", "tau2-Bench", "FRAMES", "HLE" ], "models_studied": [ "Orchestrator-8B", "GPT-5", "Qwen3-8B" ], "production_evidence": false, "summary": "NVIDIA's ToolOrchestra trains a small 8B orchestrator with RL to coordinate tools and other models, including larger LLMs, with rewards for outcome, efficiency and user tool preferences. The Orchestrator beats GPT-5 on HLE at 2.5x better efficiency and on tau2-Bench and FRAMES at about 30% of the cost.", "key_idea": "A small RL-trained orchestrator decides when to call expensive models and tools.", "relevance_score": 5, "relevance_rationale": "Strong evidence that a small model can route per step among models and tools cost-effectively.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai | hf-papers search: small model orchestrator", "cites_or_cited_by": [ "babu-2026-agentbetta", "belcak-2025-slms-future-agentic-ai", "chen-2026-lemon-counterfactual-orchestration", "chen-2026-oats-tool-selection", "cui-2026-uno-orchestra", "guo-2026-sciorch", "he-2025-information-theoretic-agentic-design", "marjanovic-2026-model-pools-mas", "nie-2026-rivet-internalizing-experts", "rainone-2026-hybrid-cloud-device-mas-lessons", "ruan-2026-aorchestra", "wang-2025-efficient-agents", "woo-2026-prefillshare" ] }, { "id": "tan-2025-langprobe", "title": "LangProBe: a Language Programs Benchmark", "authors": [ "Shangyin Tan", "Lakshya A Agrawal", "Arnav Singhvi", "Liheng Lai", "Michael J Ryan", "Dan Klein", "Omar Khattab", "Koushik Sen", "Matei Zaharia" ], "year": 2025, "venue_or_source": "Findings of EMNLP 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.20315", "code_url": null, "topic_clusters": [ "C4", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "quality", "cost" ], "cost_reduction_reported": "Optimized language programs give strong cost-quality Pareto improvements over raw model calls; smaller models in optimized programs beat larger models at a fraction of the cost", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "LangProBe benchmarks over 2000 combinations of tasks, program architectures, optimizers and LMs, studying quality-cost trade-offs of language programs. Human judgment about which compositions to pursue still matters.", "key_idea": "Cost-aware benchmark of compound LM program architectures and optimizers.", "relevance_score": 5, "relevance_rationale": "Directly measures cost-quality trade-offs of model choice plus program optimization.", "found_via": "search: LangProBe language programs benchmark cost quality optimizers", "cites_or_cited_by": [ "khattab-2023-dspy", "opsahlong-2024-mipro" ] }, { "id": "tsiourvas-2025-causal-routing", "title": "Causal LLM Routing: End-to-End Regret Minimization from Observational Data", "authors": [ "Asterios Tsiourvas", "Wei Sun", "Georgia Perakis" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2505.16037", "code_url": null, "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "observational (logged) outcomes", "regret surrogate losses" ], "cost_reduction_reported": null, "benchmarks": [ "public benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Learns routing policies end-to-end from observational data where only the deployed model's outcome is logged, by minimizing decision regret with two surrogate objectives. It avoids requiring full-feedback evaluation of all models and handles heterogeneous cost preferences.", "key_idea": "Causal, regret-minimizing router trained from logged single-model outcomes.", "relevance_score": 5, "relevance_rationale": "Directly addresses learning a router from production logs.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "dai-2026-policy-regret-embedding-routing", "frauen-2026-causal-methods-llm", "juneja-2026-cost-ordered-feasibility", "khosravi-2026-which-llm-which-work", "lai-2026-routejudge", "nguyen-2026-cupid-model-zoo", "ong-2024-routellm", "zu-2026-barouter" ] }, { "id": "wang-2025-agent-reinforce", "title": "Generalizing Test-time Compute-optimal Scaling as an Optimizable Graph", "authors": [ "Fali Wang", "Jihai Chen", "Shuhua Yang", "Runxue Bao", "Tianxiang Zhao", "Zhiwei Zhang", "Xianfeng Tang", "Hui Liu", "Qi He", "Suhang Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.00086", "code_url": null, "topic_clusters": [ "C3", "C4", "C9" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "textual feedback as gradient", "accuracy", "latency" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formulates test-time scaling as searching compute-optimal multi-LLM collaboration graphs (roles, model assignments, edges) under a fixed budget and proposes Agent-REINFORCE, an LLM-augmented REINFORCE-style search.", "key_idea": "Search over multi-LLM collaboration graphs under a compute budget.", "relevance_score": 5, "relevance_rationale": "Joint role/model/topology search under budget.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "wang-2025-agentdropout", "title": "AgentDropout: Dynamic Agent Elimination for Token-Efficient and High-Performance LLM-Based Multi-Agent Collaboration", "authors": [ "Zhexuan Wang", "Yutong Wang", "Xuebo Liu", "Liang Ding", "Miao Zhang", "Jie Liu", "Min Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.18891", "code_url": "https://github.com/wangzx1219/AgentDropout", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "communication graph adjacency" ], "cost_reduction_reported": "21.6% fewer prompt tokens and 18.4% fewer completion tokens with +1.14 performance vs SOTA", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AgentDropout learns which agents and edges to drop in each communication round by optimizing the graph adjacency matrices, removing redundant agents dynamically. It lowers token use and slightly improves accuracy over prior topology methods.", "key_idea": "Drop redundant agents and edges per round by optimizing communication graphs.", "relevance_score": 5, "relevance_rationale": "Seed; agent-level pruning for cost.", "found_via": "web search: seed", "cites_or_cited_by": [] }, { "id": "wang-2025-agenttts", "title": "AgentTTS: Large Language Model Agent for Test-time Compute-optimal Scaling Strategy in Complex Tasks", "authors": [ "Fali Wang", "Hui Liu", "Zhenwei Dai", "Jingying Zeng", "Zhiwei Zhang", "Zongyu Wu", "Chen Luo", "Zhen Li", "Xianfeng Tang", "Qi He", "Suhang Wang" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2508.00890", "code_url": null, "topic_clusters": [ "C3", "C4", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "execution feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AgentTTS searches for compute-optimal model and budget allocations per subtask in multi-stage tasks with an LLM agent guided by empirical insights, beating traditional and LLM-based baselines in search efficiency.", "key_idea": "LLM-agent search over per-subtask model and budget allocation.", "relevance_score": 5, "relevance_rationale": "Per-subtask model selection and budget allocation.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "wang-2025-efficient-agents", "title": "Efficient Agents: Building Effective Agents While Reducing Cost", "authors": [ "Ningning Wang", "Xavier Hu", "Pai Liu", "He Zhu", "Yue Hou", "Heyuan Huang", "Shengyu Zhang", "Jian Yang", "Jiaheng Liu", "Ge Zhang", "Changwang Zhang", "Jun Wang", "Yuchen Eleanor Jiang", "Wangchunshu Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.02694", "code_url": "https://github.com/OPPO-PersonalAI/OAgents", "topic_clusters": [ "C3", "C12", "C13", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "cost-of-pass" ], "cost_reduction_reported": "Efficient Agents retains 96.7% of OWL's performance while reducing cost from $0.398 to $0.228, a 28.4% improvement in cost-of-pass on GAIA (abstract)", "benchmarks": [ "GAIA" ], "models_studied": [ "OWL framework (baseline)", "multiple LLM backbones" ], "production_evidence": false, "summary": "Systematic empirical study of efficiency-effectiveness trade-offs in agent systems, varying LLM backbone, framework modules and test-time scaling on GAIA with a cost-of-pass metric. Proposes an Efficient Agents framework sized to task needs.", "key_idea": "Measure agent cost-of-pass and prune unnecessary complexity.", "relevance_score": 5, "relevance_rationale": "Directly on cost of agent systems.", "found_via": "search: efficient agents survey | web search: seed | web search: Efficient Agents reducing cost", "cites_or_cited_by": [ "bai-2026-agents-spend-your-money", "erol-2025-cost-of-pass", "gandhi-2024-budgetmlagent", "lin-2025-supervisoragent", "liu-2025-budget-aware-tool-use", "mialon-2023-gaia", "su-2025-toolorchestra", "zhang-2024-agentprune" ] }, { "id": "wang-2025-mixllm", "title": "MixLLM: Dynamic Routing in Mixed Large Language Models", "authors": [ "Xinyuan Wang", "Yanchi Liu", "Wei Cheng", "Xujiang Zhao", "Zhengzhang Chen", "Wenchao Yu", "Yanjie Fu", "Haifeng Chen" ], "year": 2025, "venue_or_source": "NAACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.18482", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query embedding with tags", "predicted quality", "predicted cost", "latency", "user feedback", "tag-enhanced query embedding" ], "cost_reduction_reported": "97.25% of GPT-4's quality at 24.18% of the cost under the time constraint (abstract)", "benchmarks": [ "routing benchmark with mixed LLMs" ], "models_studied": [ "GPT-4", "mixed open and API LLMs" ], "production_evidence": false, "summary": "MixLLM is a contextual-bandit routing system that augments query embeddings with tags, predicts quality and cost per candidate, and uses a meta-decision maker trading off quality, cost and latency. It supports continual training from feedback and changes in the candidate set.", "key_idea": "Contextual-bandit router with continual learning and dynamic candidate pools.", "relevance_score": 5, "relevance_rationale": "Seed; combines offline prediction with online adaptation.", "found_via": "seed | OpenAlex title search", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "nguyen-2024-metallm", "ong-2024-routellm" ] }, { "id": "wei-2025-barp", "title": "Learning to Route LLMs from Bandit Feedback: One Policy, Many Trade-offs", "authors": [ "Wang Wei", "Tiankai Yang", "Hongjie Chen", "Yue Zhao", "Franck Dernoncourt", "Ryan A. Rossi", "Hoda Eldardiry", "H Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.07429", "code_url": null, "topic_clusters": [ "C1", "C5", "C6" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "prompt features", "user preference vector", "partial (bandit) feedback", "bandit reward" ], "cost_reduction_reported": "Outperforms strong offline routers by at least 12.46% and the largest LLM by at least 2.45% (abstract)", "benchmarks": [ "routing benchmarks (unseen tasks)" ], "models_studied": [ "multiple LLMs" ], "production_evidence": false, "summary": "BaRP trains a router under the same partial-feedback restriction as deployment, where only the chosen model's outcome is observed, framed as a contextual bandit over prompt features and a preference vector. Operators can adjust the performance/cost trade-off at test time without retraining.", "key_idea": "Train routers with bandit feedback and a preference-conditioned policy.", "relevance_score": 5, "relevance_rationale": "Matches the deployment regime where only one model's outcome is logged.", "found_via": "search: learning to route bandit feedback | OpenAlex title: bandit feedback LLM", "cites_or_cited_by": [ "li-2025-llm-bandit", "panda-2025-pilot", "sridhar-2026-correlated-bandits-routing", "wei-2026-llm-experts-limited-feedback", "woisetschlager-2026-slarouter", "zeng-2026-metarouter" ] }, { "id": "wei-2025-moar", "title": "Multi-Objective Agentic Rewrites for Unstructured Data Processing", "authors": [ "Lindsey Linxi Wei", "Shreya Shankar", "Sepanta Zeighami", "Yeounoh Chung", "Fatma Ozcan", "Aditya G. Parameswaran" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.02289", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "accuracy", "cost", "multi-armed bandit over rewrites" ], "cost_reduction_reported": "27% higher accuracy than ABACUS on average; matches ABACUS's best accuracy at 55% of its cost, across six workloads", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MOAR is a multi-objective optimizer for DocETL pipelines that applies over 30 agentic rewrite directives (including cost-targeted ones such as model substitution and decomposition) and searches globally over whole pipelines using a multi-armed bandit to choose what to rewrite.", "key_idea": "Bandit-guided global search over LLM pipeline rewrites for accuracy-cost Pareto.", "relevance_score": 5, "relevance_rationale": "Cost-quality Pareto optimization of multi-operator LLM pipelines from a system with real users.", "found_via": "search: multi-objective agentic workflow optimization cost accuracy Pareto arXiv", "cites_or_cited_by": [ "russo-2025-abacus" ] }, { "id": "wu-2025-optimas", "title": "Optimas: Optimizing Compound AI Systems with Globally Aligned Local Rewards", "authors": [ "Shirley Wu", "Parth Sarthi", "Shiyu Zhao", "Aaron Lee", "Herumb Shandilya", "Adrian Mladenic Grobelnik", "Nurendra Choudhary", "Eddie Huang", "Karthik Subbian", "Linjun Zhang", "Diyi Yang", "James Zou", "Jure Leskovec" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2507.03041", "code_url": "https://optimas.stanford.edu", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "local reward functions aligned with global reward", "preference data" ], "cost_reduction_reported": "Average improvement of 11.92% over strong baselines across five real-world compound systems", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Optimas keeps a learned local reward function per component that is aligned with global system performance, so heterogeneous configurations (prompts, hyperparameters, model routing/selection, LoRA weights) can be updated independently while still improving the whole system.", "key_idea": "Globally aligned local rewards enable independent optimization of heterogeneous component configs.", "relevance_score": 5, "relevance_rationale": "Unified optimizer covering model selection per component alongside prompts and weights.", "found_via": "search: Optimas compound AI globally aligned local rewards", "cites_or_cited_by": [ "chen-2025-llmselector", "opsahlong-2024-mipro" ] }, { "id": "yang-2025-bamas", "title": "BAMAS: Structuring Budget-Aware Multi-Agent Systems", "authors": [ "Liming Yang", "Junyu Luo", "Xuanzhe Liu", "Yiling Lou", "Zhenpeng Chen" ], "year": 2025, "venue_or_source": "AAAI 2026 (Oral)", "source_type": "paper", "url": "https://arxiv.org/abs/2511.21572", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "cost budget", "ILP", "RL-selected topology" ], "cost_reduction_reported": "Comparable performance while reducing cost by up to 86% vs SOTA agent construction methods", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "BAMAS builds budget-aware multi-agent systems by selecting an LLM pool via integer linear programming under a cost budget and choosing a collaboration topology with RL.", "key_idea": "ILP-based LLM provisioning plus RL topology selection under budget.", "relevance_score": 5, "relevance_rationale": "Explicit budget-constrained model selection for MAS.", "found_via": "search: LLM selection for each stage of multi-stage pipeline budget constraint optimization arXiv", "cites_or_cited_by": [] }, { "id": "yarullin-2025-intercom-escalation", "title": "To escalate, or not to escalate, that is the question", "authors": [ "Ramil Yarullin" ], "year": 2025, "venue_or_source": "Fin.ai research blog (Intercom)", "source_type": "blog", "url": "https://fin.ai/research/to-escalate-or-not-to-escalate-that-is-the-question/", "code_url": null, "topic_clusters": [ "C2", "C8", "C14" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "conversation history", "escalation guidelines", "LLM-generated labels" ], "cost_reduction_reported": "Company-reported: custom ModernBERT handles 90% of cases with LLM fallback for 10%; >98% escalation accuracy; latency -0.5s; cost per resolution ~-3%", "benchmarks": [ "production A/B test" ], "models_studied": [ "ModernBERT multi-head encoder", "LLM fallback" ], "production_evidence": true, "summary": "Intercom replaced an LLM escalation decision in Fin with a multi-head ModernBERT trained on 4M production examples with LLM labels, falling back to the LLM on uncertain cases, and reports higher resolution rate in an A/B test.", "key_idea": "Distill a step of the agent into a small classifier with LLM fallback.", "relevance_score": 5, "relevance_rationale": "Production step-level downscaling with cascade fallback and A/B evidence.", "found_via": "seed list: Intercom case study", "cites_or_cited_by": [ "joglekar-2025-intercom-small-llms" ] }, { "id": "ye-2025-x-mas", "title": "X-MAS: Towards Building Multi-Agent Systems with Heterogeneous LLMs", "authors": [ "Rui Ye", "Xiangrui Liu", "Qimin Wu", "Xianghe Pang", "Zhenfei Yin", "Lei Bai", "Siheng Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.16997", "code_url": "https://github.com/MASWorks/X-MAS", "topic_clusters": [ "C3", "C4", "C13" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "per-function model benchmark scores", "per-domain per-function benchmark results" ], "cost_reduction_reported": null, "benchmarks": [ "X-MAS-Bench (21 test sets, 5 domains)", "MATH", "AIME" ], "models_studied": [ "27 LLMs" ], "production_evidence": false, "summary": "X-MAS studies multi-agent systems where each agent function is powered by a different LLM, benchmarking 27 LLMs over 5 domains and 5 MAS functions (1.7M evaluations) to find the best model per domain-function pair. Switching to heterogeneous assignment improves MATH by up to 8.4% and AIME by 47% without changing structure.", "key_idea": "Pick the best LLM per agent function; heterogeneous MAS beats homogeneous.", "relevance_score": 5, "relevance_rationale": "Seed; empirical basis for role-level model assignment.", "found_via": "web search: seed | search: X-MAS heterogeneous LLMs multi-agent systems arXiv 2505", "cites_or_cited_by": [ "fan-2026-science-of-collective-ai", "liu-2025-rcr-router", "wang-2026-oi-mas", "wang-2026-strong-weak-hmas", "xu-2026-oneflow-single-agent-baseline", "yuan-2026-mixed-vendor-clinical-mas", "yue-2025-masrouter" ] }, { "id": "yue-2025-masrouter", "title": "MasRouter: Learning to Route LLMs for Multi-Agent Systems", "authors": [ "Yanwei Yue", "Guibin Zhang", "Boyang Liu", "Guancheng Wan", "Kun Wang", "Dawei Cheng", "Yiyan Qi" ], "year": 2025, "venue_or_source": "ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.11133", "code_url": "https://github.com/yanweiyue/masrouter", "topic_clusters": [ "C1", "C3" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "query embedding", "role embedding", "LLM descriptions" ], "cost_reduction_reported": "Up to 52.07% overhead reduction vs SOTA on HumanEval; 17.21%-28.17% overhead reduction when plugged into mainstream MAS; +1.8%-8.2% on MBPP", "benchmarks": [ "MBPP", "HumanEval" ], "models_studied": [], "production_evidence": false, "summary": "Defines Multi-Agent System Routing: jointly choosing the collaboration mode, the agent roles and the LLM for each agent. A cascaded controller network builds the MAS step by step, improving accuracy and cutting cost versus prior MAS and routers, and can be plugged into existing frameworks.", "key_idea": "Cascaded controller jointly selects collaboration mode, roles and per-agent LLMs.", "relevance_score": 5, "relevance_rationale": "Seed; the core formulation of per-agent model routing in MAS.", "found_via": "web search: seed", "cites_or_cited_by": [ "chen-2025-netgpt-routing-synergy", "chen-2026-lemon-counterfactual-orchestration", "cui-2026-uno-orchestra", "dai-2025-aragog", "dylan-2026-equal-inference-cost-mas", "guo-2026-sciorch", "huang-2026-evolverouter", "kabir-2026-inframind", "li-2026-dynamic-mixed-precision-routing", "li-2026-single-agent-skills-vs-mas", "liu-2026-e3-orch", "liu-2026-ems-majority-then-stopping", "liu-2026-harness-native-routing", "lu-2026-meta-policy-delegation", "nielsen-2025-conductor", "okamoto-2026-topaz-explainable-routing", "rognvaldsson-2026-lean-prover-cost-quality", "singh-2026-planner-as-router", "singh-2026-total-cost-of-agency", "su-2025-daao", "sun-2026-routeguard", "talluri-2026-rgao-topology-budget", "tang-2026-sakana-fugu", "wang-2025-mas2", "wang-2026-amro-s-ant-colony-routing", "wang-2026-boundaryrouter", "wei-2026-step-level-cua-cascade", "wu-2025-rirs-iterative-routing-specialists", "xu-2025-trinity-coordinator", "xu-2026-learning-what-to-skip", "xu-2026-oneflow-single-agent-baseline", "yang-2025-mashost", "yang-2026-fusion-moa-fleet", "ye-2025-x-mas", "yuan-2026-flowbank", "zhou-2026-agent-as-a-router" ] }, { "id": "zhang-2025-agentic-plan-caching", "title": "Agentic Plan Caching: Test-Time Memory for Fast and Cost-Efficient LLM Agents", "authors": [ "Qizheng Zhang", "Michael Wornow", "Gerry Wan", "Kunle Olukotun" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.14852", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "keyword extraction match", "cached plan templates", "small-LM plan adaptation" ], "cost_reduction_reported": "50.31% average cost reduction and 27.28% average latency reduction while maintaining performance across real-world agent applications", "benchmarks": [ "multiple real-world agent applications" ], "models_studied": [], "production_evidence": false, "summary": "Agentic Plan Caching extracts plan templates from completed agent runs, matches new requests to cached plans using keyword extraction, and has a lightweight model adapt the template to the new task, avoiding expensive frontier-model planning. The authors argue response/context caching designed for chatbots fails for agents whose outputs depend on external data.", "key_idea": "Cache and adapt plans, not answers, so a small model can replace frontier planning on repeat task types.", "relevance_score": 5, "relevance_rationale": "Directly swaps expensive frontier planning for a cached template plus small model inside agent pipelines.", "found_via": "Cost-Efficient Serving of LLM Agents via Test-Time Plan Caching", "cites_or_cited_by": [ "basu-2026-w5h2-intent-caching", "chakraborty-2025-gencache", "chillara-2026-semanticalli", "kim-2026-agenticcache", "merchant-2026-temporal-semantic-cache-mcp", "mohamed-2026-workload-aware-caching", "shen-2026-dynbranch", "wang-2024-agent-workflow-memory", "zhang-2025-ace" ] }, { "id": "zhang-2025-evoflow", "title": "EvoFlow: Evolving Diverse Agentic Workflows On The Fly", "authors": [ "Guibin Zhang", "Kaijie Chen", "Guancheng Wan", "Heng Chang", "Hong Cheng", "Kun Wang", "Shuyue Hu", "Lei Bai" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.07373", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "task score", "cost" ], "cost_reduction_reported": "Outperforms handcrafted and automated workflows by 1.23%-29.86%; surpasses o1-preview at 12.4% of its inference cost using weaker open-source models", "benchmarks": [], "models_studied": [ "o1-preview", "open-source LLMs" ], "production_evidence": false, "summary": "EvoFlow uses a niching evolutionary algorithm to evolve a population of heterogeneous, complexity-adaptive workflows that mix different LLMs, rather than one homogeneous complex workflow, optimizing both performance and cost.", "key_idea": "Evolve a diverse population of multi-LLM workflows with cost-aware selection.", "relevance_score": 5, "relevance_rationale": "Explicit LLM heterogeneity and cost objective in automated workflow search.", "found_via": "search: multi-objective agentic workflow optimization cost accuracy Pareto arXiv", "cites_or_cited_by": [ "zhang-2024-aflow" ] }, { "id": "zhang-2025-maas-agentic-supernet", "title": "Multi-agent Architecture Search via Agentic Supernet", "authors": [ "Guibin Zhang", "Luyang Niu", "Junfeng Fang", "Kun Wang", "Lei Bai", "Xiang Wang" ], "year": 2025, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.04180", "code_url": "https://github.com/bingreeky/MaAS", "topic_clusters": [ "C3", "C4", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding", "difficulty", "query features", "inference cost", "task score" ], "cost_reduction_reported": "Requires only 6-45% of the inference costs of existing handcrafted or automated multi-agent systems while surpassing them by 0.54%-11.82%", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "MaAS optimizes an agentic supernet, a distribution over multi-agent architectures, and samples a query-dependent architecture at inference so easy queries get light systems and hard ones get more operators. It cuts inference cost sharply versus handcrafted and automated MAS while improving accuracy and transferring across datasets and backbones.", "key_idea": "Sample a per-query multi-agent architecture from a learned supernet to allocate compute.", "relevance_score": 5, "relevance_rationale": "Seed; query-adaptive MAS cost allocation.", "found_via": "web search: seed | seed list: MaAS", "cites_or_cited_by": [ "bertalanic-2026-small-llm-team-scaling-decomposition", "cai-2025-agentbalance", "chen-2026-lemon-counterfactual-orchestration", "dylan-2026-equal-inference-cost-mas", "el-2025-inefficiencies-meta-agents", "gao-2025-flowreasoner", "gharzeddine-2026-cyclic-subtask-graphs", "jiang-2025-guided-topology-diffusion", "jung-2025-futureweaver", "ke-2026-mas-orchestra", "kim-2025-science-of-scaling-agent-systems", "koh-2026-graft", "leins-2026-when-orchestration-pays-off", "li-2025-adaptive-graph-pruning", "lin-2025-supervisoragent", "liu-2026-caster", "liu-2026-cost-utility-alignment-trajectories", "shi-2026-lamas-latency-aware-mas", "su-2025-daao", "wang-2025-mas2", "wang-2026-agentconductor", "wang-2026-mcpp-budget-deadline-allocation", "wang-2026-oi-mas", "wang-2026-scale-query-level-workflows", "xu-2026-hyevo", "yang-2025-mashost", "yang-2026-harness-adaptation-slm", "yuan-2026-flowbank", "zhang-2024-aflow", "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2026-evoroute", "zhang-2026-rethinking-mas-efficiency-evaluation" ] }, { "id": "zhang-2025-router-r1", "title": "Router-R1: Teaching LLMs Multi-Round Routing and Aggregation via Reinforcement Learning", "authors": [ "Haozhen Zhang", "Tao Feng", "Jiaxuan You" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.09033", "code_url": "https://github.com/ulab-uiuc/Router-R1", "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "router LLM reasoning", "model descriptors (price, latency, example performance)", "cost reward" ], "cost_reduction_reported": "Superior performance with cost management across 7 QA benchmarks (no single headline ratio in abstract)", "benchmarks": [ "seven general and multi-hop QA benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Router-R1 makes the router itself an LLM trained with RL to interleave internal reasoning with calls to other models, aggregating their responses over multiple rounds. The rule-based reward includes format, outcome and a cost term, and conditioning on simple model descriptors lets it generalize to unseen models.", "key_idea": "RL-trained LLM router that performs multi-round routing and aggregation with a cost reward.", "relevance_score": 5, "relevance_rationale": "Seed; moves routing from one-shot classification to sequential decision-making, close to agentic routing.", "found_via": "seed | web search: Router-R1", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "bhargava-2024-ope-logged-human-feedback", "title": "Off-Policy Evaluation from Logged Human Feedback", "authors": [ "Aniruddha Bhargava", "Lalit Jain", "Branislav Kveton", "Ge Liu", "Subhojyoti Mukherjee" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.10030", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "logged human feedback on responses" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formalizes evaluating a new model using human feedback logged on another model's responses, proposing model-based and model-free estimators that estimate absolute policy values, rank policies, and can be optimized.", "key_idea": "OPE of LLM policies from logged human feedback.", "relevance_score": 5, "relevance_rationale": "Exactly the tool for evaluating a candidate routing/config policy on existing labeled logs.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "dudik-2011-doubly-robust", "huang-2025-pluralistic-ope", "wu-2024-ocean", "xiang-2026-otrope" ] }, { "id": "boisvert-2024-workarena-plus-plus", "title": "WorkArena++: Towards Compositional Planning and Reasoning-based Common Knowledge Work Tasks", "authors": [ "Léo Boisvert", "Megh Thakkar", "Maxime Gasse", "Massimo Caccia", "Thibault Le Sellier De Chezelles", "Quentin Cappart", "Nicolas Chapados", "Alexandre Lacoste", "Alexandre Drouin" ], "year": 2024, "venue_or_source": "NeurIPS 2024 Datasets and Benchmarks", "source_type": "paper", "url": "https://arxiv.org/abs/2407.05291", "code_url": "https://github.com/ServiceNow/WorkArena", "topic_clusters": [ "C8", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "task validators" ], "cost_reduction_reported": null, "benchmarks": [ "WorkArena++ (682 tasks)" ], "models_studied": [ "LLMs and VLMs", "human workers" ], "production_evidence": false, "summary": "Adds 682 compositional workflows on ServiceNow that need planning, arithmetic and logical reasoning, retrieval and context understanding, and compares agents with human workers. It also provides a mechanism to generate large numbers of ground-truth observation/action traces for fine-tuning.", "key_idea": "Harder compositional enterprise workflows plus trace generation for training.", "relevance_score": 5, "relevance_rationale": "Seed; ServiceNow enterprise workflows and trace data usable for distillation to smaller agents.", "found_via": "seed: WorkArena++", "cites_or_cited_by": [ "drouin-2024-workarena" ] }, { "id": "chen-2024-optima", "title": "Optima: Optimizing Effectiveness and Efficiency for LLM-Based Multi-Agent System", "authors": [ "Weize Chen", "Jiarui Yuan", "Chen Qian", "Cheng Yang", "Zhiyuan Liu", "Maosong Sun" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.08115", "code_url": "https://github.com/thunlp/Optima", "topic_clusters": [ "C3", "C4", "C8", "C12" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "reward: task score + token count + readability", "task reward", "token efficiency", "readability" ], "cost_reduction_reported": "Up to 2.8x performance gain with less than 10% of tokens on information-heavy tasks vs vanilla MAS (Llama 3 8B)", "benchmarks": [ "information-asymmetric QA", "complex reasoning tasks" ], "models_studied": [ "Llama 3 8B" ], "production_evidence": false, "summary": "Optima trains the agents of a multi-agent system with an iterative generate-rank-select-train loop (SFT, DPO and hybrids) whose reward balances task success, token efficiency and readability, using MCTS-style data generation. The trained MAS communicates far more tersely and outperforms both single-agent and vanilla MAS baselines.", "key_idea": "Train MAS agents with a reward that penalizes tokens to get efficient communication.", "relevance_score": 5, "relevance_rationale": "Seed; training-based MAS cost reduction.", "found_via": "web search: seed | snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [ "cemri-2025-mast-why-mas-fail", "dang-2025-puppeteer-evolving-orchestration", "ding-2026-calibrate-then-act", "fan-2026-science-of-collective-ai", "huang-2026-evolverouter", "li-2026-single-agent-skills-vs-mas" ] }, { "id": "dai-2024-c2mab-v", "title": "Cost-Effective Online Multi-LLM Selection with Versatile Reward Models", "authors": [ "Xiangxiang Dai", "Jin Li", "Xutong Liu", "Anqi Yu", "John C. S. Lui" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.16587", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "combinatorial bandit rewards", "cost", "reward models per task type", "task reward", "LLM price", "combinatorial selection" ], "cost_reduction_reported": null, "benchmarks": [ "three application scenarios with nine LLMs" ], "models_studied": [ "nine LLMs" ], "production_evidence": false, "summary": "C2MAB-V is a cost-aware combinatorial multi-armed bandit for selecting multiple LLMs per query across collaborative task types with different reward models. It uses online feedback and confidence bounds to balance exploration, exploitation and cost.", "key_idea": "Combinatorial bandit selection of LLM subsets under cost.", "relevance_score": 5, "relevance_rationale": "Online multi-LLM selection.", "found_via": "snowball: cited-by ding-2024-hybrid-llm | ti:C2MAB-V / OpenAlex title search", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "huang-2026-constrained-bandits-time-varying", "li-2025-stageroute", "liu-2025-off-cmab", "nguyen-2024-metallm", "poon-2025-online-multi-llm-context-evolution", "xia-2024-which-llm-to-play" ] }, { "id": "dekoninck-2024-cascade-routing", "title": "A Unified Approach to Routing and Cascading for LLMs", "authors": [ "Jasper Dekoninck", "Maximilian Baader", "Martin Vechev" ], "year": 2024, "venue_or_source": "ICML (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2410.10347", "code_url": null, "topic_clusters": [ "C1", "C2", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "quality estimator", "cost estimator", "quality estimates", "cost estimates" ], "cost_reduction_reported": "Cascade routing consistently outperforms routing or cascading alone by a large margin (no single number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Derives an optimal cascading strategy, proves optimality of an existing routing strategy, and unifies them as cascade routing, which can route and then escalate. Analysis identifies quality-estimator accuracy as the key factor for success of any model selection scheme.", "key_idea": "Cascade routing: provably optimal unification of routing and cascading.", "relevance_score": 5, "relevance_rationale": "Theory and practice; mentions agentic systems and finds quality estimation is the bottleneck.", "found_via": "snowball: cited-by ong-2024-routellm | arXiv title search: cascade LLM", "cites_or_cited_by": [ "chen-2026-resample-or-reroute", "chen-2026-routing-oracle-stochastic", "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "drouin-2024-workarena", "title": "WorkArena: How Capable Are Web Agents at Solving Common Knowledge Work Tasks?", "authors": [ "Alexandre Drouin", "Maxime Gasse", "Massimo Caccia", "Issam H. Laradji", "Manuel Del Verme", "Tom Marty", "Léo Boisvert", "Megh Thakkar", "Quentin Cappart", "David Vazquez", "Nicolas Chapados", "Alexandre Lacoste" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2403.07718", "code_url": "https://github.com/ServiceNow/WorkArena", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "task validators" ], "cost_reduction_reported": null, "benchmarks": [ "WorkArena (33 tasks, ~19.9K instances)" ], "models_studied": [], "production_evidence": false, "summary": "A remote-hosted benchmark of 33 knowledge-work tasks on the ServiceNow platform, with certifiable oracles and validators, plus the BrowserGym environment for designing and evaluating web agents. Agents show promise but a large gap to full automation remains, especially for open models.", "key_idea": "Enterprise ITSM-platform web-agent benchmark with verifiable tasks.", "relevance_score": 5, "relevance_rationale": "Seed; directly the enterprise/ITSM workflow setting of interest.", "found_via": "seed: WorkArena", "cites_or_cited_by": [ "boisvert-2024-workarena-plus-plus", "chezelles-2024-browsergym" ] }, { "id": "gandhi-2024-budgetmlagent", "title": "BudgetMLAgent: A Cost-Effective LLM Multi-Agent system for Automating Machine Learning Tasks", "authors": [ "Shubham Gandhi", "Manasi Patwardhan", "Lovekesh Vig", "Gautam Shroff" ], "year": 2024, "venue_or_source": "AIMLSystems 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2411.07464", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "cascade failure", "ask-the-expert calls" ], "cost_reduction_reported": "94.2% cost reduction ($0.931 to $0.054 per run) with 32.95% vs 22.72% success rate compared to GPT-4 single agent on MLAgentBench", "benchmarks": [ "MLAgentBench" ], "models_studied": [ "Gemini-Pro", "GPT-4", "Mixtral", "CodeLlama" ], "production_evidence": false, "summary": "BudgetMLAgent builds a multi-agent system around free or cheap base models, using profiling, retrieval of past observations, LLM cascades and occasional ask-the-expert calls to GPT-4 for planning. It beats a GPT-4 single agent on MLAgentBench at a small fraction of the cost.", "key_idea": "Cheap base model with cascades and rare expert calls for planning.", "relevance_score": 5, "relevance_rationale": "Concrete step-level escalation with large cost savings.", "found_via": "web search: single vs multi-agent cost", "cites_or_cited_by": [ "guo-2026-eet-early-termination", "wang-2025-efficient-agents" ] }, { "id": "hu-2024-routerbench", "title": "RouterBench: A Benchmark for Multi-LLM Routing System", "authors": [ "Qitian Jason Hu", "Jacob Bieker", "Xiuyu Li", "Nan Jiang", "Benjamin Keigwin", "Gaurav Ranganath", "Kurt Keutzer", "Shriyash Kaustubh Upadhyay" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "benchmark", "url": "https://arxiv.org/abs/2403.12031", "code_url": "https://github.com/withmartian/routerbench", "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "precomputed per-model outcomes", "cost", "per-query model outcomes" ], "cost_reduction_reported": null, "benchmarks": [ "RouterBench (405k inference outcomes)", "RouterBench (405k inference outcomes over standard benchmarks)" ], "models_studied": [], "production_evidence": false, "summary": "RouterBench provides over 405k precomputed inference outcomes across a set of representative LLMs and tasks, so routers can be evaluated offline without calling models. It also gives a theoretical framing of routing on the cost-quality plane and compares routing approaches.", "key_idea": "Standard offline benchmark of per-model outcomes and costs for evaluating routers.", "relevance_score": 5, "relevance_rationale": "De facto evaluation substrate for query routers (by Martian).", "found_via": "seed | seed: RouterBench", "cites_or_cited_by": [ "huang-2025-routereval", "ibm-2024-llm-routers-blog", "kumar-2026-task-session-routing-eval", "li-2026-llmrouterbench", "lu-2025-routerarena", "martian-docs", "somerstep-2025-carrot", "yang-2026-twinrouterbench" ] }, { "id": "jung-2024-trust-or-escalate", "title": "Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement", "authors": [ "Jaehun Jung", "Faeze Brahman", "Yejin Choi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.18370", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "judge confidence (Simulated Annotators)", "calibrated thresholds", "judge confidence", "simulated annotators", "conformal-style guarantee" ], "cost_reduction_reported": "On a Chatbot Arena subset where GPT-4 almost never reaches 80% human agreement, cascaded selective evaluation with models like Mistral-7B guarantees over 80% agreement at almost 80% coverage", "benchmarks": [ "Chatbot Arena subset", "Chatbot Arena" ], "models_studied": [ "GPT-4", "Mistral-7B" ], "production_evidence": false, "summary": "Selective LLM-as-judge evaluation that trusts a judge only when its calibrated confidence is high, providing a provable human-agreement guarantee. Cascaded Selective Evaluation starts with cheap judges and escalates to stronger ones.", "key_idea": "Cascade of judges with provable human-agreement guarantees.", "relevance_score": 5, "relevance_rationale": "Cheap, guaranteed quality signals for configuration search without ground truth.", "found_via": "arXiv title search: escalation LLM | keyword: judge escalation", "cites_or_cited_by": [] }, { "id": "kapoor-2024-ai-agents-that-matter", "title": "AI Agents That Matter", "authors": [ "Sayash Kapoor", "Benedikt Stroebl", "Zachary S. Siegel", "Nitya Nadgir", "Arvind Narayanan" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.01502", "code_url": null, "topic_clusters": [ "C2", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "cost", "accuracy" ], "cost_reduction_reported": "Simple baselines match complex SOTA agents at far lower cost; the paper designs a joint cost-accuracy optimization that greatly reduces cost while maintaining accuracy (search summary cites 50x cost spread for similar accuracy)", "benchmarks": [ "agent benchmarks incl. HumanEval", "HumanEval", "other agent benchmarks (see paper)" ], "models_studied": [], "production_evidence": false, "summary": "Critiques agent benchmarking: accuracy-only leaderboards produce needlessly costly agents, and simple baselines (retry, warming, escalation) sit on the cost-accuracy Pareto frontier. Also covers holdout sets, overfitting and reproducibility, and argues for jointly optimizing cost and accuracy.", "key_idea": "Evaluate agents on the cost-accuracy Pareto frontier, not accuracy alone.", "relevance_score": 5, "relevance_rationale": "Seed; foundational for cost-aware agent evaluation and simple escalation baselines.", "found_via": "web search: seed | web search: AI Agents That Matter cost accuracy Pareto", "cites_or_cited_by": [ "dennis-2026-in-context-obsoletes-orchestration", "erol-2025-cost-of-pass", "fan-2026-science-of-collective-ai", "kapoor-2025-hal-holistic-agent-leaderboard", "ke-2026-mas-orchestra", "kim-2025-science-of-scaling-agent-systems", "li-2024-more-agents-is-all-you-need", "peng-2026-inference-economics-coding-agents", "smekal-2026-task-spec-token-spend", "yu-2026-when-mas-help-information-bottleneck" ] }, { "id": "nguyen-2024-metallm", "title": "MetaLLM: A High-performant and Cost-efficient Dynamic Framework for Wrapping LLMs", "authors": [ "Quang H. Nguyen", "Thinh Dao", "Duy C. Hoang", "Juliette Decugis", "Saurav Manchanda", "Nitesh V. Chawla", "Khoa D. Doan", "Duy Cao Hoang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.10834", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query embedding", "bandit reward (accuracy minus cost)", "task reward", "API cost" ], "cost_reduction_reported": null, "benchmarks": [ "classification and multiple-choice QA tasks" ], "models_studied": [ "OpenAI models", "Together AI models", "OpenAI API models", "open-source LLMs" ], "production_evidence": false, "summary": "MetaLLM frames per-query LLM selection for classification and multiple-choice QA as a multi-armed bandit balancing accuracy and cost. Experiments on OpenAI, Together AI and open-source models show improved accuracy-cost trade-offs.", "key_idea": "Bandit formulation of cost-aware LLM wrapping.", "relevance_score": 5, "relevance_rationale": "Online-learning baseline for routing.", "found_via": "snowball: cited-by ong-2024-routellm | ti:MetaLLM (arXiv/OpenAlex)", "cites_or_cited_by": [ "atalar-2025-sequential-bandits", "dai-2024-c2mab-v", "ding-2024-hybrid-llm", "li-2025-llm-bandit", "ong-2024-routellm", "panda-2025-pilot", "wang-2025-mixllm" ] }, { "id": "nie-2024-online-cascade-learning", "title": "Online Cascade Learning for Efficient Inference over Streams", "authors": [ "Lunyiu Nie", "Zhimin Ding", "Erdong Hu", "Christopher Jermaine", "Swarat Chaudhuri" ], "year": 2024, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2402.04513", "code_url": null, "topic_clusters": [ "C2", "C5", "C8" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "deferral policy", "LLM demonstrations", "deferral policy confidence" ], "cost_reduction_reported": "Parallels LLM accuracy while cutting inference costs by as much as 90% across four benchmarks", "benchmarks": [ "four stream benchmarks", "four stream-processing text benchmarks" ], "models_studied": [ "logistic regression", "small LMs", "LLM (GPT-class)" ], "production_evidence": false, "summary": "Learns a cascade online from a stream, from logistic regression up to an LLM, treating it as imitation learning where small models imitate LLM outputs over time. Provides a no-regret algorithm and robustness to distribution shift.", "key_idea": "Cascade learned online by imitating LLM demonstrations.", "relevance_score": 5, "relevance_rationale": "Learning cheap substitutes from live traffic.", "found_via": "snowball: related work of FrugalGPT (2305.05176) | ti:\"Online Cascade Learning\"", "cites_or_cited_by": [ "jitkrittum-2025-unirouter", "ramirez-2023-cache-and-distil", "ren-2025-agserve", "stogiannidis-2023-cache-me-if-you-can", "wu-2025-inter-cascade" ] }, { "id": "saadfalcon-2024-archon", "title": "Archon: An Architecture Search Framework for Inference-Time Techniques", "authors": [ "Jon Saad-Falcon", "Adrian Gamarra Lafuente", "Shlok Natarajan", "Nahum Maru", "Hristo Todorov", "Etash Guha", "E. Kelly Buchanan", "Mayee Chen", "Neel Guha", "Christopher Ré", "Azalia Mirhoseini" ], "year": 2024, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2409.15254", "code_url": null, "topic_clusters": [ "C4", "C9" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "benchmark score", "token budget" ], "cost_reduction_reported": "Archon architectures advance the Pareto frontier of accuracy vs. maximum token budget; outperform o1, GPT-4o and Claude 3.5 Sonnet by an average of 15.1%", "benchmarks": [], "models_studied": [ "o1", "GPT-4o", "Claude 3.5 Sonnet" ], "production_evidence": false, "summary": "Archon is a framework for selecting and combining inference-time techniques (ensembling, sampling, ranking, fusion, critique, verification, unit tests) and LLMs into layered architectures. It uses Bayesian-optimization-based inference-time architecture search under a compute budget.", "key_idea": "Inference-time architecture search (ITAS) over layered combinations of LLMs and inference techniques.", "relevance_score": 5, "relevance_rationale": "Jointly searches model choice and technique composition under a token budget; directly relevant to config optimization with cost.", "found_via": "seed: Archon", "cites_or_cited_by": [] }, { "id": "sun-2024-adaswitch", "title": "AdaSwitch: Adaptive Switching between Small and Large Agents for Effective Cloud-Local Collaborative Learning", "authors": [ "Hao Sun", "Jiayi Wu", "Hengyi Cai", "Xiaochi Wei", "Yue Feng", "Bo Wang", "Shuaiqiang Wang", "Yan Zhang", "Dawei Yin" ], "year": 2024, "venue_or_source": "EMNLP 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2410.13181", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "self-detected error by local agent" ], "cost_reduction_reported": "Competitive with cloud agent on some tasks while using much less computational overhead", "benchmarks": [ "7 benchmarks: mathematical reasoning, complex QA" ], "models_studied": [ "various local and cloud LLMs" ], "production_evidence": false, "summary": "A small local agent handles simple reasoning steps and, when it introspectively detects an error, asks a larger cloud agent to take over that step. The step-level switching improves the local agent substantially and sometimes approaches the cloud agent at much lower cost.", "key_idea": "Local small agent escalates individual reasoning steps to a cloud LLM when it detects its own errors.", "relevance_score": 5, "relevance_rationale": "Step-level escalation inside a trajectory.", "found_via": "web search: big-little collaboration", "cites_or_cited_by": [] }, { "id": "yao-2024-tau-bench", "title": "τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", "authors": [ "Shunyu Yao", "Noah Shinn", "Pedram Razavi", "Karthik Narasimhan" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.12045", "code_url": "https://github.com/sierra-research/tau-bench", "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "database end-state comparison" ], "cost_reduction_reported": null, "benchmarks": [ "τ-retail", "τ-airline" ], "models_studied": [ "gpt-4o", "other function-calling agents" ], "production_evidence": false, "summary": "Benchmark of multi-turn conversations between an LLM-simulated user and a tool-using agent bound by domain policies, graded by comparing the final database state to an annotated goal. Introduces the pass^k metric for consistency across repeated trials; gpt-4o succeeds on under 50% of tasks and pass^8 falls below 25% in retail.", "key_idea": "pass^k exposes reliability gaps that pass@1 hides in customer-service agents.", "relevance_score": 5, "relevance_rationale": "Seed; the standard enterprise-style agent benchmark and origin of pass^k reliability metric.", "found_via": "seed: tau-bench", "cites_or_cited_by": [ "barres-2025-tau2-bench", "mehta-2025-clear", "zhu-2025-agentic-benchmark-checklist" ] }, { "id": "zhang-2024-aflow", "title": "AFlow: Automating Agentic Workflow Generation", "authors": [ "Jiayi Zhang", "Jinyu Xiang", "Zhaoyang Yu", "Fengwei Teng", "Xionghui Chen", "Jiaqi Chen", "Mingchen Zhuge", "Xin Cheng", "Sirui Hong", "Jinlin Wang", "Bingnan Zheng", "Bang Liu", "Yuyu Luo", "Chenglin Wu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.10762", "code_url": "https://github.com/FoundationAgents/AFlow", "topic_clusters": [ "C3", "C4", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "execution feedback", "benchmark score" ], "cost_reduction_reported": "Smaller models outperform GPT-4o on specific tasks at 4.55% of its inference cost in dollars; 5.7% average improvement over SOTA baselines across six benchmarks", "benchmarks": [ "six benchmark datasets" ], "models_studied": [ "GPT-4o (baseline)", "GPT-4o" ], "production_evidence": false, "summary": "AFlow searches code-represented agentic workflows with MCTS, refining them from execution feedback. Beyond a 5.7% average gain, optimized workflows let smaller models beat GPT-4o on some tasks at under 5% of its cost.", "key_idea": "Workflow search lets small models replace large ones.", "relevance_score": 5, "relevance_rationale": "Architecture search as a downscaling lever.", "found_via": "snowball: cited-by zhang-2025-maas-agentic-supernet | seed: AFlow", "cites_or_cited_by": [ "chen-2026-agentslimming", "du-2026-swift", "gao-2025-flowreasoner", "nie-2025-w4s", "wang-2025-evoagentx", "wang-2025-scoreflow", "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2025-evoflow", "zhang-2025-maas-agentic-supernet", "zhao-2025-a2flow", "zhou-2025-mass" ] }, { "id": "zhang-2024-agentprune", "title": "Cut the Crap: An Economical Communication Pipeline for LLM-based Multi-Agent Systems", "authors": [ "Guibin Zhang", "Yanwei Yue", "Zhixun Li", "Sukwon Yun", "Guancheng Wan", "Kun Wang", "Dawei Cheng", "Jeffrey Xu Yu", "Tianlong Chen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.02506", "code_url": null, "topic_clusters": [ "C3", "C10", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "message-graph importance" ], "cost_reduction_reported": "Comparable results to SOTA topologies at $5.6 vs $43.7; 28.1%-72.8% token reduction when plugged into existing MAS", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "AgentPrune identifies communication redundancy in multi-agent pipelines and does one-shot pruning on the spatial-temporal message-passing graph to keep only useful messages. It reaches state-of-the-art-level accuracy at a fraction of token cost and improves robustness to adversarial agents.", "key_idea": "One-shot pruning of the inter-agent message graph removes redundant communication.", "relevance_score": 5, "relevance_rationale": "Seed; direct cost reduction inside MAS.", "found_via": "web search: seed", "cites_or_cited_by": [ "chen-2026-agentslimming", "chen-2026-lemon-counterfactual-orchestration", "fan-2025-agent-gspo", "fan-2026-science-of-collective-ai", "feng-2025-heterogeneous-swarms", "hong-2026-dats-difficulty-aware-topology", "jiang-2025-guided-topology-diffusion", "ke-2025-mas-zero", "li-2025-adaptive-graph-pruning", "mieczkowski-2026-latte-adaptive-task-graphs", "rainone-2026-hybrid-cloud-device-mas-lessons", "shi-2026-lamas-latency-aware-mas", "wang-2025-efficient-agents", "wang-2025-efficient-moa-serving", "wang-2025-mas2", "wang-2026-agentconductor", "wang-2026-random-routing-sparse-debate", "yao-2026-hieramas", "ye-2026-conditional-progressive-pruning-debate", "zhang-2024-aflow", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "zhang-2024-g-designer", "title": "G-Designer: Architecting Multi-agent Communication Topologies via Graph Neural Networks", "authors": [ "Guibin Zhang", "Yanwei Yue", "Xiangguo Sun", "Guancheng Wan", "Miao Yu", "Junfeng Fang", "Kun Wang", "Tianlong Chen", "Dawei Cheng" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.11782", "code_url": null, "topic_clusters": [ "C3", "C4", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "task embedding", "agent profiles", "graph auto-encoder" ], "cost_reduction_reported": "Reduces token consumption by up to 95.33% on HumanEval", "benchmarks": [ "MMLU", "HumanEval", "six benchmarks total" ], "models_studied": [], "production_evidence": false, "summary": "G-Designer uses a variational graph auto-encoder over agent nodes and a task node to generate a task-adaptive communication topology for each task. It reaches strong accuracy while cutting token consumption on easy tasks and stays robust to adversarial agents.", "key_idea": "Generate a per-task communication topology with a graph auto-encoder.", "relevance_score": 5, "relevance_rationale": "Seed; query-adaptive topology reduces MAS cost.", "found_via": "web search: seed | search: AgentPrune G-Designer multi-agent topology token cost arXiv", "cites_or_cited_by": [ "cai-2025-agentbalance", "chen-2026-lemon-counterfactual-orchestration", "dylan-2026-equal-inference-cost-mas", "feng-2025-heterogeneous-swarms", "gao-2025-flowreasoner", "gharzeddine-2026-cyclic-subtask-graphs", "hong-2026-dats-difficulty-aware-topology", "huang-2026-evolverouter", "jiang-2025-guided-topology-diffusion", "ke-2025-mas-zero", "li-2025-adaptive-graph-pruning", "liu-2026-e3-orch", "shi-2026-lamas-latency-aware-mas", "wang-2025-mas2", "wang-2026-random-routing-sparse-debate", "wu-2026-card-conditional-topology", "yang-2025-mashost", "ye-2026-conditional-progressive-pruning-debate", "zhang-2024-aflow", "zhang-2024-agentprune", "zhang-2025-maas-agentic-supernet" ] }, { "id": "abdollahi-2026-hdlforge", "title": "HDLFORGE: A Two-Stage Multi-Agent Framework for Efficient Verilog Code Generation with Adaptive Model Escalation", "authors": [ "Armin Abdollahi", "Saeid Shokoufa", "Negin Ashrafi", "Mehdi Kamal", "Massoud Pedram" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.04646", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "compile/lint/smoke-test diagnostics", "calibrated score" ], "cost_reduction_reported": "91.2% and 91.8% Pass@1 on VerilogEval Human/V2 with roughly 50% lower median latency", "benchmarks": [ "VerilogEval Human", "VerilogEval V2", "RTLLM" ], "models_studied": [ "Qwen" ], "production_evidence": false, "summary": "HDLFORGE runs a medium-sized coder by default and escalates to an ultra-large model only when cheap diagnostics (compilation, lint, smoke tests) give a low calibrated score. A formal agent turns bounded-model-checking traces into micro-tests.", "key_idea": "Diagnostics-gated escalation from medium to ultra-large models in a multi-agent coding pipeline.", "relevance_score": 4, "relevance_rationale": "Agentic escalation using execution signals; domain is Verilog.", "found_via": "arXiv query ti:\"agent\" AND ti:\"escalate\"", "cites_or_cited_by": [] }, { "id": "ali-2026-harness-effect", "title": "The Harness Effect: How Orchestration Design Sets the Token Economics of Enterprise Agentic AI", "authors": [ "Muayad Sayed Ali", "Aliaksandra Novik", "Anji Boddupally", "Artem Yavorskyi", "Chris Nickerson", "Daniel Rica", "Emily DuGranrut", "Felix Leung", "Garrett Prince", "Grace Barnett", "Heath Robinson", "Hosain Al Ahmad", "Jesse Resnick", "Juan Carlos Farah", "Jyothi Swaroop Meruga", "Leonid Kuznetsov", "Luke Gorham", "Marie Schmoll", "Michael Paciullo", "Saumya Das", "Sharath Sheripally", "Tommy Griscom", "Mykyta Osadchyi", "Neha Mantri", "Nick Westrum", "Olivia Benowitz", "Parikshith Kulkarni", "Radik Chernyshov", "Rakshith Vasudev", "Rohith Nadimpally", "Vikas Gangadevi", "Waseem AlShikh" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.06906", "code_url": null, "topic_clusters": [ "C3", "C10", "C12", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Harness cuts blended cost per task 41% ($0.21->$0.12), wall-clock 44%, tokens 38% at quality parity (0.78->0.81) across six models on 22 tasks; quality per dollar +82%", "benchmarks": [ "22 locked enterprise evaluation tasks" ], "models_studied": [ "Claude Sonnet 4.6", "Gemini 3.1", "Gemini Flash 3.5", "Qwen 3.6", "GLM 5.1", "Palmyra X6" ], "production_evidence": false, "summary": "A controlled swap that keeps six models fixed and changes only the orchestration layer, comparing a conventional production loop with Writer's agent harness. On this workload the harness moved cost per task more than the whole spread of model choices, and quality gains scaled with model strength.", "key_idea": "Orchestration harness design is a larger cost lever than model choice.", "relevance_score": 4, "relevance_rationale": "Enterprise evidence that bounds the value of model routing relative to harness design.", "found_via": "arXiv query ti:enterprise AND ti:agent AND abs:\"model\" AND abs:\"cost\"", "cites_or_cited_by": [] }, { "id": "amp-2026-subagent-model-assignment", "title": "A Faster Librarian (and related Amp notes on per-role subagent models)", "authors": [ "Amp (Sourcegraph)" ], "year": 2026, "venue_or_source": "Amp news (ampcode.com)", "source_type": "blog", "url": "https://ampcode.com/news/a-faster-librarian", "code_url": null, "topic_clusters": [ "C3", "C9", "C14" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "role" ], "cost_reduction_reported": "Librarian subagent moved from Sonnet 4.6 (medium) to GPT-5.5 (no reasoning): mean latency 237s->81s, F1 0.47->0.48, average cost $1.21->$0.69 (43% cheaper) on internal eval; search subagent on Gemini 3 Flash vs Haiku 4.5: 3x faster at same quality (~8 vs ~2.5 parallel calls, ~3 vs ~9 turns); summarisation/compaction model switch 4-6x faster and ~30x cheaper (vendor-reported)", "benchmarks": [ "Amp internal subagent evals" ], "models_studied": [ "Claude Sonnet 4.6", "GPT-5.5", "Gemini 3 Flash", "Claude Haiku 4.5", "GLM-5.2", "GPT-5.6 Sol", "GPT-5.6 Terra", "Claude Fable" ], "production_evidence": true, "summary": "A series of short Amp product notes documents a production coding agent that assigns different models per role: main agent, oracle, thread reader, code reviewer, codebase search subagent, GitHub 'Librarian' subagent and summariser, each swapped independently as cheaper models improve. The Librarian note gives an eval breakdown of the gains from a non-reasoning model plus websocket transport; 'A Dial for You' (https://ampcode.com/news/a-dial-for-you) lists the role-to-model mapping per mode and reports that swapping the default model overnight drew no complaints.", "key_idea": "Role-specific subagent model choices in a shipping agent, tuned with small internal evals.", "relevance_score": 4, "relevance_rationale": "Production evidence of subagent-level downscaling with per-role numbers, albeit brief.", "found_via": "direct: ampcode.com/chronicle", "cites_or_cited_by": [ "furman-2026-replit-free-the-models" ] }, { "id": "anthropic-task-budgets-docs", "title": "Task budgets (Claude Platform docs, beta)", "authors": [ "Anthropic" ], "year": 2026, "venue_or_source": "Claude Platform documentation", "source_type": "docs", "url": "https://platform.claude.com/docs/en/build-with-claude/task-budgets", "code_url": null, "topic_clusters": [ "C3", "C9", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "remaining token budget" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude Opus 5.5" ], "production_evidence": false, "summary": "A beta feature giving Claude an advisory token budget for an entire agentic loop (thinking, tool calls, tool results, output); the model sees a running countdown and prioritises work, wrapping up gracefully as the budget is consumed. Positioned as complementary to effort: effort sets per-step thoroughness, task budgets cap total work across the trajectory.", "key_idea": "Expose a trajectory-level token budget to the model so it self-regulates spend on long-horizon tasks.", "relevance_score": 4, "relevance_rationale": "Trajectory-level budget control inside agents is close to the premise's step-level cost lever.", "found_via": "direct: platform.claude.com docs", "cites_or_cited_by": [ "anthropic-effort-docs" ] }, { "id": "apparaju-2026-compute-allocation-retrieval-agents", "title": "Compute Allocation for Reasoning-Intensive Retrieval Agents", "authors": [ "Sreeja Apparaju", "Nilesh Gupta" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.14635", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "stage-wise capacity sweeps" ], "cost_reduction_reported": "Re-ranking gains +7.5 NDCG@10 from stronger models while query expansion gains only +1.1 from weak to strong (BRIGHT, Gemini 2.5 family)", "benchmarks": [ "BRIGHT" ], "models_studied": [ "Gemini 2.5 family" ], "production_evidence": false, "summary": "Studies how to spend compute across the stages of reasoning-intensive retrieval pipelines for agents by varying model size, thinking and re-ranking depth per stage. Strong models matter for re-ranking but not query expansion, and thinking helps little at either stage.", "key_idea": "Stage-specific model sizing: spend capacity on re-ranking, not query expansion.", "relevance_score": 4, "relevance_rationale": "Controlled per-stage capacity allocation evidence for agent pipelines.", "found_via": "arXiv query ti:\"compute allocation\" AND abs:agent", "cites_or_cited_by": [] }, { "id": "bae-2026-contextual-queueing-bandits", "title": "Learning to Route and Schedule LLMs from User Retrials via Contextual Queueing Bandits", "authors": [ "Seoungbin Bae", "Junyoung Son", "Dabeen Lee" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.02061", "code_url": null, "topic_clusters": [ "C1", "C5", "C11" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query embedding", "user retrials (implicit feedback)", "queue state" ], "cost_reduction_reported": null, "benchmarks": [ "SPROUT", "EmbedLLM", "RouterBench", "WildChat-1M" ], "models_studied": [], "production_evidence": false, "summary": "Formulates joint LLM routing and request scheduling as a contextual queueing bandit in which an unsatisfied user retrying a query serves as implicit negative feedback, avoiding explicit ratings. A Thompson-sampling algorithm with decaying forced exploration achieves sublinear routing regret while keeping queues stable, and is evaluated on offline routing datasets and real WildChat conversation logs.", "key_idea": "Treat user retrials as free implicit feedback to learn query-to-LLM routing online under queueing dynamics.", "relevance_score": 4, "relevance_rationale": "Online routing learned from implicit production-style feedback (retrials); directly fills the implicit-feedback gap for routers.", "found_via": "query: abs:\"bandit\" AND abs:\"LLM\" AND abs:\"routing\"", "cites_or_cited_by": [] }, { "id": "bala-2026-set-valued-agent-routing", "title": "Multi-Agent Routing as Set-Valued Prediction: A WildChat Benchmark and Cost-Aware Evaluation", "authors": [ "Ananto Nayan Bala", "Faisal Muhammad Shah" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.28925", "code_url": null, "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "prompt text", "agent cost tiers" ], "cost_reduction_reported": null, "benchmarks": [ "WildChat-derived 3,000-prompt, 12-agent benchmark", "WildChat-derived 3,000 prompts, 12-agent catalog" ], "models_studied": [ "encoders", "linear multilabel", "zero-shot LLM" ], "production_evidence": false, "summary": "Treats routing prompts to agents as multi-label set prediction where over-selection costs money, releasing a WildChat-derived benchmark over a 12-agent catalog with set metrics and a cost-tiered weighted routing setting. Supervised routers clearly beat nearest-neighbor and zero-shot LLM routing.", "key_idea": "Agent routing as cost-aware set-valued prediction.", "relevance_score": 4, "relevance_rationale": "Benchmark for routing among agents.", "found_via": "web search: routing among agents | web search: agentic routing benchmark multi-turn", "cites_or_cited_by": [] }, { "id": "basu-2026-w5h2-intent-caching", "title": "Why Agent Caching Fails and How to Fix It: Structured Intent Canonicalization with Few-Shot Learning", "authors": [ "Abhinaba Basu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.18922", "code_url": null, "topic_clusters": [ "C2", "C10" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "structured intent decomposition (W5H2)", "SetFit few-shot classifier", "risk-controlled selective prediction" ], "cost_reduction_reported": "intent matching 91.1% on MASSIVE in ~2 ms vs 37.9% for GPTCache and 68.8% for a 20B LLM at 3,447 ms; a five-tier cascade handles 85% of interactions locally, projecting 97.5% cost reduction", "benchmarks": [ "MASSIVE", "NyayaBench v2" ], "models_studied": [ "SetFit", "20B-parameter LLM" ], "production_evidence": false, "summary": "Argues agent caches fail because surface similarity misses task intent; decomposes requests into a structured intent (W5H2) predicted by a few-shot SetFit classifier and keys the cache on it, inside a multi-tier cascade with risk-controlled selective prediction guarantees.", "key_idea": "Key agent caches on canonicalized structured intent, with a local-first cascade.", "relevance_score": 4, "relevance_rationale": "Combines intent caching with a cascade and statistical risk control.", "found_via": "snowball: cited-by zhang-2025-agentic-plan-caching", "cites_or_cited_by": [ "bang-2023-gptcache", "zhang-2025-agentic-plan-caching" ] }, { "id": "bertalanic-2026-cost-of-consensus", "title": "The Cost of Consensus: Isolated Self-Correction Prevails Over Unguided Homogeneous Multi-Agent Debate", "authors": [ "Blaž Bertalanič", "Carolina Fortuna" ], "year": 2026, "venue_or_source": "ACM Conference on AI and Agentic Systems", "source_type": "paper", "url": "https://arxiv.org/abs/2605.00914", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Debate uses 2.1-3.4x more tokens (up to 28,631 per problem) than isolated self-correction for equal or lower accuracy", "benchmarks": [ "GSM-Hard", "MMLU-Hard" ], "models_studied": [ "Qwen2.5-7B", "Llama-3.1-8B", "Ministral-3-8B" ], "production_evidence": false, "summary": "A controlled study of homogeneous 10-agent debate with 7-8B models finds sycophantic conformity, contextual fragility and consensus collapse, and that isolated self-correction beats debate at 2-3x lower token cost.", "key_idea": "Unguided homogeneous debate costs more for no gain over self-correction.", "relevance_score": 4, "relevance_rationale": "Evidence against costly multi-agent debate at small scale.", "found_via": "arXiv title search: multi-agent cost", "cites_or_cited_by": [] }, { "id": "bhatti-2026-proteus", "title": "PROTEUS: SLA-Aware Routing via Lagrangian RL for Multi-LLM Serving Systems", "authors": [ "Amit Singh Bhatti", "Vishal Vaddina", "Dagnachew Birru" ], "year": 2026, "venue_or_source": "EuroMLSys 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.19402", "code_url": null, "topic_clusters": [ "C1", "C6", "C14" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "Lagrangian dual variable", "accuracy target as input", "accuracy target tau", "query features" ], "cost_reduction_reported": "90.1% accuracy on RouterBench within 1.3% of oracle; cost savings up to 89.8% vs best fixed model (abstract)", "benchmarks": [ "RouterBench", "SPROUT" ], "models_studied": [ "11 models (RouterBench)", "14 models (SPROUT)" ], "production_evidence": false, "summary": "PROTEUS accepts an accuracy target at runtime and uses Lagrangian RL with a learned dual variable so a single router satisfies the full range of accuracy floors. OmniRouter meets floors only 22% of the time in comparison.", "key_idea": "SLA-conditioned router via Lagrangian RL.", "relevance_score": 4, "relevance_rationale": "Operator-facing accuracy targets.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: cited-by wei-2025-barp", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "bholani-2026-self-healing-router", "title": "Graph-Based Self-Healing Tool Routing for Cost-Efficient LLM Agents", "authors": [ "Neeraj Bholani" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.01548", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "tool health monitors", "cost-weighted tool graph" ], "cost_reduction_reported": "Matches ReAct correctness while reducing control-plane LLM calls by 93% (9 vs 123) across 19 scenarios", "benchmarks": [ "19 scenarios over three graph topologies" ], "models_studied": [], "production_evidence": false, "summary": "Self-Healing Router treats most agent control flow as deterministic routing over a cost-weighted tool graph with Dijkstra. When a tool fails, edges are reweighted and the path recomputed; the LLM is called only when no feasible path exists.", "key_idea": "Replace most LLM control decisions with graph routing; call the LLM only to escalate.", "relevance_score": 4, "relevance_rationale": "An architectural cost lever that removes LLM calls from routine steps.", "found_via": "arXiv query ti:\"cost-efficient\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "bouchard-2026-escalation-decision-theory", "title": "Is Escalation Worth It? A Decision-Theoretic Characterization of LLM Cascades", "authors": [ "Dylan Bouchard" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06350", "code_url": null, "topic_clusters": [ "C1", "C2", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "confidence", "shadow prices", "confidence threshold" ], "cost_reduction_reported": "A lightweight pre-generation router beats the best cascade policy on 4 of 5 datasets (MATH, MMLU, TriviaQA, SimpleQA, LiveCodeBench), eight models", "benchmarks": [ "MATH", "MMLU", "TriviaQA", "SimpleQA", "LiveCodeBench" ], "models_studied": [ "eight models from five providers" ], "production_evidence": false, "summary": "Decision-theoretic analysis of LLM cascades via constrained optimization and duality: characterizes the cost-quality frontier of two-model and k-model threshold cascades and finds long fixed chains underperform the pairwise envelope.", "key_idea": "Duality-based frontier characterization for cascades.", "relevance_score": 4, "relevance_rationale": "Constrained optimization theory for escalation.", "found_via": "search: LLM routing constrained optimization | arXiv search: \"LLM cascade\"", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "bouchard-2026-trajectory-uq-agents", "title": "Beyond Single-Turn Confidence: Trajectory-Adapted Uncertainty Quantification for LLM Agents", "authors": [ "Dylan Bouchard", "Mohit Singh Chauhan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.11552", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "token probability", "resampled consistency", "reflexive self-assessment" ], "cost_reduction_reported": null, "benchmarks": [ "BFCL-v4", "tau2-bench" ], "models_studied": [], "production_evidence": false, "summary": "Evaluates UQ for multi-turn tool-use agents at trajectory level; reflexive self-assessment is the strongest low-cost baseline in most settings, and token-probability scores are sensitive to aggregation across turns.", "key_idea": "Trajectory-adapted UQ; self-assessment is a cheap strong baseline.", "relevance_score": 4, "relevance_rationale": "Practical guidance on which confidence signal to use in agents.", "found_via": "hf-papers search: uncertainty quantification LLM agents", "cites_or_cited_by": [] }, { "id": "bussing-2026-mo-capo", "title": "MO-CAPO: Multi-Objective Cost-Aware Prompt Optimization", "authors": [ "Jan Büssing", "Moritz Schlager", "Timo Heiß", "Tom Zehle", "Matthias Feurer" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.18869", "code_url": null, "topic_clusters": [ "C4", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "performance", "input/output token cost" ], "cost_reduction_reported": "Outperforms NSGA-II baseline on 8 of 12 cases (noisy R2), often at considerably lower budget", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MO-CAPO extends CAPO to true multi-objective prompt optimization of performance and a deployment-oriented inference-cost objective covering input and output tokens, with budget allocation for efficient search, returning a Pareto set of prompts.", "key_idea": "Multi-objective prompt optimization with a deployment-realistic token-cost objective.", "relevance_score": 4, "relevance_rationale": "Cost-quality Pareto prompt optimization; directly usable for per-module cost-aware configuration.", "found_via": "search: cost-aware prompt optimization LLM token cost Pareto arXiv", "cites_or_cited_by": [ "zehle-2025-capo" ] }, { "id": "cao-2026-early-outcome-prediction-audit", "title": "Testing-Driven Reliability Audit of Trajectory-Based Early Outcome Prediction for LLM Agents: Target-Specific Calibration Transfer Persists Within a Single Benchmark", "authors": [ "Yan-Ze Cao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.25647", "code_url": null, "topic_clusters": [ "C6", "C7", "C13" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "trajectory prefix features" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-bench Verified", "TerminalBench" ], "models_studied": [], "production_evidence": false, "summary": "Audits whether trajectory-based early-outcome predictors, used to stop agent runs once the result is predictable, stay calibrated when applied to agents they were not trained on, using leave-one-agent-out tests on public SWE-bench trajectories. Finds that calibration failures are concentrated in specific target agent and head combinations.", "key_idea": "Early-outcome predictors for agents need target-specific calibration checks.", "relevance_score": 4, "relevance_rationale": "Checks the reliability of success prediction used to cut agent costs.", "found_via": "query: predicting agent success trajectory", "cites_or_cited_by": [] }, { "id": "chang-2026-agentic-workload-characterization", "title": "From LLM Inference to Agentic Workloads: Characterization and Implications for Serving Systems", "authors": [ "Chaokun Chang", "Yukun Zhou", "Kaihua Fu", "Dakai An", "Tianyu Feng", "Hanfeng Lu", "Sheng Yao", "Pu Guo", "Yinghao Yu", "Yizhou Shan", "Bo Li", "Binhang Yuan", "Wei Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.15127", "code_url": null, "topic_clusters": [ "C10", "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "cross-request redundancy" ], "cost_reduction_reported": "task-aware serving cut latency 29-40%; communication-aware placement up to 4.5x; state offloading cut memory 4.6x; tool-result caching removed 35.2% of redundant search calls and 19.3% of aggregate search latency", "benchmarks": [ "production traces from three agent applications" ], "models_studied": [], "production_evidence": true, "summary": "Characterizes agentic workloads versus plain LLM inference and, using production traces from three applications, finds heavy redundancy in search queries and web fetches; four design explorations show these findings translate into latency, memory and tool-call savings.", "key_idea": "Agent workloads have exploitable redundancy and structure beyond LLM inference.", "relevance_score": 4, "relevance_rationale": "Production-trace evidence of caching and serving opportunities in agents.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "liu-2026-copilot-traces" ] }, { "id": "chang-2026-cascadedebate", "title": "CascadeDebate: Multi-Agent Deliberation for Cost-Aware LLM Cascades", "authors": [ "Raeyoung Chang", "Dongwook Kwon", "Jisoo Lee", "Nikhil Verma" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.12262", "code_url": null, "topic_clusters": [ "C2", "C12" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "confidence", "multi-agent consensus" ], "cost_reduction_reported": "Outperforms single-model cascades and standalone multi-agent systems by up to 26.75%; online threshold optimiser gives 20.98-52.33% relative gains over fixed policies", "benchmarks": [ "five benchmarks in science, medicine, general knowledge" ], "models_studied": [], "production_evidence": false, "summary": "Inserts a small multi-agent deliberation at each tier's escalation boundary so uncertain cases can be resolved by consensus before paying for a larger model or human. An online optimiser adjusts thresholds.", "key_idea": "Deliberate cheaply before escalating.", "relevance_score": 4, "relevance_rationale": "Combines multi-agent and cascade cost control.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "chatrath-2026-ready", "title": "READY or Not: Reliable Enterprise Agent Deployment", "authors": [ "Veronica Chatrath", "Bryan Zhu", "Jingxuan Fan", "George Pu", "Soham Dinesh Tiwari", "Soham Dan", "Ryan Young", "Yuan", "Li", "Yuang Yao", "Apaar Shanker", "Minglai Yang", "Daniel Yue Zhang", "Yunzhong He", "Ying Liu", "Chenguang Wang", "Zhijun Yin", "Yuan Xue" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.02095", "code_url": null, "topic_clusters": [ "C2", "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "reliability target", "human oversight policy", "cost" ], "cost_reduction_reported": "Two systems 0.3 points apart in autonomous accuracy need 39.2% vs 29.6% human review to reach 76% reliability", "benchmarks": [ "clinical-audit case study (16 agent systems, 750 cases)" ], "models_studied": [ "16 agent systems" ], "production_evidence": false, "summary": "Framework to qualify agents for enterprise deployment by measuring reliability and operating cost of the human-AI system under candidate oversight policies, selecting the cheapest policy that meets a reliability target, and statistically qualifying it on held-out cases.", "key_idea": "Qualify agents by min-cost oversight policy at a reliability target.", "relevance_score": 4, "relevance_rationale": "Deployment-oriented cost/reliability evaluation with deferral to humans.", "found_via": "web search: TheAgentCompany benchmark", "cites_or_cited_by": [] }, { "id": "chen-2026-co-failure-ceiling", "title": "When Does Combining Language Models Help? A Co-Failure Ceiling on Routing, Voting, and Mixture-of-Agents Across 67 Frontier Models", "authors": [ "Josef Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.27288", "code_url": null, "topic_clusters": [ "C1", "C12", "C13", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "all-model co-failure rate", "all-models-wrong rate (beta)" ], "cost_reduction_reported": null, "benchmarks": [ "mathematics", "code", "open-ended math", "execution-graded code", "GPQA-Diamond" ], "models_studied": [ "67 models from 21 providers" ], "production_evidence": false, "summary": "Proves that any policy outputting one member's answer (routing, voting, cascades) is capped at one minus the rate at which all models fail together, and gives a finite-sample certificate for this ceiling. Empirically the tail is underestimated by correlation models.", "key_idea": "Co-failure ceiling bounds achievable router gains.", "relevance_score": 4, "relevance_rationale": "Useful headroom diagnostic before building a router.", "found_via": "snowball: cited-by ong-2024-routellm | arXiv search: single vs multi-agent", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "chen-2026-da-grpo-cloud-offloading", "title": "Joint Continual Learning of Local Language Models and Cloud Offloading Decisions with Budget Constraints", "authors": [ "Evan Chen", "Wenzhi Fang", "Shiqiang Wang", "Christopher Brinton" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.00166", "code_url": null, "topic_clusters": [ "C3", "C5", "C8" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "cloud-usage budget in advantage" ], "cost_reduction_reported": null, "benchmarks": [ "math reasoning", "code generation" ], "models_studied": [], "production_evidence": false, "summary": "DA-GRPO lets a local SLM jointly learn task skill and when to request cloud LLM help during continual learning, building a cloud-usage budget directly into GRPO advantages. It reduces forgetting and keeps cloud usage stable.", "key_idea": "Budget-constrained RL teaches a local model when to offload to the cloud.", "relevance_score": 4, "relevance_rationale": "Learned escalation policy under a budget.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "chen-2026-hm-router", "title": "HM-ROUTER: Joint Model and Harness Routing for Agentic Systems", "authors": [ "Hao Mark Chen", "Royson Lee", "Yasuyuki Okoshi", "Dimitris Anastasiou", "Wayne Luk", "Hongxiang Fan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.32213", "code_url": "https://github.com/hmarkc/HM-Router", "topic_clusters": [ "C1", "C3", "C4" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "model and harness representations", "CP-decomposition interaction term" ], "cost_reduction_reported": "Leads at all seven evaluated cost budgets; +7.3 points mean routing accuracy over strongest learned baseline", "benchmarks": [ "benchmark from 12 public agent benchmarks (293 routes, 73 models, 25 harnesses)" ], "models_studied": [ "73 models" ], "production_evidence": false, "summary": "HM-Router jointly selects a model and an agent harness per query, learning shared model and harness representations with a tensor-decomposition interaction term so it can predict unobserved model-harness pairs. It beats learned baselines across cost budgets.", "key_idea": "Route over (model, harness) pairs, not just models.", "relevance_score": 4, "relevance_rationale": "Configuration-level routing for agents.", "found_via": "arXiv title search: agent routing", "cites_or_cited_by": [ "kapoor-2025-hal-holistic-agent-leaderboard", "liu-2026-harness-native-routing", "son-2026-swe-router" ] }, { "id": "chen-2026-lemon-counterfactual-orchestration", "title": "COLLATOR: Compositional Multi-Agent Orchestration with Counterfactual Reinforcement Learning", "authors": [ "Xudong Chen", "Yixin Liu", "Hua Wei", "Kaize Ding" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.14483", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "counterfactual credit on role/capacity/dependency edits" ], "cost_reduction_reported": "Best average performance among evaluated orchestration methods with improved accuracy-token trade-off", "benchmarks": [ "MMLU", "GSM8K", "AQuA", "MultiArith", "SVAMP", "HumanEval" ], "models_studied": [], "production_evidence": false, "summary": "LEMON is an LLM orchestrator that writes a unified orchestration spec of agent duties, capacity levels and dependencies per task, trained with GRPO plus localized counterfactual credit that edits individual role, capacity or dependency fields. It improves accuracy per token.", "key_idea": "Learn capacity (model-size) assignment per agent with counterfactual credit.", "relevance_score": 4, "relevance_rationale": "Per-agent capacity assignment with fine-grained credit.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "cai-2025-agentbalance", "su-2025-daao", "su-2025-toolorchestra", "yue-2025-masrouter", "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "chen-2026-signals-trajectory-triage", "title": "Signals: Trajectory Sampling and Triage for Agentic Interactions", "authors": [ "Shuguang Chen", "Adil Hafeez", "Salman Paracha" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.00356", "code_url": null, "topic_clusters": [ "C6", "C7", "C8" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "cheap interaction signals" ], "cost_reduction_reported": "Signal-based sampling: 82% informativeness vs 54% random, 1.52x efficiency per informative trajectory on tau-bench", "benchmarks": [ "tau-bench" ], "models_studied": [], "production_evidence": false, "summary": "Computes cheap signals from live agent interactions and attaches them as attributes to triage which trajectories are worth reviewing, without affecting online behavior.", "key_idea": "Cheap signals to triage production trajectories.", "relevance_score": 4, "relevance_rationale": "Bridge from production traffic to preference data and distillation.", "found_via": "hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [ "cemri-2025-mast-why-mas-fail" ] }, { "id": "chillara-2026-semanticalli", "title": "SemanticALLI: Caching Reasoning, Not Just Responses, in Agentic Systems", "authors": [ "Varun Chillara", "Dylan Kline", "Christopher Alvares", "Evan Wooten", "Huan Yang", "Shlok Khetan", "Cade Bauer", "Tré Guillory", "Tanishka Shah", "Yashodhara Dhariwal", "Volodymyr Pavlov", "George Popstefanov" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.16286", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "structured intermediate representation matches" ], "cost_reduction_reported": "baseline monolithic caching capped at 38.7% hit rate; caching the structured Visualization Synthesis stage reached 83.10% hit rate, bypassing 4,023 LLM calls with 2.66 ms median hit latency", "benchmarks": [ "analytics/visualization agent workload" ], "models_studied": [], "production_evidence": false, "summary": "In an analytics agent, decomposing generation into intent resolution and visualization synthesis makes structured intermediate representations cacheable. Even when users rarely repeat themselves, pipeline stages repeat at structured checkpoints where caching is reliable.", "key_idea": "Cache structured intermediate artifacts of the agent pipeline rather than final responses.", "relevance_score": 4, "relevance_rationale": "Step-level caching lesson for production agent pipelines.", "found_via": "Cost-Efficient Serving of LLM Agents via Test-Time Plan Caching", "cites_or_cited_by": [ "zhang-2025-agentic-plan-caching" ] }, { "id": "chu-2026-lqm-contextroute", "title": "Latency-Quality Routing for Functionally Equivalent Tools in LLM Agents", "authors": [ "Kexin Chu", "Dawei Xiang", "Wei Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.14241", "code_url": null, "topic_clusters": [ "C3", "C5" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "latency", "LLM-as-judge feedback", "query-specific quality estimate" ], "cost_reduction_reported": "+2.18 pp F1 over SW-UCB on web-search load benchmark; up to +18 pp accuracy over SW-UCB on StrategyQA high-heterogeneity setting", "benchmarks": [ "web search load benchmark", "StrategyQA", "retriever pools" ], "models_studied": [], "production_evidence": false, "summary": "LQM-ContextRoute is a contextual-bandit router that picks among functionally equivalent tool providers (search APIs, retrievers, LLM backends) under runtime load. It ranks providers by expected quality per service cycle, avoiding the collapse that additive latency-quality rewards cause, and learns online from judge feedback.", "key_idea": "Online bandit routing among equivalent tool/LLM providers.", "relevance_score": 4, "relevance_rationale": "Tool/backend routing inside agents learned online without gold labels.", "found_via": "arXiv query ti:\"routing\" AND ti:\"tool\" | snowball: cited-by taberner-miller-2026-paretobandit", "cites_or_cited_by": [ "taberner-miller-2026-paretobandit" ] }, { "id": "deng-2026-cfrc-residual-completion-handoffs", "title": "Do Not Restart: Residual Completion for Stateful Agent Handoffs", "authors": [ "Runzhi Deng", "Yiming Zhong", "Fang Zhao", "Pan Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.13800", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "residual contract of accepted progress" ], "cost_reduction_reported": "Comparable macro accuracy to strong full-task agents at 22.0%-34.6% mean per-surface cost", "benchmarks": [ "five tool-agent environments" ], "models_studied": [ "two same-provider model pairs" ], "production_evidence": false, "summary": "When routers or cascades hand a tool agent's task to another model mid-way, CFRC freezes a contract of accepted progress and completed effects so the successor only completes the residual, admitting execution when the remainder is covered. It keeps accuracy at a fraction of full-task cost.", "key_idea": "Hand off only the residual task with commitment contracts, not a restart.", "relevance_score": 4, "relevance_rationale": "Makes mid-trajectory handoffs cheaper and safer.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "liu-2026-harness-native-routing", "son-2026-swe-router", "wei-2026-step-level-cua-cascade" ] }, { "id": "dennis-2026-in-context-obsoletes-orchestration", "title": "In-Context Prompting Obsoletes Agent Orchestration for Procedural Tasks", "authors": [ "Simon Dennis", "Michael Diamond", "Rivaan Patil", "Kevin Shabahang", "Hao Guo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.27891", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "In-context approach scores 4.53-5.00 vs 4.17-4.84 for LangGraph orchestration; failure rates 11.5%/0.5%/5% vs 24%/9%/17%", "benchmarks": [ "travel booking", "Zoom support", "insurance claims" ], "models_studied": [], "production_evidence": false, "summary": "For procedural multi-turn tasks, putting the whole procedure in the system prompt and letting the model self-orchestrate beats an external LangGraph orchestrator on quality and failure rate across three domains. External orchestration is argued to be unnecessary for modern frontier models on such tasks.", "key_idea": "Self-orchestration in context beats external orchestrators for procedures.", "relevance_score": 4, "relevance_rationale": "Architecture simplification evidence.", "found_via": "snowball: cited-by kapoor-2024-ai-agents-that-matter", "cites_or_cited_by": [ "kapoor-2024-ai-agents-that-matter", "kim-2025-science-of-scaling-agent-systems" ] }, { "id": "dip-2026-sevra-selective-verification", "title": "Think Again or Think Longer? Selective Verification for Budget-Aware Reasoning", "authors": [ "Sajib Acharjee Dip", "Dawei Zhou", "Liqing Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.19808", "code_url": null, "topic_clusters": [ "C2", "C7", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "serving-visible attempt state", "recoverability gate" ], "cost_reduction_reported": "MATH: 76.3% vs 75.5% for always-verify with 26.8% fewer post-generation tokens; GSM8K: verifies 3.0% of examples, accuracy 93.4% to 94.5%, 91.2% fewer verification tokens; a longer initial solve matches with fewer tokens", "benchmarks": [ "MATH", "GSM8K", "CommonsenseQA" ], "models_studied": [ "Qwen3-4B" ], "production_evidence": false, "summary": "A serving-layer gate decides whether to keep a solver's answer or invoke verification. Finds selective verification helps but tuning the initial budget first is often cheaper.", "key_idea": "Tune initial budget first, then selectively verify.", "relevance_score": 4, "relevance_rationale": "Practical deployment rule for reasoning budgets.", "found_via": "arXiv title search: thinking budget", "cites_or_cited_by": [] }, { "id": "dong-2026-learning-to-stop-cost-study", "title": "When Does Learning to Stop Help? A Cost-Aware Study of Early Exits in Reasoning Models", "authors": [ "Zhe Dong", "Fang Qin", "Manish Shah" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.30852", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "confidence", "entropy", "answer stability", "learned stopper" ], "cost_reduction_reported": "Learned stopping gives +3.2 to +21.2 pp additional token saving on Qwen3 free-form math; the same policy saves 32% tokens with KV-cache forking but costs 121% extra under black-box repeated prefilling", "benchmarks": [ "GSM8K", "MATH-500", "MMLU-Pro", "AIME-90", "GPQA" ], "models_studied": [ "Qwen3", "DeepSeek-R1-distilled" ], "production_evidence": false, "summary": "Controlled comparison of early-exit rules at matched risk of losing correct answers across 18 settings, including probe overhead. Learned stoppers win where answers oscillate; simple thresholds suffice elsewhere; savings depend heavily on the serving regime.", "key_idea": "Early-exit savings depend on trajectory structure and serving cost model.", "relevance_score": 4, "relevance_rationale": "Deployment-focused decision procedure for stopping rules.", "found_via": "arXiv title search: early stopping reasoning", "cites_or_cited_by": [] }, { "id": "dong-2026-llm-shepherding", "title": "Pay for Hints, Not Answers: LLM Shepherding for Cost-Efficient Inference", "authors": [ "Ziming Dong", "Hardik Sharma", "Evan O'Toole", "Jaya Prakash Champati", "Kui Wu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.22132", "code_url": null, "topic_clusters": [ "C1", "C2", "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "predicted need for hint", "hint length" ], "cost_reduction_reported": "Reduces costs by 42-94% vs LLM-only on GSM8K, CNK12, HumanEval, MBPP; up to 2.8x cost reduction vs state-of-the-art routing and cascading at matched accuracy", "benchmarks": [ "GSM8K", "CNK12", "HumanEval", "MBPP" ], "models_studied": [], "production_evidence": false, "summary": "Requests only a short prefix of the LLM response as a hint for the small model, generalising routing and cascading. A two-stage predictor decides whether a hint is needed and how many tokens to buy.", "key_idea": "Buy a partial LLM hint instead of a full answer.", "relevance_score": 4, "relevance_rationale": "Token-level budget control between SLM and LLM.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "dou-2026-conformal-cascade", "title": "Conformal Cascade: Distribution-Free Accuracy Guarantees for Multi-Tier LLM Inference", "authors": [ "Yifan Dou", "Shikan Lian", "Shibo Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.25018", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "conformal prediction set size" ], "cost_reduction_reported": "Strictly improves over strongest calibration-tuned heuristic cascade on most family-benchmark pairs (18 MC benchmarks); on easy benchmarks most queries stay on the small model at no accuracy cost", "benchmarks": [ "18 multiple-choice benchmarks" ], "models_studied": [ "four open-weight model families" ], "production_evidence": false, "summary": "Defers whenever the calibrated conformal prediction set at a tier contains more than one answer, giving a distribution-free finite-sample accuracy guarantee for the cascade. Cost is characterised as a function of the risk level.", "key_idea": "Conformal set size as a guaranteed deferral rule.", "relevance_score": 4, "relevance_rationale": "Formal accuracy guarantees for multi-tier cascades.", "found_via": "arXiv search: learning to defer LLM", "cites_or_cited_by": [] }, { "id": "du-2026-swift", "title": "Why Search When You Can Transfer? Amortized Agentic Workflow Design from Structural Priors", "authors": [ "Shiyi Du", "Jiayuan Liu", "Weihua Du", "Yue Huang", "Jiayi Li", "Yingtao Luo", "Xiangliang Zhang", "Vincent Conitzer", "Carl Kingsford" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.25012", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "workflow demonstrations" ], "cost_reduction_reported": "Reduces marginal per-task optimization cost by three orders of magnitude vs SOTA search-based method", "benchmarks": [], "models_studied": [ "GPT-4o-mini", "Grok", "Qwen", "Gemma" ], "production_evidence": false, "summary": "SWIFT synthesizes an executable workflow for an unseen task from structural priors and cross-task workflow demonstrations, skipping iterative search while outperforming search-based methods on five benchmarks.", "key_idea": "Amortized workflow design by transfer instead of per-task search.", "relevance_score": 4, "relevance_rationale": "Slashes the optimization cost of workflow search itself.", "found_via": "search: FlowBank query-adaptive agentic workflows precompute reuse", "cites_or_cited_by": [ "zhang-2024-aflow" ] }, { "id": "dubey-2026-telemetry-failure-detection", "title": "Real-Time Detection and Repair of LLM Agent Failures", "authors": [ "Sunny Dubey" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.02464", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "step telemetry", "echo-state network", "CUSUM" ], "cost_reduction_reported": "Microsecond-cost monitors detect 0.71 of failures at 5% false-alarm (AUROC 0.872), versus per-step LLM judging that costs more than the agent", "benchmarks": [ "AFTraj-2K", "2,823 agent episodes" ], "models_studied": [ "qwen2.5 7b/3b", "llama3.1 8b", "gemini-2.5-flash" ], "production_evidence": false, "summary": "Detects mid-episode agent failures (loops, cascading tool errors, drift) from observable step telemetry with microsecond-cost monitors instead of an LLM judge per step.", "key_idea": "Cheap telemetry monitors instead of LLM step judges.", "relevance_score": 4, "relevance_rationale": "Exactly the cost-conscious failure signal needed for escalation.", "found_via": "hf-papers search: agent failure detection", "cites_or_cited_by": [] }, { "id": "duy-2026-online-l2d-varying-experts", "title": "Online Learning-to-Defer with Varying Experts", "authors": [ "Dang Hoang Duy", "Yannis Montreuil", "Maxime Meyer", "Axel Carlier", "Lai Xing Ng", "Wei Tsang Ooi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.12340", "code_url": null, "topic_clusters": [ "C2", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "model score", "queried-action bandit feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Introduces an online multiclass learning-to-defer algorithm for streaming data where the pool of experts changes and feedback is observed only for the chosen action. Gives regret bounds for true deferral loss that scale with the number of models and experts.", "key_idea": "Learning-to-defer under bandit feedback with a time-varying expert pool.", "relevance_score": 4, "relevance_rationale": "Online deferral with changing experts and partial feedback is the theoretical core of online cascades with model churn.", "found_via": "query: learning to defer online bandit feedback", "cites_or_cited_by": [] }, { "id": "fang-2026-budget-control-search-agents", "title": "Inference-Time Budget Control for LLM Search Agents", "authors": [ "Zhengru Fang", "Senkang Forest Hu", "Zhonghao Chang", "Yu Guo", "Yihang Tao", "Hongyao Liu", "Mengzhe Ruan", "Jun Huang", "Yuguang Fang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.05701", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "value-of-information score", "remaining dual budget" ], "cost_reduction_reported": null, "benchmarks": [ "four multi-hop QA benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "For search agents under hard caps on both tool calls and tokens, a controller scores each feasible action (retrieve, decompose, commit) by estimated marginal task value per unit budget. A selective finalizer rewrites the answer only for low-risk form errors; budget-dependent penalties drive most of the gain.", "key_idea": "Value-of-information budget allocation across agent actions under dual hard budgets.", "relevance_score": 4, "relevance_rationale": "Per-task budgeted agent execution (sub-theme e).", "found_via": "arXiv query ti:\"budget\" AND ti:agent AND ti:llm", "cites_or_cited_by": [] }, { "id": "feng-2026-llmrouter", "title": "LLMRouter: Unified Infrastructure for Developing, Evaluating, and Deploying LLM Routers", "authors": [ "Tao Feng", "Fangxu Yu", "Haozhen Zhang", "Zhongjie Dai", "Liangqi Yuan", "Zijie Lei", "Weizhi Zhang", "Kunlun Zhu", "Haodong Yue", "Keyang Xuan", "Ge Liu", "Jiaxuan You" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "repo", "url": "https://arxiv.org/abs/2608.06867", "code_url": null, "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "context/model encoders", "scoring functions", "decision rules" ], "cost_reduction_reported": "Learned routers outperform strongest fixed-model baseline by 14.6% relative (abstract)", "benchmarks": [ "xRouteBench" ], "models_studied": [], "production_evidence": false, "summary": "Unifies routing as a sequential decision process with five components and releases LLMRouter, an open-source library of 16+ routers, plus xRouteBench across generic, memory, vision, time-series and personalized routing.", "key_idea": "Unified router infrastructure and benchmark.", "relevance_score": 4, "relevance_rationale": "Practical toolkit.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "fisch-2026-pandoras-router", "title": "Pandora's AI Model Routing Box: Efficient Allocation with Costly Value Estimation", "authors": [ "Adam Fisch", "Shubhendu Trivedi", "Fantine Huot", "William W. Cohen", "Michael Kaisers", "Mirella Lapata", "Kate Larson", "Jacob Eisenstein" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.20316", "code_url": null, "topic_clusters": [ "C1", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "costly value estimation", "value of information" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formalizes routing with costly value estimation as Pandora's Box optimal search, giving closed-form value-of-information rules for whether refining an estimate for a specialist is worth it, with centralized and decentralized (bidding) variants.", "key_idea": "Optimal search theory for when to pay for better routing estimates.", "relevance_score": 4, "relevance_rationale": "Principled theory for routing with expensive signals (e.g. partial traces).", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "frauen-2026-causal-methods-llm", "title": "Causal Methods for LLM Development and Evaluation", "authors": [ "Dennis Frauen", "Marie Brockschmidt", "Konstantin Hess", "Hao-Rui Ma", "Yu-Chen Ma", "Abdurahman Maarouf", "Maresa Schröder", "J. Schweisthal", "Yu-Xin Wang", "Athiya Deviyani", "S. Parbhoo", "R. G. Krishnan", "S. Feuerriegel" ], "year": 2026, "venue_or_source": "Proceedings of the 32nd ACM SIGKDD Conference on Knowledge Discovery and Data Mining V.2", "source_type": "paper", "url": "https://arxiv.org/abs/2605.25998", "code_url": null, "topic_clusters": [ "C6", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "logged data", "judge scores" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Position and tutorial paper (KDD 2026) arguing that many LLM development questions, including whether to route a prompt to a larger or smaller model under cost limits, are causal. It reviews how confounded logs, biased judges and non-stationary deployment call for causal inference and off-policy methods.", "key_idea": "Routing and evaluation from logs should be treated as causal inference.", "relevance_score": 4, "relevance_rationale": "Frames router learning from logged data as causal and points to concrete estimators.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "gao-2026-decisionbench", "title": "DecisionBench: A Benchmark for Emergent Delegation in Long-Horizon Agentic Workflows", "authors": [ "Yuxuan Gao", "Megan Wang", "Yi Ling Yu", "Zijian Carl Ma", "Ao Qu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.19099", "code_url": null, "topic_clusters": [ "C3", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "delegation interface", "model profiles" ], "cost_reduction_reported": "Counterfactual perfect delegation is 15-31 points above measured performance", "benchmarks": [ "GAIA", "tau-bench", "BFCL multi-turn" ], "models_studied": [ "11 models from 7 vendor families" ], "production_evidence": false, "summary": "DecisionBench is a substrate for studying emergent delegation, where an agent can call peer models, with metrics for quality, cost, latency, delegation rate and routing fidelity over 23,375 instances. Mean quality barely changes across awareness conditions, but a counterfactual ceiling shows large unrealized delegation headroom.", "key_idea": "Benchmark for when agents should delegate steps to other models.", "relevance_score": 4, "relevance_rationale": "Evaluation for in-trajectory delegation.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [] }, { "id": "ghosh-2026-grade-gated-routing", "title": "Route, Communicate, and Reason: Gated Routing and Adaptive Depth for Efficient Multi-Agent Reasoning", "authors": [ "Sudipto Ghosh", "Tanmoy Chakraborty" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.10836", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "learned gates for agent selection, depth, communication, pruning" ], "cost_reduction_reported": "+4.8 points over strongest baseline on MMLU-Pro at half the active compute", "benchmarks": [ "GSM8K", "MMLU-Pro", "GPQA", "AIME-2025" ], "models_studied": [], "production_evidence": false, "summary": "GRADE uses four learned gates to choose which agents to consult, how deep in a hierarchy a query goes, when to communicate and which branches to prune, trained with a multi-agent GRPO variant over a hot-swappable expert registry.", "key_idea": "Learned gates control agent choice, depth and communication per query.", "relevance_score": 4, "relevance_rationale": "Adaptive MAS compute allocation.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration" ] }, { "id": "gillespie-2026-enterprise-agent-routing-scaling", "title": "Scaling Enterprise Agent Routing: Degradation, Diagnosis, and Recovery", "authors": [ "Kellen Gillespie", "Robyn Perry" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.17519", "code_url": null, "topic_clusters": [ "C1", "C13", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "embedding shortlisting" ], "cost_reduction_reported": null, "benchmarks": [ "110-agent, 584-tool catalog from a deployed enterprise assistant", "1,435 human-labeled production utterances" ], "models_studied": [ "three frontier models" ], "production_evidence": true, "summary": "Studies routing requests to specialized agents in a deployed enterprise productivity assistant as the catalog grows to 110 agents and 584 tools. Routing F1 drops 16-23 points, split into retrieval and confusion gaps, and embedding shortlisting recovers 10-17 points on real traffic.", "key_idea": "Agent-routing accuracy degrades with catalog size; shortlisting recovers it.", "relevance_score": 4, "relevance_rationale": "Production evidence on routing among agents.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems" ] }, { "id": "gola-2026-langsmith-fine-tuning", "title": "Introducing LangSmith Fine-Tuning", "authors": [ "Ankush Gola", "Jake Broekhuizen", "Vivek Trivedy" ], "year": 2026, "venue_or_source": "LangChain blog", "source_type": "product", "url": "https://www.langchain.com/blog/langsmith-fine-tuning", "code_url": null, "topic_clusters": [ "C6", "C8", "C14" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "curated successful trajectories", "trace review" ], "cost_reduction_reported": "Engine agent: Kimi K3 + SFT 96.0 vs Kimi K3 90.0 vs GPT-5.6 Sol 87.0 on internal IssueBench subset; OpenSWE Review: Qwen-3.8-27B + SFT F1 48.9%->53.7% with 29.8% fewer model calls and 29.4% fewer tool requests per review (vendor-reported, internal agents)", "benchmarks": [ "IssueBench (internal)", "internal PR review set" ], "models_studied": [ "Kimi K3", "Qwen-3.8-27B", "GPT-5.6 Sol" ], "production_evidence": true, "summary": "LangChain launches smithtune, a CLI that turns agent trajectories stored in LangSmith into SFT datasets, trains on Fireworks or Baseten, evaluates and deploys. Applied to two of LangChain's own production agents, fine-tuned open models beat a frontier model on issue detection and matched review quality with about 30% fewer model and tool calls; a less selective first dataset lowered F1, prompting a trace-review curation stage. Baseten's companion posts describe the training infrastructure (Baseten Loops).", "key_idea": "Curated production agent traces can be distilled into open models that are both better and cheaper per task.", "relevance_score": 4, "relevance_rationale": "Distillation from production agent logs with cost-per-task effects, from real internal agents.", "found_via": "direct: langchain.com/blog index; baseten.co/blog", "cites_or_cited_by": [ "openai-model-optimization-distillation-track" ] }, { "id": "gravara-2026-compass", "title": "Compass: Optimizing Compound AI Workflows for Dynamic Adaptation", "authors": [ "Milos Gravara", "Juan Luis Herrera", "Stefan Nastic" ], "year": 2026, "venue_or_source": "IEEE CLUSTER 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2603.20821", "code_url": null, "topic_clusters": [ "C4", "C5", "C11" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "queue depth", "profiled latency", "accuracy threshold" ], "cost_reduction_reported": "COMPASS-V reduces configuration evaluations by 57.5% on average vs exhaustive search (up to 95.3%); runtime adaptation achieves 90-98% SLO compliance, +71.6% over static high-accuracy baselines", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Compass discovers multiple accuracy-feasible Pareto configurations of a compound AI workflow offline and switches between them at runtime based on load, using a queuing-theory planner and a queue-depth-driven controller.", "key_idea": "Offline Pareto config discovery plus online load-aware configuration switching.", "relevance_score": 4, "relevance_rationale": "Links config optimization to runtime adaptation for production SLOs.", "found_via": "search: joint optimization model selection prompt compound AI system cost quality arXiv", "cites_or_cited_by": [ "gravara-2026-atlas" ] }, { "id": "gravara-2026-plaiground", "title": "PLAIground: SLO-Driven Runtime Model Selection for Compound AI Systems in the Edge-Cloud-Space Continuum", "authors": [ "Milos Gravara", "Cynthia Marcelino", "Andrija Stanisic", "Stefan Nastic" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.14356", "code_url": null, "topic_clusters": [ "C3", "C4", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "SLOs on accuracy, latency, cost" ], "cost_reduction_reported": "Up to 91.3% accuracy while maintaining SLO compliance; fixed-model strategies violate cost and latency budgets up to 21x or miss accuracy by 4%", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "PLAIground enables runtime per-task model switching in compound AI workflows through a model abstraction with task/data contracts and an SLO-driven selector (Pixie).", "key_idea": "Runtime SLO-driven per-task model selection.", "relevance_score": 4, "relevance_rationale": "Per-step model selection under cost/latency SLOs.", "found_via": "snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [] }, { "id": "grotov-2026-speculative-uncertainty", "title": "How to Speculate about Uncertainty in Agentic Coding? A Draft-Model Gate Method", "authors": [ "Konstantin Grotov", "Valentin Malykh" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.05274", "code_url": null, "topic_clusters": [ "C3", "C6", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "draft-model cross-likelihoods", "phase-aware features" ], "cost_reduction_reported": "Pre-execution veto gate cuts execution error rate by 6-8 pp and token cost by 14-19% in deployment", "benchmarks": [], "models_studied": [ "Qwen3-Coder-480B", "Claude 3.5 Sonnet" ], "production_evidence": false, "summary": "Recovers a failure-likelihood signal for a black-box coding agent by having a small open draft model score the agent's generated trajectory in one forward pass and calibrating phase-aware features. The score can drive routing, human intervention or extra compute; a veto gate is demonstrated.", "key_idea": "A small draft model scores a black-box agent's actions to predict failure before execution.", "relevance_score": 4, "relevance_rationale": "Cheap, black-box step-level failure predictor that can feed a router; cost reductions reported.", "found_via": "query: abs:\"predict\" AND abs:\"trajectory\" AND abs:\"failure\" AND abs:\"LLM agent\"", "cites_or_cited_by": [] }, { "id": "guan-2026-symphony-coord", "title": "Symphony-Coord: Adaptive Routing for Multi-Agent LLM Systems", "authors": [ "Zhaoyang Guan", "Huixi Cao", "Ming Zhong", "Yin Wang", "Guanyu Liu", "Eric Yang", "Lynn Ai", "Yongxin Ni", "Bill Shi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.00966", "code_url": null, "topic_clusters": [ "C3", "C5" ], "routing_granularity": "subagent", "learning_regime": "bandit-online", "signals_used": [ "task features", "agent state", "delayed feedback" ], "cost_reduction_reported": null, "benchmarks": [ "simulation", "LLM benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Symphony-Coord casts agent selection for subtasks as an online contextual bandit, with a candidate-screening step and a LinUCB selector updated by delayed post-execution feedback, and proves sublinear regret. It improves routing efficiency and recovery under shifts and agent failures.", "key_idea": "Contextual-bandit routing of subtasks to agents.", "relevance_score": 4, "relevance_rationale": "Online learning for agent routing.", "found_via": "arXiv title search: routing multi-agent system LLM", "cites_or_cited_by": [] }, { "id": "guo-2026-eet-early-termination", "title": "EET: Experience-Driven Early Termination for Cost-Efficient Software Engineering Agents", "authors": [ "Yaoqi Guo", "Ying Xiao", "Jie M. Zhang", "Mark Harman", "Yiling Lou", "Yang Liu", "Zhenpeng Chen" ], "year": 2026, "venue_or_source": "ACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.05777", "code_url": "https://github.com/IanWalls/EET", "topic_clusters": [ "C9", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "structured experience from prior runs" ], "cost_reduction_reported": "Total cost reduced 19%-55% (32% average) with at most 0.2% loss in resolution rate; API calls, input and output tokens down 21%, 30%, 25%", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "three representative SE agents" ], "production_evidence": false, "summary": "EET extracts experience from past issue-resolution runs and uses it to stop patch generation and selection early when further iterations are unlikely to help, cutting coding-agent cost by about a third.", "key_idea": "Experience-driven early stopping for coding agents.", "relevance_score": 4, "relevance_rationale": "Step-level cost lever from logged runs.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "gandhi-2024-budgetmlagent" ] }, { "id": "guo-2026-sciorch", "title": "SciOrch: Learning to Orchestrate Expert LLMs for Solving Frontier Multimodal Scientific Reasoning Tasks", "authors": [ "Jingru Guo", "Xiangyuan Xue", "Lian Zhang", "Wanghan Xu", "Siki Chen", "Philip H. S. Torr", "Wan-Li Ouyang", "Lei Bai", "Zhenfei Yin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.15872", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "MCTS-based offline rollouts", "API cost" ], "cost_reduction_reported": null, "benchmarks": [ "frontier multimodal scientific reasoning" ], "models_studied": [], "production_evidence": false, "summary": "SciOrch trains an 8B orchestrator to decompose scientific questions, delegate sub-problems to selected commercial frontier models via API, and synthesize the answer. Because each action is an expensive API call, the authors replace online RL rollouts with MCTS-based data collection.", "key_idea": "Small trained orchestrator delegating sub-problems to frontier APIs.", "relevance_score": 4, "relevance_rationale": "ToolOrchestra-style small-orchestrator/big-worker design; cost is treated mainly as a training constraint.", "found_via": "snowball: cited-by su-2025-toolorchestra / yue-2025-masrouter", "cites_or_cited_by": [ "su-2025-toolorchestra", "yue-2025-masrouter" ] }, { "id": "guo-2026-think-when-needed-ranking", "title": "Think When Needed: Model-Aware Reasoning Routing for LLM-based Ranking", "authors": [ "Huizhong Guo", "Tianjun Wei", "Dongxia Wang", "Yingpeng Du", "Ziyan Wang", "Jie Zhang", "Zhu Sun" ], "year": 2026, "venue_or_source": "SIGIR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.18146", "code_url": null, "topic_clusters": [ "C1", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "ranking features (candidate dispersion)", "model-aware difficulty checklist" ], "cost_reduction_reported": "+6.3% NDCG@10 with 49.5% fewer tokens on MovieLens with Qwen3-4B", "benchmarks": [ "MovieLens", "two other ranking datasets" ], "models_studied": [ "Qwen3-4B", "other open LLMs" ], "production_evidence": false, "summary": "A lightweight router head uses pre-generation features to decide whether an LLM ranker should reason, and picks its operating point on a validation Pareto frontier under system constraints.", "key_idea": "Pre-generation think/no-think router for LLM ranking.", "relevance_score": 4, "relevance_rationale": "Deployable per-request reasoning routing.", "found_via": "web search: routing between thinking and non-thinking modes", "cites_or_cited_by": [] }, { "id": "hajimiri-2026-skill-memory-worth-tokens", "title": "Are Online Skill and Memory Modules Always Worth Their Tokens? A Budget-Constrained Study of Web Agents", "authors": [ "Sina Hajimiri", "Masih Aminbeidokhti", "Jose Dolz", "Ismail Ben Ayed", "Issam H. Laradji", "Spandana Gella", "Nicolas Gontier" ], "year": 2026, "venue_or_source": "EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2606.15017", "code_url": null, "topic_clusters": [ "C10", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "token-matched budget" ], "cost_reduction_reported": "Token-matched vanilla actor matches or beats AWM, ASI and ReasoningBank in aggregate success while often using fewer tokens", "benchmarks": [ "WebArena (four domains)", "WorkArena-L1" ], "models_studied": [ "Gemini 3 Flash", "GPT-5.4-mini", "Qwen 3.6-27B" ], "production_evidence": false, "summary": "Re-evaluates memory, workflow and skill modules for web agents under a fixed total token budget and finds a vanilla actor given the same tokens for extra steps matches or beats them. Run-to-run variance is large and should be reported.", "key_idea": "Many agent add-on modules lose to a budget-matched plain actor.", "relevance_score": 4, "relevance_rationale": "Architecture-level cost evidence under matched budgets.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "bechard-2026-terminal-agents-suffice" ] }, { "id": "hasanaliyev-2026-expected-reward-prediction", "title": "Expected Reward Prediction, with Applications to Model Routing", "authors": [ "Kenan Hasanaliyev", "Silas Alberti", "Jenny Hamer", "Dheeraj Rajagopal", "Kevin Robinson", "Jasper Snoek", "Victor Veitch", "Alexander Nicholas D'Amour" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.20217", "code_url": null, "topic_clusters": [ "C1", "C6", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "reward model scores", "prompt" ], "cost_reduction_reported": null, "benchmarks": [ "open-perfectblend" ], "models_studied": [ "Llama3.1-Instruct 8B/70B", "Gemma2-IT 9B/27B", "Gemma1-IT 7B" ], "production_evidence": false, "summary": "Shows the expected reward an LLM would earn on a prompt under repeated sampling can be predicted before generation, and uses this prediction for cost-controlled routing that beats category-level baselines.", "key_idea": "Predict expected reward-model score per model per prompt for routing.", "relevance_score": 4, "relevance_rationale": "Reward modeling for routing, extensible to new models.", "found_via": "snowball: cited-by nguyen-2024-metallm", "cites_or_cited_by": [ "lu-2023-zooter" ] }, { "id": "he-2026-market-aware-routing", "title": "You Cannot Pick a Provider From the Price List: Market-Aware Routing for Open-Weight LLM Inference", "authors": [ "Liang He", "Jing-Bo Wen", "Yi-Xiong Chen", "Yue Yang", "Qi-Zhen Lan", "Kang-Ning Cui", "Xi-Lu Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.37902", "code_url": null, "topic_clusters": [ "C1", "C5", "C14" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "measured provider quality", "latency", "availability", "price" ], "cost_reduction_reported": "Certification moved live traffic from a premium anchor provider to a substantially cheaper certified endpoint (matched quality)", "benchmarks": [], "models_studied": [ "open-weight LLMs across multiple providers" ], "production_evidence": true, "summary": "Measures live inference endpoints across providers serving the same open-weight models over several waves and finds that quality, latency and availability vary widely and are not predicted by price, and that the map drifts over time. Proposes FACET, an online provider router that certifies per-provider, per-task feasibility and fails safe to an anchor provider.", "key_idea": "Same-model provider routing with online feasibility certification under drifting provider quality.", "relevance_score": 4, "relevance_rationale": "Real endpoint measurements show drift; online safe routing design is directly usable in gateways.", "found_via": "snowball: cited-by woisetschlager-2026-slarouter", "cites_or_cited_by": [ "woisetschlager-2026-slarouter" ] }, { "id": "healy-2026-tool-selection-hallucination-probes", "title": "Internal Representations as Indicators of Hallucinations in Agent Tool Selection", "authors": [ "Kait Healy", "Bharathi Srinivasan", "Visakh Madathil", "Jing Wu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.05214", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "internal representations" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Detects tool-calling hallucinations (wrong tool, malformed params, tool bypass) in real time from internal representations in the same forward pass, up to 86.4% accuracy.", "key_idea": "Single-pass probe for tool-call errors.", "relevance_score": 4, "relevance_rationale": "Cheap step-level check for tool agents.", "found_via": "hf-papers search: hallucination detection single pass", "cites_or_cited_by": [] }, { "id": "hong-2026-maspob", "title": "MASPOB: Bandit-Based Prompt Optimization for Multi-Agent Systems with Graph Neural Networks", "authors": [ "Zhi Hong", "Qian Zhang", "Jiahang Sun", "Zhiwei Shang", "Mingze Kong", "Xiangyi Wang", "Yao Shu", "Zhongxiang Dai" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.02630", "code_url": null, "topic_clusters": [ "C4", "C5" ], "routing_granularity": "subagent", "learning_regime": "bandit-online", "signals_used": [ "UCB uncertainty", "GNN topology embedding" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Optimizes the prompts of a fixed multi-agent workflow under a tight evaluation budget by treating the problem as a bandit with UCB exploration, a graph neural network to capture coupling induced by topology, and coordinate ascent to make the search linear rather than exponential.", "key_idea": "Sample-efficient bandit optimization of per-agent prompts in a multi-agent system.", "relevance_score": 4, "relevance_rationale": "Budgeted configuration optimization for multi-agent systems.", "found_via": "query: contextual bandit prompt optimization online", "cites_or_cited_by": [] }, { "id": "hu-2026-cost-inefficient-coding-agents", "title": "Analyzing and Mitigating Cost-Inefficient Behaviors in Coding Agents", "authors": [ "Yiran Hu", "Nan Jiang", "Shanchao Liang", "Anik Dey", "Yi Wu", "Lin Tan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.30725", "code_url": null, "topic_clusters": [ "C10", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "trace analysis", "trajectory analysis" ], "cost_reduction_reported": "developer-designed skills reduced cost by up to 41.73%; subsumed retrieval, similar script generation and test re-execution account for up to 22.75% of task cost", "benchmarks": [ "SWE-bench Verified", "SWE-bench Pro" ], "models_studied": [ "Claude Code", "Mini-SWE-Agent" ], "production_evidence": false, "summary": "Analyzes 1,200 coding-agent trajectories (Claude Code, Mini-SWE-Agent) and identifies three recurring wasteful behaviors: subsumed retrieval, similar script generation and test re-execution. Developer-written skills reduce cost up to 42%, while structure-aware retrieval and agent-synthesized skills help less or even raise cost.", "key_idea": "Target recurring wasteful behaviors in coding agents with human-designed skills.", "relevance_score": 4, "relevance_rationale": "Token-consumption analysis with mitigation for coding agents.", "found_via": "web search: AI agents that matter | SWE-agent context management trajectory reduction cost coding agents arXiv | web search: coding agents spend tokens", "cites_or_cited_by": [ "bai-2026-agents-spend-your-money", "xiao-2025-agentdiet-trajectory-reduction" ] }, { "id": "huang-2026-rlcascaderouter", "title": "RLCascadeRouter: Quality-Estimator-Free Cascade Routing via Reinforcement Learning", "authors": [ "Shihong Huang", "Shengjie Wang", "Hong Ma", "Zhou Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.15817", "code_url": null, "topic_clusters": [ "C1", "C2", "C6" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "observed responses", "trajectory returns" ], "cost_reduction_reported": null, "benchmarks": [ "LLMRouterBench (ten benchmarks)", "ten LLMRouterBench benchmarks" ], "models_studied": [ "thirteen LLMs" ], "production_evidence": false, "summary": "Formulates cascade routing as an MDP with stop and model-selection actions and optimises the performance-cost objective directly with RL, avoiding separate quality estimators. Supports unseen models without retraining.", "key_idea": "RL cascade routing without explicit quality estimators.", "relevance_score": 4, "relevance_rationale": "Decision-focused learning for cascades.", "found_via": "snowball: cited-by AutoMix (2310.12963) | snowball: cited-by panda-2025-pilot", "cites_or_cited_by": [ "aggarwal-2023-automix" ] }, { "id": "huang-2026-uniscale", "title": "UniScale: Adaptive Unified Inference Scaling via Online Joint Optimization of Model Routing and Test-Time Scaling", "authors": [ "Kaiyu Huang", "Xingyu Wang", "Mingze Kong", "Zhubo Shi", "Yuqian Hou", "Hong Xu", "Zhongxiang Dai", "Minchen Yu", "Qingjiang Shi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.30898", "code_url": null, "topic_clusters": [ "C1", "C5", "C9" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query features", "cost model", "reward feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Unifies model routing and test-time-scaling choices into a single action space and learns the joint policy online as a contextual bandit with LinUCB plus explicit cost modeling. Reports better quality-cost trade-offs than routing-only or TTS-only baselines in dynamic inference settings.", "key_idea": "Learn routing and test-time compute jointly online with a LinUCB contextual bandit.", "relevance_score": 4, "relevance_rationale": "Online bandit over combined model-choice and compute-budget actions; transferable to agent step configuration.", "found_via": "query: abs:\"bandit\" AND abs:\"LLM\" AND abs:\"routing\"", "cites_or_cited_by": [] }, { "id": "hussain-2026-coverage-illusion-rag", "title": "The Coverage Illusion: From Pre-retrieval Routing Failure to Post-retrieval Cascades in a Production RAG System", "authors": [ "Zafar Hussain", "Kristoffer Nielbo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.27220", "code_url": null, "topic_clusters": [ "C2", "C6", "C10", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "retrieval returned no documents", "retrieval result emptiness" ], "cost_reduction_reported": "Post-retrieval cascade: +0.140 Composite Overall vs Always-HyDE, 31.8% lower latency, 72.2% of real queries served without LLM augmentation", "benchmarks": [ "Danish National Encyclopedia production traffic (20,000 query-workflow pairs)", "20,000 query-workflow pairs from production traffic" ], "models_studied": [], "production_evidence": true, "summary": "Case study on a production RAG system showing synthetic queries overstate the need for LLM query augmentation (90% vs 27.8% on real traffic). A cheapest-first post-retrieval cascade escalates only when retrieval fails.", "key_idea": "Escalate pipeline steps only after cheap steps fail on real traffic.", "relevance_score": 4, "relevance_rationale": "Production evidence; pipeline-step cascade.", "found_via": "arXiv search: LLM cascade deferral cost quality | arXiv: routing production traffic LLM", "cites_or_cited_by": [] }, { "id": "jiang-2026-cota-tiny-advisors", "title": "Don't Solve, Just Compare: Tiny Advisors for Runtime Intervention in LLM Agents", "authors": [ "Yanze Jiang", "Mingxuan Li", "Yuhao Wang", "Shengfang Zhai", "Jiaheng Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.21027", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "counterfactual same-prefix branches", "pairwise action comparison" ], "cost_reduction_reported": null, "benchmarks": [ "WebShop", "ALFWorld", "tau^3-Retail" ], "models_studied": [], "production_evidence": false, "summary": "COTA adds a 0.5B comparator to an LLM agent at runtime. The comparator judges the actor's proposed action against alternatives and returns non-binding advice, rather than acting as a costly expert solver or task-capable critic. It is trained from counterfactual branches with the same prefix.", "key_idea": "A tiny comparator model as a cheap runtime advisor in place of a big expert model.", "relevance_score": 4, "relevance_rationale": "Big-small collaboration inside agent loops, with the small model taking a cheap supervisory role; the abstract claims the best performance-cost trade-off but gives no number.", "found_via": "arXiv query ti:advisor AND abs:\"LLM\" AND abs:agent AND abs:cost", "cites_or_cited_by": [] }, { "id": "jiang-2026-spike", "title": "SPIKE: An Adaptive Dual Controller Framework for Cost-Efficient Long-Horizon Game Agents", "authors": [ "Wencan Jiang", "Jiangning Zhang", "Jianbiao Mei", "Jinzhuo Liu", "Yu Yang", "Xiaobin Hu", "Zhucun Xue", "Yong Liu", "Dacheng Tao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.18636", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "visual change", "task progress", "repeated actions", "failure signals" ], "cost_reduction_reported": "Token consumption reduced 54.9% and latency 40.8%; +5.0 pp SR and +9.3 pp budgeted SR over strongest baselines on StarDojo Lite-100", "benchmarks": [ "StarDojo Lite-100" ], "models_studied": [], "production_evidence": false, "summary": "SPIKE splits a long-horizon game agent into a low-frequency strategic controller and a token-budgeted reactive controller. An event trigger decides when to escalate from reactive execution to strategic reasoning, so the expensive reasoning is reused across stable stretches.", "key_idea": "Event-triggered escalation from a cheap reactive controller to an expensive strategic controller.", "relevance_score": 4, "relevance_rationale": "Per-step escalation policy with cost numbers; the controllers differ in reasoning mode, not necessarily in model.", "found_via": "arXiv query ti:\"cost-efficient\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "kabeer-2026-pearl", "title": "PEARL: Performance-Efficient Agentic Retrieval-Augmented Generation Through Learned Orchestration", "authors": [ "S. Kabeer", "A. Sirisha", "R. G. Reddy" ], "year": 2026, "venue_or_source": "ICCBI 2026", "source_type": "paper", "url": "https://doi.org/10.1109/ICCBI68589.2026.11619898", "code_url": null, "topic_clusters": [ "C1", "C3", "C8" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "GPT-4 routing traces" ], "cost_reduction_reported": "Distilled Flan-T5-small (80M) orchestrator achieves Routing Accuracy Score 0.942 on Indian legal queries", "benchmarks": [ "Indian legal queries" ], "models_studied": [ "Flan-T5-small", "GPT-4" ], "production_evidence": false, "summary": "PEARL replaces a GPT-4 orchestrator in a multi-agent RAG system with an 80M Flan-T5 trained on 12,000 GPT-4 routing traces, classifying queries into one of four agent pipeline configurations.", "key_idea": "Distill the LLM orchestrator into a tiny classifier from its own traces.", "relevance_score": 4, "relevance_rationale": "A router learned from orchestrator traces (sub-theme c).", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [] }, { "id": "kabir-2026-inframind", "title": "INFRAMIND: Infrastructure-Aware Multi-Agent Orchestration", "authors": [ "Ahasan Kabir", "Jiaqi Xue", "Meng Zheng", "Qian Lou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.11440", "code_url": null, "topic_clusters": [ "C3", "C4", "C11" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "queue depth", "KV-cache pressure", "latency", "task/model features" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "INFRAMIND argues that multi-agent orchestrators and learned routers ignore serving-infrastructure state, so preferred models queue up while equally capable alternatives sit idle. It feeds noisy runtime signals into planning, per-step routing and scheduling decisions for multi-agent pipelines on shared GPU clusters.", "key_idea": "Make per-step model routing in MAS aware of live serving load.", "relevance_score": 4, "relevance_rationale": "Extends MasRouter-style routing with infrastructure signals relevant to production cost and latency.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "kapoor-2026-trim-stepwise-routing", "title": "TRIM: Hybrid Inference via Targeted Stepwise Routing in Multi-Step Reasoning Tasks", "authors": [ "Vansh Kapoor", "Aman Gupta", "Hao Chen", "Anurag Beniwal", "Jing Huang", "Aviral Kumar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.10245", "code_url": null, "topic_clusters": [ "C2", "C3", "C9" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "process reward model score", "step-level uncertainty", "budget" ], "cost_reduction_reported": "On MATH-500 simplest threshold policy gives 5x higher cost efficiency than prior routing methods; advanced policies match the strong model using 80% fewer expensive-model tokens; up to 6x higher cost efficiency on AIME", "benchmarks": [ "MATH-500", "AIME" ], "models_studied": [], "production_evidence": false, "summary": "TRIM routes only the reasoning steps that are likely to derail a solution to a large model, letting a small model write routine continuations. A PRM flags risky steps, and policies range from simple thresholds to long-horizon policies that reason about accuracy-cost trade-offs under step-correctness uncertainty.", "key_idea": "Escalate only the critical steps (as flagged by a PRM) to the big model.", "relevance_score": 4, "relevance_rationale": "Clean step-level big/small routing with strong cost numbers; math reasoning rather than tool-use agents, but the mechanism maps directly to per-step agent escalation.", "found_via": "arXiv query ti:\"stepwise\" AND ti:routing; also cited-by lee-2025-steer", "cites_or_cited_by": [ "ye-2026-roro-process-reward-routing" ] }, { "id": "katz-2026-cursor-token-efficiency", "title": "Improved token efficiency for longer agent runs", "authors": [ "Jediah Katz", "Connor O'Keefe", "Calvin Yee" ], "year": 2026, "venue_or_source": "Cursor blog", "source_type": "blog", "url": "https://cursor.com/blog/improved-token-efficiency", "code_url": null, "topic_clusters": [ "C3", "C10", "C12", "C14" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Harness changes cut user token costs 7% with no quality loss: system prompt trimmed ~66%; MCP tools moved to dynamic context (-46.9% total tokens in sessions calling an MCP tool); offloading built-in tools cut static tool-description tokens 60%; cold cache misses -20%; line numbers every 10th line cut cache-read tokens 1.6% (vendor-reported, production traffic)", "benchmarks": [ "Cursor production traffic" ], "models_studied": [], "production_evidence": true, "summary": "Breaks down where agent inference spend goes in production (output, uncached and cached input by context source) and reports harness changes: trimming instructions models now follow natively, loading rarely needed tools on demand, reordering static context for cache reuse, compressing file-read line numbers, and restraining subagent spawning and subagent model switching. Notes the coordination tax of context-isolated subagents.", "key_idea": "Many small harness-level context changes compound into measurable production token savings without quality loss.", "relevance_score": 4, "relevance_rationale": "Production evidence on context and subagent levers in a large agent product.", "found_via": "direct: cursor.com/blog index", "cites_or_cited_by": [ "katz-2026-cursor-dynamic-context-discovery", "okeefe-2026-cursor-router" ] }, { "id": "khaled-2026-pyramid-moa", "title": "Pyramid MoA: A Probabilistic Framework for Cost-Optimized Anytime Inference", "authors": [ "Arindam Khaled" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.19509", "code_url": null, "topic_clusters": [ "C2", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "value of computation", "router confidence" ], "cost_reduction_reported": "Up to 42.9% compute savings on GSM8K/MMLU while nearly matching the 68.1% Oracle baseline; zero-shot transfer to HumanEval and MATH 500", "benchmarks": [ "MBPP", "GSM8K", "MMLU", "HumanEval", "MATH 500" ], "models_studied": [], "production_evidence": false, "summary": "Frames cascades as anytime algorithms and builds a hierarchical mixture-of-agents with a decision-theoretic escalation rule derived from value-of-computation theory. Also finds that passing small-model reasoning upward can help or hurt the large model depending on correctness.", "key_idea": "Anytime-computation escalation rule for hierarchical mixture-of-agents.", "relevance_score": 4, "relevance_rationale": "Escalation theory applied to multi-agent hierarchies.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "khraishi-2026-model-switching-drift", "title": "Evaluating Performance Drift from Model Switching in Multi-Turn LLM Systems", "authors": [ "Raad Khraishi", "Iman Zafar", "Katie Myles", "Greig A Cowan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.03111", "code_url": null, "topic_clusters": [ "C3", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "prefix influence", "suffix susceptibility" ], "cost_reduction_reported": null, "benchmarks": [ "CoQA", "Multi-IF" ], "models_studied": [ "GPT-5-nano", "GPT-5-mini" ], "production_evidence": false, "summary": "The paper measures what happens when a multi-turn system switches models mid-interaction, so that a later model must continue a prefix written by a different model. Even a single-turn handoff moves Multi-IF strict success by -8 to +13 pp; the effect decomposes into per-model prefix influence and suffix susceptibility terms, which explain about 70% of the variance.", "key_idea": "Mid-conversation model handoffs cause significant, model-pair-specific drift.", "relevance_score": 4, "relevance_rationale": "Key risk evidence for per-step routing; complements ganz-2026-handoff-tax.", "found_via": "arXiv query ti:\"model switching\" AND abs:llm", "cites_or_cited_by": [] }, { "id": "kim-2026-agent-memory-distillation", "title": "Agent Memory Distillation: Empowering Small LLM Agents with Hierarchical Teacher Memory", "authors": [ "Taeil Kim", "Kangsan Kim", "Sung Ju Hwang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.07169", "code_url": null, "topic_clusters": [ "C8", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "hierarchical teacher memory" ], "cost_reduction_reported": "Average accuracy gains of 27.2, 11.2 and 3.4 points on AppWorld, BFCL V3 and ToolSandbox for 4B-8B students with GPT-5-mini teacher", "benchmarks": [ "AppWorld", "BFCL V3", "ToolSandbox" ], "models_studied": [ "GPT-5-mini", "4B-8B students" ], "production_evidence": false, "summary": "Distills a teacher's successful experience into hierarchical memory (task, subtask) that small agents retrieve, rather than updating weights.", "key_idea": "Memory-based distillation for small agents.", "relevance_score": 4, "relevance_rationale": "Training-free way to lift cheap agents using teacher traces.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "koster-2026-memboost", "title": "MemBoost: A Memory-Boosted Framework for Cost-Aware LLM Inference", "authors": [ "Joris Köster", "Zixuan Liu", "Siavash Khajavi", "Zizhan Zheng" ], "year": 2026, "venue_or_source": "ICML MemFM Workshop 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2603.26557", "code_url": null, "topic_clusters": [ "C2", "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "retrieved memory relevance", "uncertainty-based escalation" ], "cost_reduction_reported": null, "benchmarks": [ "simulated interactive workloads" ], "models_studied": [], "production_evidence": false, "summary": "MemBoost lets a cheap model answer using retrieved past answers and supporting information, reuses answers when possible, grows memory continuously and escalates hard or uncertain queries to a stronger model. It reports large reductions in large-model calls at near strong-model quality.", "key_idea": "Memory-backed answer reuse plus cost-aware escalation.", "relevance_score": 4, "relevance_rationale": "Merges semantic caching with cascade-style escalation.", "found_via": "semantic caching LLM false positive hits correctness study", "cites_or_cited_by": [ "cheema-2025-tweakllm" ] }, { "id": "kotte-2026-ucci", "title": "UCCI: Calibrated Uncertainty for Cost-Optimal LLM Cascade Routing", "authors": [ "Varun Kotte" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.18796", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "token margin uncertainty", "isotonic calibration" ], "cost_reduction_reported": "Cuts inference cost by 31% (95% CI 27-35%) at micro-F1 0.91 on a production NER workload of 75,000 queries (4B/12B LLMs); ECE from 0.12 to 0.03", "benchmarks": [ "production NER workload" ], "models_studied": [ "4B and 12B instruction-tuned LLMs" ], "production_evidence": true, "summary": "Calibrates token-margin uncertainty into per-query error probabilities with isotonic regression and picks the escalation threshold by constrained cost minimisation. Evaluated on a production named-entity workload, beating entropy thresholds, split-conformal routing and FrugalGPT-style thresholds.", "key_idea": "Calibrate first, then threshold: cost-optimal escalation with guarantees.", "relevance_score": 4, "relevance_rationale": "Production-traffic evidence for calibrated cascades.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "kulkarni-2026-financial-mas-orchestration-benchmark", "title": "Benchmarking Multi-Agent LLM Architectures for Financial Document Processing: A Comparative Study of Orchestration Patterns, Cost-Accuracy Tradeoffs and Production Scaling Strategies", "authors": [ "Siddhant Kulkarni", "Yukta Kulkarni" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.22651", "code_url": null, "topic_clusters": [ "C12", "C13", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Reflexive loop F1 0.943 at 2.3x sequential cost; hierarchical supervisor-worker F1 0.921 at 1.4x cost; hybrid configs recover 89% of reflexive gains at 1.15x cost", "benchmarks": [ "10,000 SEC filings (10-K, 10-Q, 8-K)" ], "models_studied": [ "five frontier and open-weight LLMs" ], "production_evidence": false, "summary": "Benchmarks four orchestration architectures (sequential, parallel fan-out, hierarchical supervisor-worker, reflexive loop) for financial document extraction across five LLMs, measuring F1, latency, cost per document and tokens. Hierarchical designs sit best on the cost-accuracy frontier, and hybrids with caching, model routing and adaptive retry get most of the accuracy at little extra cost.", "key_idea": "Orchestration pattern choice drives cost-accuracy; hybrids with routing recover most gains cheaply.", "relevance_score": 4, "relevance_rationale": "Production-style cost comparison of MAS patterns, including model routing ablations.", "found_via": "OpenAlex search: orchestrator worker model cost", "cites_or_cited_by": [] }, { "id": "lai-2026-routejudge", "title": "RouteJudge: An Open Platform for Reproducible and Preference-Aware LLM Routing", "authors": [ "Guannan Lai", "Haoran Hu", "Han-Jia Ye" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.18774", "code_url": "https://github.com/LAMDA-Model-Reuse/ORBIT", "topic_clusters": [ "C1", "C5", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "anonymous pairwise user preferences", "cost", "latency" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "An online platform where several routing strategies each pick a model for the same query, and anonymous pairwise user preferences between the resulting responses are attributed back to the routers. Logs query, decisions, responses, preferences, cost and latency, and ships ORBIT, a toolbox for implementing and submitting routers.", "key_idea": "Arena-style live preference evaluation at the router level rather than the model level.", "relevance_score": 4, "relevance_rationale": "Closest thing to an A/B/interleaving harness for routers; logs are reusable for off-policy learning.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "lai-2026-routing-collapse", "title": "When Routing Collapses: On the Degenerate Convergence of LLM Routers", "authors": [ "Guannan Lai", "Han-Jia Ye" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.03478", "code_url": "https://github.com/AIGNLAI/EquiRouter", "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "learned model rankings", "predicted performance", "rankings" ], "cost_reduction_reported": "Reduces cost by about 17% at GPT-4-level performance vs strongest prior router on RouterBench (abstract)", "benchmarks": [ "RouterBench", "unimodal and multimodal routing benchmarks" ], "models_studied": [ "GPT-4", "router baselines", "EquiRouter" ], "production_evidence": false, "summary": "Identifies 'routing collapse': as budgets rise, routers default to the most expensive model even when cheaper ones suffice, due to training on scalar score prediction. EquiRouter learns rankings directly to restore use of small models.", "key_idea": "Decision-aware ranking objective fixes routers over-selecting the strongest model.", "relevance_score": 4, "relevance_rationale": "Explains a common production failure mode.", "found_via": "snowball: cited-by ong-2024-routellm | web search: SPROUT cost-aware routing benchmark", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "lai-2026-saverouter", "title": "Routing Should Pay for Itself: Sparse Supervision for Economical LLM Routing", "authors": [ "Guannan Lai", "Gelin Bian", "Hao-Xuan Ma", "Jun-Peng Jiang", "Long Chen", "Jian-Dong Liu", "Zhi-Hao Tan", "Han-Jia Ye" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.37402", "code_url": "https://github.com/LAMDA-Model-Reuse/SaveRouter", "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "sparse selectively-acquired feedback", "partial query-model feedback", "shared capability information" ], "cost_reduction_reported": "Uses ~33-41% of training feedback at competitive quality; reduces break-even deployment volume ~1.9-9.5x (abstract)", "benchmarks": [ "four routing benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "SaveRouter accounts for the upfront cost of collecting query-model feedback, acquiring only informative labels and sharing capability information across queries so the router pays back sooner.", "key_idea": "Account for supervision cost in router economics.", "relevance_score": 4, "relevance_rationale": "Enterprise ROI of building routers.", "found_via": "search: LLM router | snowball: cited-by wei-2025-barp", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "lee-2026-dart", "title": "DART: Draft-Agreement Routing for Training-Free Adaptive Thinking Budgets in Hybrid Reasoning Models", "authors": [ "Jungseob Lee", "Seongtae Hong", "Seungjun Lee", "Jaehyung Seo", "Junyoung Son", "Sugyeong Eo", "Chanjun Park", "Hyeongju Park", "Hyeonseok Moon", "Heuiseok Lim" ], "year": 2026, "venue_or_source": "EMNLP 2026 Findings", "source_type": "paper", "url": "https://arxiv.org/abs/2606.23181", "code_url": "https://github.com/js-lee-AI/DART", "topic_clusters": [ "C2", "C9" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "agreement of two no-think drafts", "draft entropy" ], "cost_reduction_reported": "Preserves or improves always-thinking accuracy while cutting thinking tokens by 32-73%; up to +9.0 points on Olympiad math and +22.5 on code", "benchmarks": [ "Olympiad-level math", "code" ], "models_studied": [ "0.6B-32B hybrid reasoning models", "API-hosted models" ], "production_evidence": false, "summary": "Samples two cheap no-think drafts and answers directly if they agree; otherwise predicts a thinking budget from draft entropy. Training-free and works with API-only models.", "key_idea": "Draft agreement decides think vs not, entropy sets the budget.", "relevance_score": 4, "relevance_rationale": "Training-free, API-compatible reasoning-mode cascade.", "found_via": "web search: routing between thinking and non-thinking modes", "cites_or_cited_by": [] }, { "id": "lee-2026-drift-aware-routing", "title": "Drift-Aware LLM Routing with Sparse Contexts and Shared Budgets", "authors": [ "Cheung Hao Lee", "Patrick Wong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.00662", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "sparse prompt embedding directions", "audit labels", "resource shadow prices" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formulates non-stationary budgeted routing with high-dimensional sparse contexts and drift in both request mix and model frontier; the algorithm uses a rolling audit window, pessimistic reward/optimistic cost estimates and online shadow prices, with regret bounds under bounded drift.", "key_idea": "Sparse, drift-aware contextual bandit routing with shared budgets and audit sampling.", "relevance_score": 4, "relevance_rationale": "Theory for drift and audit-rate trade-offs; highly relevant framing, little empirical evidence.", "found_via": "OpenAlex title: LLM routing drift", "cites_or_cited_by": [ "taberner-miller-2026-paretobandit" ] }, { "id": "lee-2026-routing-gap-task-type", "title": "Most of the LLM Routing Gap Is Task Type", "authors": [ "Janghoon Lee" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.23023", "code_url": null, "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "task type" ], "cost_reduction_reported": null, "benchmarks": [ "294 questions, 7 task types, 3 languages" ], "models_studied": [ "14 models" ], "production_evidence": false, "summary": "Argues most of the oracle routing gap is explained by task type, and that run-to-run noise (5.37% of model-question pairs flipping between identical runs) makes small router wins uninterpretable.", "key_idea": "Routing gains are mostly task-type effects plus noise.", "relevance_score": 4, "relevance_rationale": "Methodological caution for claimed routing gains.", "found_via": "web search: RouterEval benchmark routing LLMs", "cites_or_cited_by": [ "li-2026-llmrouterbench" ] }, { "id": "levit-2026-proxydrift", "title": "Privacy-Preserving Data Drift Detection and Recovery for Large-Scale LLM Applications via Proxy Representations", "authors": [ "Michael Levit", "Josh Ledgard", "Haoyu Dong", "Vishwas Suryanarayanan", "Eyal Kolman", "Sharon Tan", "Qiang Gan", "Vishal Chowdhary" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.08245", "code_url": null, "topic_clusters": [ "C5", "C6", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "LLM-derived proxy descriptors", "user satisfaction linkage" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Presents ProxyDrift, which measures drift between production traffic and offline evaluation sets using non-PII proxy descriptors produced by LLM classification of interactions, and refreshes evaluation sets with synthetic queries generated to match. Deployed for an application serving hundreds of millions of users, it also links proxy distributions to user satisfaction.", "key_idea": "Privacy-preserving drift monitoring and eval-set refresh from proxy representations of production traffic.", "relevance_score": 4, "relevance_rationale": "Production-scale method to keep offline evaluation (and thus router training data) aligned with live traffic.", "found_via": "query: \"drift\" production LLM monitoring evaluation", "cites_or_cited_by": [] }, { "id": "li-2026-afanet-gnn-attribution", "title": "Beyond LLM-Based Reasoning: Lightweight GNNs for Agent Failure Attribution", "authors": [ "Ting-Wei Li", "Yuanchen Bei", "Xiao Lin", "Hanghang Tong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.18575", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "graph neural network over traces" ], "cost_reduction_reported": "Near-zero inference cost while matching or beating LLM-based attribution baselines", "benchmarks": [ "Who&When" ], "models_studied": [], "production_evidence": false, "summary": "Frames agent failure attribution as graph learning and trains lightweight GNNs (AFANet) that match or exceed LLM-based attributors with far fewer parameters and near-zero inference cost.", "key_idea": "Cheap GNN replaces LLM judges for failure attribution.", "relevance_score": 4, "relevance_rationale": "Cheap enough to run on all production traces.", "found_via": "WebSearch: failure attribution multi-agent", "cites_or_cited_by": [ "zhang-2025-which-agent-when" ] }, { "id": "li-2026-counterfactual-tool-ranking", "title": "Counterfactual Tool Ranking under Utility, Cost, and Privilege Constraints", "authors": [ "Jiapeng Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.22819", "code_url": null, "topic_clusters": [ "C3", "C6" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "exact-propensity logs", "doubly robust", "direct regression" ], "cost_reduction_reported": null, "benchmarks": [ "BFCL" ], "models_studied": [], "production_evidence": false, "summary": "Evaluates counterfactual tool-selection policies from logs with exact propensities over eleven enterprise-style tools and MCP transport, plus 1,930 BFCL tasks. Finds that direct regression can beat doubly robust estimation in-distribution (MAE 0.0139 vs 0.0272) while DR is better under environment shift.", "key_idea": "Empirical comparison of OPE estimators for agent tool-choice policies, including under shift.", "relevance_score": 4, "relevance_rationale": "One of few OPE studies on agent action choices with propensity-logged data.", "found_via": "query: doubly robust estimator LLM evaluation", "cites_or_cited_by": [] }, { "id": "li-2026-dynamic-mixed-precision-routing", "title": "Dynamic Mixed-Precision Routing for Efficient Multi-step LLM Interaction", "authors": [ "Yuanzhe Li", "Jianing Deng", "Jingtong Hu", "Tianlong Chen", "Song Wang", "Huanrui Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.02711", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "KL-divergence precision sensitivity" ], "cost_reduction_reported": "Strong accuracy-cost trade-off over single-precision baselines", "benchmarks": [ "ALFWorld", "WebShop" ], "models_studied": [], "production_evidence": false, "summary": "DMR routes each decision step of a long-horizon agent between a high-precision and a low-precision quantized LLM, trained with KL-based supervision to find precision-sensitive steps and then GRPO. It improves the accuracy-cost trade-off.", "key_idea": "Per-step routing between full-precision and quantized models.", "relevance_score": 4, "relevance_rationale": "Step-level downscaling via quantization rather than model size.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "li-2026-single-agent-skills-vs-mas", "title": "When Single-Agent with Skills Replace Multi-Agent Systems and When They Fail", "authors": [ "Xiaoxiao Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.04748", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "skill selection accuracy" ], "cost_reduction_reported": "Substantially reduced token usage and latency while maintaining competitive accuracy (preliminary)", "benchmarks": [ "reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Argues a multi-agent system can be compiled into a single agent that selects from a skill library, trading communication for skill selection, and finds preliminary token and latency savings. Skill-selection accuracy shows a sharp phase transition as the library grows, which hierarchical routing mitigates.", "key_idea": "Compile MAS into a single agent with skills; watch the skill-library capacity limit.", "relevance_score": 4, "relevance_rationale": "Architecture-level alternative to MAS with cost savings.", "found_via": "web search: scaling agents cost", "cites_or_cited_by": [ "chen-2024-optima", "kim-2025-science-of-scaling-agent-systems", "yue-2025-masrouter" ] }, { "id": "liao-2026-hyfunc", "title": "HyFunc: Accelerating LLM-based Function Calls for Agentic AI through Hybrid-Model Cascade and Dynamic Templating", "authors": [ "Weibin Liao", "Jian-guang Lou", "Haoyi Xiong" ], "year": 2026, "venue_or_source": "KDD 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.13665", "code_url": "https://github.com/MrBlankness/HyFunc", "topic_clusters": [ "C2", "C3", "C11" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "soft-token intent from large model" ], "cost_reduction_reported": "Inference latency 0.828 s, outperforming all baselines, with 80.1% on BFCL", "benchmarks": [ "BFCL" ], "models_studied": [], "production_evidence": false, "summary": "For function calling, a large model compresses intent into one soft token that guides a retriever and a small prefix-tuned model to produce the call, with templated boilerplate injected in vLLM.", "key_idea": "Large model plans, small model emits the tool call.", "relevance_score": 4, "relevance_rationale": "Split of tool-call generation between models in agents.", "found_via": "arXiv title search: cascade LLM", "cites_or_cited_by": [] }, { "id": "liu-2026-copilot-traces", "title": "Agentic Coding in the Wild: Characterizing GitHub Copilot Traces at Production Scale", "authors": [ "Banruo Liu", "Haoran Qiu", "Íñigo Goiri", "Rodrigo Fonseca", "Ricardo Bianchini", "Esha Choukse" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.00101", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "idle-time prediction" ], "cost_reduction_reported": "a lightweight idle-time predictor captures 86-90% of total user idle time for proactive resource decisions", "benchmarks": [ "GitHub Copilot production traces (3.2M users, 13M sessions, 761M LLM calls, 95T tokens)" ], "models_studied": [], "production_evidence": true, "summary": "First production-scale characterization of agentic coding workloads from sampled GitHub Copilot traces: sessions are sparse user turns that each expand into long autonomous loops of LLM calls coupled with tools, with long-tailed token use and minutes-long idle gaps at turn boundaries that challenge chatbot-oriented serving assumptions.", "key_idea": "Production agentic coding traffic looks nothing like chat; design serving around loops and idle gaps.", "relevance_score": 4, "relevance_rationale": "Rare production-scale evidence on agent workload shape (tokens, calls, idleness).", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "chang-2026-agentic-workload-characterization", "xia-2026-mori" ] }, { "id": "liu-2026-hcp-mad", "title": "HCP-MAD:Heterogeneous Consensus-Progressive Reasoning for Efficient Multi-Agent Debate", "authors": [ "Yiqing Liu", "Hantao Yao", "Wu Liu", "Allen He", "Yongdong Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.09679", "code_url": "https://github.com/fuyu66/HCP-MAD", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "heterogeneous pair consensus" ], "cost_reduction_reported": "Improves accuracy while substantially reducing token costs across six benchmarks", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "HCP-MAD first checks consensus between two heterogeneous agents and stops early, then runs a short pair debate, and escalates only unresolved tasks to collective voting with more agents, adapting token spend to difficulty.", "key_idea": "Progressive escalation from pair consensus to full debate.", "relevance_score": 4, "relevance_rationale": "Escalation within MAS for cost.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems" ] }, { "id": "liu-2026-treecredit", "title": "Shared Prefixes, Better Credit: Adaptive Routing for Multi-Agent Reasoning", "authors": [ "Yiqing Liu", "Zihao Wang", "Hantao Yao", "Wu Liu", "Yongdong Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.02291", "code_url": null, "topic_clusters": [ "C3", "C6" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "shared-prefix suffix credit", "cumulative cost" ], "cost_reduction_reported": "Modestly improves accuracy while substantially reducing inference cost on six reasoning benchmarks", "benchmarks": [ "six reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "TreeCredit builds shared-prefix collaboration trees from the same intermediate state to credit each operator by correctness and added cost of its continuation, then trains a lightweight pairwise state router that picks the next operator in multi-agent reasoning.", "key_idea": "State-matched credit assignment for per-step operator routing.", "relevance_score": 4, "relevance_rationale": "Better training signal for step-level MAS routing.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "ke-2026-mas-orchestra", "su-2025-daao", "wang-2026-oi-mas" ] }, { "id": "liu-2026-vllm-semantic-router", "title": "vLLM Semantic Router: Signal Driven Decision Routing for Mixture-of-Modality Models", "authors": [ "Xunzhuo Liu", "Huamin Chen", "Samzong Lu", "Yossi Ovadia", "Guohong Wen", "Hao Wu", "Zhengda Tan", "Jintao Zhang", "Senan Zedan", "Yehudit Kerido", "Liav Weiss", "Haichen Zhang", "Bishen Yu", "Asaad Balum", "Noa Limoy", "Abdallah Samara", "Baofa Fan", "Brent Salisbury", "Ryan Cook", "Zhijie Wang", "Qiping Pan", "Rehan Khan", "Avishek Goswami", "Houston H. Zhang", "Shuyi Wang", "Ziang Tang", "Fang Han", "Zohaib Hassan", "Jianqiao Zheng", "Avinash Changrani", "Xue", "Liu", "Bowei He" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "product", "url": "https://arxiv.org/abs/2603.04444", "code_url": null, "topic_clusters": [ "C1", "C11", "C14" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "13 signal types", "Boolean decision rules", "neural classifiers" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Technical report of vLLM Semantic Router, which composes heterogeneous signals (heuristics, classifiers for semantics, safety, modality) via configurable Boolean rules into deployment-specific routing policies, with plugin chains for hallucination detection and memory.", "key_idea": "Composable signal-driven routing for production gateways.", "relevance_score": 4, "relevance_rationale": "Open-source production-oriented router.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "lumer-2026-dont-break-the-cache", "title": "Don't Break the Cache: An Evaluation of Prompt Caching for Long-Horizon Agentic Tasks", "authors": [ "Elias Lumer", "Faheem Nizar", "Akshaya Jangiti", "Kevin Frank", "Anmol Gulati", "Mandar Phadate", "Vamse Kumar Subbiah" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.06007", "code_url": null, "topic_clusters": [ "C10", "C13", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "cache block placement strategy" ], "cost_reduction_reported": "prompt caching reduced API cost by 41-80% and improved time-to-first-token by 13-31% across providers; naive full-context caching can paradoxically increase latency", "benchmarks": [ "DeepResearch Bench" ], "models_studied": [ "OpenAI, Anthropic and Google models" ], "production_evidence": false, "summary": "Evaluates provider prompt caching for long-horizon agents over 500+ sessions with 10,000-token system prompts, comparing full-context caching, system-prompt-only caching and caching that excludes dynamic tool results. Placing dynamic content at the end and excluding dynamic tool results gives more consistent gains than caching everything.", "key_idea": "How you place cache breakpoints in agent prompts decides whether caching helps.", "relevance_score": 4, "relevance_rationale": "Directly quantifies caching savings for multi-turn agentic workloads on frontier APIs.", "found_via": "\"Don't Break the Cache\" prompt caching agentic tasks evaluation", "cites_or_cited_by": [ "anthropic-2024-prompt-caching", "google-2024-gemini-context-caching", "khailo-2026-keepalive-economics", "openai-2024-prompt-caching", "xu-2026-tokenpilot" ] }, { "id": "luo-2026-agentark", "title": "AgentArk: Distilling Multi-Agent Intelligence into a Single LLM Agent", "authors": [ "Yinyi Luo", "Yiqiao Jin", "Weichen Yu", "Mengqi Zhang", "Srijan Kumar", "Xiaoxiao Li", "Weijie Xu", "Xin Chen", "Jindong Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.03955", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "multi-agent debate traces" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Distills multi-agent debate dynamics into a single LLM agent, shifting computation from inference to training while keeping reasoning and self-correction benefits.", "key_idea": "Single agent inherits multi-agent intelligence.", "relevance_score": 4, "relevance_rationale": "Removes MAS inference cost.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "malik-2026-progressive-crystallization", "title": "Progressive Crystallization: Turning Agent Exploration into Deterministic, Lower-Cost Workflows in Production", "authors": [ "Arun Malik" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.07052", "code_url": null, "topic_clusters": [ "C8", "C12", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "repeated validated agent behaviors" ], "cost_reduction_reported": "Deterministic execution rose 0% to 45% over eight months; per-incident agent cost reduced >70% despite doubling incident volume (production cloud-networking AIOps)", "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Progressive crystallization promotes repeatedly validated agent behaviors in a production IT-ops system into deterministic workflows, and demotes them on regression. Over eight months on tens of thousands of incidents per month it shifted executions off the LLM.", "key_idea": "Graduate agent trajectories into deterministic workflows to remove LLM calls.", "relevance_score": 4, "relevance_rationale": "Real production evidence for learning from traces to cut agent cost.", "found_via": "arXiv query ti:\"agent\" AND ti:\"production\" AND abs:\"cost\" AND abs:\"model\"", "cites_or_cited_by": [] }, { "id": "manoharan-2026-t2mo", "title": "Task-to-Model Optimization for Enterprise LLM Coding Assistants: A Data-Driven Framework for Cost-Optimal Routing", "authors": [ "Srinivasan Manoharan", "Junhua Zhao", "Fangbo Tu", "Haifeng Wu", "Jian Wan", "Maliah Rajan M", "Ashwin Hegde", "Mithun Sasidharan", "Kalyan Chakravarthi Podamekala" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.08528", "code_url": null, "topic_clusters": [ "C1", "C6", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "telemetry", "task taxonomy", "difficulty grading", "cost per completed task" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Describes a nine-stage methodology for cost-optimal model routing in enterprise coding assistants, optimizing cost per completed task including escalations, with shadow-mode deployment and governance.", "key_idea": "Cost-per-completed-task objective for enterprise routing.", "relevance_score": 4, "relevance_rationale": "Enterprise methodology (no empirical results reported).", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "marjanovic-2026-model-pools-mas", "title": "Mo'Models, Mo'Problems: How to best select model pools when designing Multi-Agent Systems", "authors": [ "Sara Vera Marjanovic", "Jia-Chen Xu", "A. Laptev", "Grigor Nalbandyan", "Erik Arakelyan", "Evelina Bakhaturina" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.17306", "code_url": null, "topic_clusters": [ "C3", "C12", "C13" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "model size", "accuracy", "answer diversity" ], "cost_reduction_reported": null, "benchmarks": [ "scientific reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Evaluates eight strategies for picking candidate models for multi-agent systems, for routing-based and aggregation-based (voting, judge) architectures. Growing the candidate pool often pushes performance below the best single model; selecting within one model family worked best.", "key_idea": "Bigger heterogeneous model pools often hurt MAS; select candidates carefully.", "relevance_score": 4, "relevance_rationale": "Cautionary evidence for heterogeneous model assignment.", "found_via": "snowball: cited-by su-2025-toolorchestra", "cites_or_cited_by": [ "su-2025-toolorchestra" ] }, { "id": "mia-2026-adaptive-reasoning-agentic-ai", "title": "Don't Overthink, Don't Underthink: Toward Adaptive Reasoning in Agentic AI", "authors": [ "Md Jueal Mia", "M. Hadi Amini" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.26442", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "tool-decision latency", "token consumption", "token-limit exhaustion" ], "cost_reduction_reported": null, "benchmarks": [ "MATH-500", "GAIA validation" ], "models_studied": [], "production_evidence": false, "summary": "Argues reasoning needs in agents change during planning, tool use and multi-agent interaction, so fixed budgets misallocate compute. Characterises over- and under-reasoning on MATH-500 and GAIA: over-reasoning raises cost without accuracy gains, under-reasoning yields incomplete answers.", "key_idea": "Reasoning must be allocated dynamically across an agent workflow.", "relevance_score": 4, "relevance_rationale": "Agent-specific framing of reasoning budget misallocation.", "found_via": "arXiv title search: overthinking agents", "cites_or_cited_by": [] }, { "id": "mitra-2026-mosaic-moa-scheduling", "title": "MOSAIC: Efficient Mixture-of-Agent Scheduling via Adaptive Aggregation and Inference Concurrency", "authors": [ "Saptarshi Mitra", "Yifan Zhang", "Rachid Karami", "Phyo Pyae Moe Aung", "Nazmul Takbir", "Sreetama Sarkar", "Souvik Kundu", "Sitao Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.03014", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "inter-expert agreement", "offline-profiled costs" ], "cost_reduction_reported": "Up to 2.5x expert-stage, 4.23x aggregator-stage and 1.7-2.3x end-to-end speedups vs baseline scheduler, accuracy within 0.1pp (4-GPU system)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MOSAIC schedules Mixture-of-Agents workloads with an ILP that places experts and assigns prompts per worker using offline-profiled costs. It skips the heavy aggregator LLM when experts already agree.", "key_idea": "Agreement-gated aggregator skipping plus ILP scheduling for MoA.", "relevance_score": 4, "relevance_rationale": "MoA cost study with a selective-skip mechanism (sub-theme g).", "found_via": "arXiv query ti:\"mixture-of-agents\" AND abs:cost", "cites_or_cited_by": [] }, { "id": "mohamed-2026-workload-aware-caching", "title": "Workload-Aware Caching for Multi-Agent Systems", "authors": [ "Anas Mohamed", "Kaizan Haque", "Azal Ahmad Khan", "Chetan Sharma", "Shuwen Ge", "Ali Anwar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.20495", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "recomputation cost", "DAG dependency count", "agent invocation frequency" ], "cost_reduction_reported": "up to 64.7% latency reduction vs uncached baseline and 31.1% average latency reduction over the next-best finite-capacity cache, with accuracy on par or better", "benchmarks": [ "three multi-agent benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "A content-cache replacement policy for multi-agent systems that scores entries by recomputation cost, dependency count in the agent DAG and invocation frequency. It is shown to be complementary to plan caching and parallel agent execution.", "key_idea": "Evict multi-agent cache entries by value to the workflow, not recency.", "relevance_score": 4, "relevance_rationale": "Caching policy designed specifically for multi-agent workflows.", "found_via": "Cost-Efficient Serving of LLM Agents via Test-Time Plan Caching", "cites_or_cited_by": [ "zhang-2025-agentic-plan-caching" ] }, { "id": "moslem-2026-cluster-route-escalate", "title": "Cluster, Route, Escalate: Cascaded Framework for Cost-Aware LLM Serving", "authors": [ "Yasmin Moslem", "Magdalena Kacmajor", "Vasudevan Nedumpozhimana", "Ammar Abbas", "Solmaz Panahi", "David Lynch", "Zhuangzhuang Nie", "Alexandros Agapitos", "Aleksandar Milenovic", "Hongmeng Song", "Yucheng Shi", "Yue Pan", "Patricia Buffini", "John D. Kelleher" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.27457", "code_url": null, "topic_clusters": [ "C1", "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query clusters", "quality estimation" ], "cost_reduction_reported": "Retains 97-99% of the strongest model's accuracy while reducing time per output token", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Stage one clusters queries and assigns each cluster its most cost-effective model under a tunable budget; stage two escalates outputs judged low-quality by a quality estimator. Needs only task-correctness labels and adapts to model pool changes.", "key_idea": "Cluster-level routing followed by QE-triggered escalation.", "relevance_score": 4, "relevance_rationale": "Practical two-stage route-then-cascade recipe.", "found_via": "arXiv title search: cascade LLM", "cites_or_cited_by": [] }, { "id": "moslem-2026-routing-cascading-survey", "title": "Dynamic Model Routing and Cascading for Efficient LLM Inference: A Survey", "authors": [ "Yasmin Moslem", "John D. Kelleher" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.04445", "code_url": null, "topic_clusters": [ "C1", "C2", "C5", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Systematic survey of multi-LLM routing and cascading across independently trained models, organized by paradigm (difficulty, preferences, clustering, uncertainty, RL, multimodality, cascading). Adds a conceptual framework for characterizing routing systems and discusses trade-offs.", "key_idea": "Most up-to-date taxonomy of routing and cascading.", "relevance_score": 4, "relevance_rationale": "Core C15 survey for query-level routing.", "found_via": "snowball: cited-by ong-2024-routellm | arXiv title search: cascade LLM | snowball: cited-by panda-2025-pilot", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "ni-2026-chimera", "title": "Chimera: Latency- and Performance-Aware Multi-agent Serving for Heterogeneous LLMs", "authors": [ "Kangqi Ni", "Wenyue Hua", "Xiaoxiang Shi", "Jiang Guo", "Shiyu Chang", "Tianlong Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.22206", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "semantic routing confidence", "predicted remaining output length", "congestion" ], "cost_reduction_reported": "End-to-end latency reduced 1.2-2.4x and task performance +8.0-9.5 pp on average vs baselines incl. vLLM", "benchmarks": [ "code generation and math reasoning workflows" ], "models_studied": [], "production_evidence": false, "summary": "Chimera schedules multi-agent workflow stages across a heterogeneous LLM cluster. It combines semantic routing for per-model confidence, prediction of remaining workflow output length, and congestion estimates for load balancing.", "key_idea": "Joint per-stage model routing and load balancing for heterogeneous MAS serving.", "relevance_score": 4, "relevance_rationale": "Per-stage heterogeneous model assignment at the serving layer.", "found_via": "arXiv query ti:heterogeneous AND ti:\"multi-agent\" AND ti:llm", "cites_or_cited_by": [] }, { "id": "ning-2026-hrbench", "title": "HRBench: Benchmarking and Understanding Thinking-Mode Switch Strategies in Hybrid-Reasoning LLMs", "authors": [ "Yansong Ning", "Mianpeng Liu", "Jingwen Ye", "Weidong Zhang", "Hao Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.28398", "code_url": "https://github.com/usail-hkust/HRBench", "topic_clusters": [ "C9", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "prompt-based selection", "external routing", "speculative execution" ], "cost_reduction_reported": null, "benchmarks": [ "5 reasoning benchmarks (math, science, code)" ], "models_studied": [ "Qwen3.5-2B", "Kimi-K2.5-1.1T", "6 LLMs total" ], "production_evidence": false, "summary": "Unified benchmark for thinking-mode switching in hybrid-reasoning LLMs, reimplementing 12+ methods across three strategy families and four training regimes. Prompt-based methods give good token-accuracy trade-offs, routing is more stable, speculative methods cost more.", "key_idea": "Controlled comparison of think/no-think switching strategies.", "relevance_score": 4, "relevance_rationale": "Benchmark for reasoning-mode routing.", "found_via": "arXiv title search: hybrid thinking", "cites_or_cited_by": [] }, { "id": "oh-2026-progress-advantage", "title": "Neglected Free Lunch from Post-training: Progress Advantage for LLM Agents", "authors": [ "Changdae Oh", "Wendi Li", "Seongheon Park", "Samuel Yeh", "Tanwi Mallick", "Sharon Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.26080", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "policy log-prob progress advantage" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows RL-post-trained agent policies implicitly provide a step-level progress signal (progress advantage) that outperforms confidence baselines and trained reward models without extra training.", "key_idea": "Free process reward from the post-trained policy itself.", "relevance_score": 4, "relevance_rationale": "Zero-cost step-level signal.", "found_via": "hf-papers search: uncertainty quantification LLM agents", "cites_or_cited_by": [] }, { "id": "peng-2026-inference-economics-coding-agents", "title": "Inference Economics of Enterprise Coding Agents: A Case Study of Cloud vs. On-Premise LLMs", "authors": [ "Sheng-Wei Peng", "Yi-Hsun Lin", "Yi-Pei Lee" ], "year": 2026, "venue_or_source": "arXiv (industrial case study)", "source_type": "paper", "url": "https://arxiv.org/abs/2607.13080", "code_url": null, "topic_clusters": [ "C10", "C12", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM telemetry", "Git history" ], "cost_reduction_reported": "Prompt caching (99.3% hit rate) cut realized API cost 88.6% to $0.57 per million tokens; on-prem saves 40.1% TCO under shared GPUs but dedicated reservation costs 43.8% more; local config had fix-commit ratio 74.9% vs 45.9%", "benchmarks": [ "production monorepo, two 28-day periods" ], "models_studied": [ "Claude Opus 4.7/4.8 (Claude Code)", "GLM-5.1/5.2 (Opencode, NVFP4)" ], "production_evidence": true, "summary": "A longitudinal single-developer case study on a production monorepo comparing API Claude Opus in Claude Code against on-prem quantized GLM in Opencode. Caching makes the API cheaper per token than on-prem, the local model causes far more repair commits, and hybrid routing gateways trade defect rate for savings rather than dominating.", "key_idea": "Real telemetry: caching reshapes cloud vs on-prem agent economics.", "relevance_score": 4, "relevance_rationale": "Production evidence on coding-agent cost levers and routing trade-offs.", "found_via": "snowball: cited-by wang-2025-efficient-agents", "cites_or_cited_by": [ "kapoor-2024-ai-agents-that-matter" ] }, { "id": "pham-2026-agentstop", "title": "AgentStop: Terminating Local AI Agents Early to Save Energy in Consumer Devices", "authors": [ "Dzung Pham", "Kleomenis Katevas", "Ali Shahin Shamsabadi", "Hamed Haddadi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.15206", "code_url": "https://github.com/brave-experiments/AgentStop", "topic_clusters": [ "C6", "C9", "C12" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "token log probabilities", "execution signals" ], "cost_reduction_reported": "Reduces wasted energy by 15-20% with <5% utility drop on web QA and coding benchmarks", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Measures the time, token and energy overhead of local LLM agents on consumer hardware and introduces AgentStop, a lightweight supervisor that predicts from cheap execution signals which trajectories will fail and terminates them early.", "key_idea": "Predict failing trajectories from cheap signals and stop them early to save compute.", "relevance_score": 4, "relevance_rationale": "Success prediction applied directly to cut agent cost.", "found_via": "query: abs:\"predict\" AND abs:\"trajectory\" AND abs:\"failure\" AND abs:\"LLM agent\"", "cites_or_cited_by": [] }, { "id": "phillips-2026-when-to-use-multi-agent-systems", "title": "Building multi-agent systems: When and how to use them", "authors": [ "Cara Phillips" ], "year": 2026, "venue_or_source": "Claude blog (Anthropic)", "source_type": "blog", "url": "https://claude.com/blog/building-multi-agent-systems-when-and-how-to-use-them", "code_url": null, "topic_clusters": [ "C12", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "States multi-agent systems typically use 3-10x more tokens than single-agent approaches for equivalent tasks", "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Anthropic guidance recommending single agents by default and multi-agent designs only for context protection, parallelization or specialization, since coordination costs otherwise exceed benefits. Reports that multi-agent systems usually spend 3-10x more tokens due to duplicated context, coordination messages and summarization, and advises decomposing by context boundaries.", "key_idea": "Use multi-agent only for context isolation, parallelism or specialization; otherwise it costs 3-10x more tokens.", "relevance_score": 4, "relevance_rationale": "Practitioner guidance grounded in customer deployments on when MAS is worth its cost.", "found_via": "web search: Anthropic multi-agent research system", "cites_or_cited_by": [] }, { "id": "rabanser-2026-agent-reliability", "title": "Towards a Science of AI Agent Reliability", "authors": [ "Stephan Rabanser", "Sayash Kapoor", "Peter Kirgis", "Kangheng Liu", "Saiteja Utpala", "Arvind Narayanan" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.16666", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "consistency", "robustness", "predictability", "safety metrics" ], "cost_reduction_reported": null, "benchmarks": [ "multiple agent benchmarks" ], "models_studied": [ "frontier agents" ], "production_evidence": false, "summary": "Proposes twelve metrics that break agent reliability into consistency, robustness, predictability and safety, borrowing from safety-critical engineering, and shows single success rates hide operational flaws.", "key_idea": "Multi-dimensional reliability profile beyond accuracy.", "relevance_score": 4, "relevance_rationale": "Reliability metrics needed to validate cheaper configurations.", "found_via": "web search: agent reliability metrics", "cites_or_cited_by": [ "kapoor-2025-hal-holistic-agent-leaderboard" ] }, { "id": "rainone-2026-hybrid-cloud-device-mas-lessons", "title": "When Cloud Agents Meet Device Agents: Lessons from Hybrid Multi-Agent Systems", "authors": [ "Corrado Rainone", "Davide Belli", "Bence Major", "Arash Behboodi" ], "year": 2026, "venue_or_source": "arXiv (workshop)", "source_type": "paper", "url": "https://arxiv.org/abs/2605.30102", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Qualcomm researchers adapt two MAS architectures for hybrid on-device SLM and cloud LLM inference and map how design choices move systems along the Pareto frontier of power, cost and accuracy. SLMs benefit from LLM help, but the best hybrid architecture is task-dependent and more frontier compute does not reliably help.", "key_idea": "Systematic design-space study of hybrid SLM/LLM multi-agent systems.", "relevance_score": 4, "relevance_rationale": "Evidence on role-level placement of small vs large models.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai", "kim-2025-science-of-scaling-agent-systems", "su-2025-toolorchestra", "zhang-2024-agentprune" ] }, { "id": "raj-2026-trace-router", "title": "TRACE-ROUTER: Task-Consistent and Adaptive Online Routing for Agentic AI", "authors": [ "Ritik Raj", "Souvik Kundu", "Sarbartha Banerjee", "Dheemanth Joshi", "Ishita Vohra", "Tushar Krishna" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.22465", "code_url": null, "topic_clusters": [ "C1", "C3", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "terminal task reward", "latency" ], "cost_reduction_reported": "On tau2-Bench beats latency-matched interpolation between models by 7-8 accuracy points; non-dominated Pareto points on three agentic benchmarks", "benchmarks": [ "tau2-Bench", "Terminal-Bench", "third agentic benchmark" ], "models_studied": [], "production_evidence": false, "summary": "TRACE-Router argues per-call routers cannot attribute delayed task rewards, so it assigns each agentic task to a model once at admission with a contextual bandit, pins all calls to it and learns from terminal reward and latency.", "key_idea": "Task-level bandit routing aligned with delayed agent rewards.", "relevance_score": 4, "relevance_rationale": "Counterpoint to per-step routing: task-level pinning with online learning.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "son-2026-swe-router" ] }, { "id": "rajput-2026-cheap-verifiers-blind-spots", "title": "Cheap Verifiers, Large Blind Spots: Measuring the Reliability Cost of Cost-Saving Cascades", "authors": [ "Dushyant Rajput", "Nirdesh Chauhan", "Siddharth Kosaraju" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.01345", "code_url": null, "topic_clusters": [ "C2", "C7", "C8" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "verifier acceptance" ], "cost_reduction_reported": "Frontier verifier drives blind spot to ~0.05 but escalates 46% of hard-MATH queries vs a 39% true error rate; cascade dashboard shows flat 3% error while true error swings up to 32%", "benchmarks": [ "MATH" ], "models_studied": [ "students 0.5B-32B" ], "production_evidence": false, "summary": "Measures self-improving cascades where the cheap model is fine-tuned on verifier rejections. Finds verifier blind spots grow with student capability, corrective fine-tuning collapses the student, and in-loop metrics hide real degradation.", "key_idea": "Metrics computed through a cheap verifier cannot certify a cascade.", "relevance_score": 4, "relevance_rationale": "Important caution for closed-loop cascades using judges.", "found_via": "snowball: cited-by AutoMix (2310.12963)", "cites_or_cited_by": [ "aggarwal-2023-automix" ] }, { "id": "ruan-2026-aorchestra", "title": "AOrchestra: Automating Sub-Agent Creation for Agentic Orchestration", "authors": [ "Jianhao Ruan", "Zhihao Xu", "Yiran Peng", "Fashen Ren", "Zhaoyang Yu", "Xinbing Liang", "Jinyu Xiang", "Yongru Chen", "Bang Liu", "Chenglin Wu", "Yuyu Luo", "Jiayi Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.03786", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "per-step curation of instruction, context, tools, model" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AOrchestra models any sub-agent as an (instruction, context, tools, model) tuple that the orchestrator fills at each step, spawning specialized executors on demand and choosing the model for each.", "key_idea": "Orchestrator picks the model (and context/tools) for each spawned sub-agent.", "relevance_score": 4, "relevance_rationale": "Per-subagent model choice in orchestration.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "gao-2025-single-or-multi-agent-why-not-both", "su-2025-toolorchestra" ] }, { "id": "ryser-2026-shipping-cost-mas", "title": "Measuring Tradeoffs in Multi-Agent Models: a Case Study in Shipping Cost Calculation", "authors": [ "André Ryser", "Paolo Rosso", "Julien Audiffren", "Simon Ruffieux", "Philippe Goetschmann", "P. Cudré-Mauroux" ], "year": 2026, "venue_or_source": "Swiss Conference on Data Science (SDS) 2026", "source_type": "paper", "url": "https://doi.org/10.1109/SDS70563.2026.00040", "code_url": null, "topic_clusters": [ "C3", "C12", "C14" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "An industrial case study that decomposes a customer-facing shipping-cost agent into sub-tasks so that smaller open-weight models can handle them in a multi-agent system. With optimization, small models are a cost-effective alternative to frontier models in low-risk settings.", "key_idea": "Decompose an industrial agent so small models can serve subtasks.", "relevance_score": 4, "relevance_rationale": "Industrial mixed-model agent case study (sub-theme h).", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [] }, { "id": "saleh-2026-illusion-agentic-complexity-readme", "title": "The Illusion of Agentic Complexity in README.md Generation: Evaluating Single-Agent vs. Multi-Agent RAG Systems", "authors": [ "Abu Saleh", "Tesfay Welegebreal Tesfay", "Phuong T. Nguyen", "Juri Di Rocco", "Muhammad Umar Zeshan", "Davide Di Ruscio" ], "year": 2026, "venue_or_source": "ICSME 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2606.30524", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Single-agent pipeline matches MAS lexical quality while using 86% fewer tokens at twice the speed", "benchmarks": [ "README generation for GitHub repositories" ], "models_studied": [], "production_evidence": false, "summary": "Compares a single-agent RAG pipeline, a specialized MAS and a developer-guided planning variant for README generation. The single agent matches lexical quality with 86% fewer tokens, while MAS gives more consistent structure and developer-guided plans give the best quality.", "key_idea": "Single agent matches MAS quality at a fraction of tokens on a real SE task.", "relevance_score": 4, "relevance_rationale": "Concrete single- vs multi-agent cost evidence.", "found_via": "OpenAlex search: single vs multi-agent tokens", "cites_or_cited_by": [ "gao-2025-single-or-multi-agent-why-not-both" ] }, { "id": "selvaraj-2026-adept", "title": "ADEPT: adaptive delegation of expert prompts to trained SLMs for agentic AI use cases", "authors": [ "Suganya Selvaraj", "Gautam Tripathi", "Chetana Amancharla" ], "year": 2026, "venue_or_source": "International Conference on Artificial Intelligence, Automation, and Algorithms (SPIE) 2026", "source_type": "paper", "url": "https://doi.org/10.1117/12.3097560", "code_url": null, "topic_clusters": [ "C3", "C8", "C14" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "novelty/complexity of prompt" ], "cost_reduction_reported": "41.5% reduction in LLM calls and 39.8% cost savings over two months in ITOps and cybersecurity", "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "ADEPT starts with LLMs answering novel prompts and progressively fine-tunes SLMs on expert outputs to take over repetitive, well-understood tasks, with routing between them. It was evaluated over two months in IT operations and cybersecurity.", "key_idea": "Progressive LLM-to-SLM delegation trained on the LLM's own production outputs.", "relevance_score": 4, "relevance_rationale": "Production-style evidence for learning delegation from traffic.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [] }, { "id": "sethi-2026-trace-action-agreement", "title": "Don't Overthink It: Inter-Rollout Action Agreement as a Free Adaptive-Compute Signal for LLM Agents", "authors": [ "Khushal Sethi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.08369", "code_url": null, "topic_clusters": [ "C3", "C7", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "inter-rollout action agreement" ], "cost_reduction_reported": "TrACE-4 matches SC-4 accuracy with 33% (GSM8K) and 39% (MiniHouse) fewer LLM calls; TrACE-8 matches SC-8 with 55% and 65% fewer calls", "benchmarks": [ "GSM8K (n=50)", "MiniHouse (n=30)", "GSM8K", "MiniHouse" ], "models_studied": [ "Qwen 2.5 3B Instruct" ], "production_evidence": false, "summary": "At each agent step, samples a few candidate actions and commits immediately if they agree, otherwise samples more up to a cap. Training-free per-timestep adaptive compute; small-scale evaluation.", "key_idea": "Action agreement across rollouts sets per-step compute.", "relevance_score": 4, "relevance_rationale": "Per-step adaptive compute for agents (small study).", "found_via": "arXiv title search: overthinking agents | web search: adaptive compute per step", "cites_or_cited_by": [] }, { "id": "shaikh-2026-bayesian-self-escalation", "title": "Knowing When to Ask for Help: Bayesian Self-Escalation in Hierarchical LLM Agents", "authors": [ "Nadeem Shaikh" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.24087", "code_url": null, "topic_clusters": [ "C2", "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "learned competence posterior during generation", "learned competence posterior" ], "cost_reduction_reported": "On a Qwen2.5-Coder 1.5B->7B code cascade (MBPP, 257 tasks) the escalation frontier dominates post-hoc routing at equal cost", "benchmarks": [ "MBPP", "MBPP (Qwen2.5-Coder 1.5B to 7B cascade)", "simulation" ], "models_studied": [ "Qwen2.5-Coder 1.5B", "Qwen2.5-Coder 7B" ], "production_evidence": false, "summary": "Treats intra-generation delegation as Bayesian optimal stopping over a learned belief about eventual task success, and proves the optimal policy is a time-varying threshold with regret and sample guarantees. Validated in simulation and a small real code cascade.", "key_idea": "Escalate mid-reasoning via optimal stopping on a learned competence belief.", "relevance_score": 4, "relevance_rationale": "Third regime between pre-routing and post-hoc verification; relevant to agent steps.", "found_via": "arXiv title search: escalation LLM | web search: uncertainty-triggered escalation", "cites_or_cited_by": [] }, { "id": "shankar-2026-task-cascades", "title": "Task Cascades for Efficient Unstructured Data Processing", "authors": [ "Shreya Shankar", "Sepanta Zeighami", "Aditya Parameswaran" ], "year": 2026, "venue_or_source": "SIGMOD 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.05536", "code_url": null, "topic_clusters": [ "C2", "C4" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "proxy-model confidence", "agent-generated surrogate tasks" ], "cost_reduction_reported": "At 90% target accuracy, reduces end-to-end cost by an average of 36% vs model cascades across eight real-world document tasks", "benchmarks": [ "eight document processing tasks" ], "models_studied": [], "production_evidence": false, "summary": "Generalises model cascades by also varying the document portion and the operation at each stage, with an LLM agent proposing simplified or related tasks. Offers statistical accuracy guarantees.", "key_idea": "Cascade over (model, input slice, surrogate task).", "relevance_score": 4, "relevance_rationale": "Cascade design search akin to configuration optimisation.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "si-2026-policy-guided-stepwise-routing", "title": "Policy-Guided Stepwise Model Routing for Cost-Effective Reasoning", "authors": [ "Wenwen Si", "Insup Lee", "Osbert Bastani" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06116", "code_url": null, "topic_clusters": [ "C2", "C3", "C9" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "CoT state", "threshold calibration" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K", "MATH500", "OmniMath" ], "models_studied": [], "production_evidence": false, "summary": "Formulates routing intermediate CoT states to models of different sizes as constrained decision making, solved with a small RL-trained control policy plus threshold calibration, avoiding a large PRM. It improves the accuracy-cost trade-off over handcrafted routers on open and closed models.", "key_idea": "Small RL control policy for stepwise model routing.", "relevance_score": 4, "relevance_rationale": "Learned per-step router without a big PRM.", "found_via": "snowball: cited-by lee-2025-steer / si-2025-ccpo", "cites_or_cited_by": [ "lee-2025-steer", "si-2025-ccpo" ] }, { "id": "singh-2026-nl2code-semantic-cache", "title": "Semantic Caching and Intent-Driven Context Optimization for Multi-Agent Natural Language to Code Systems", "authors": [ "Harmohit Singh" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.11687", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "LLM-based equivalence detection", "dual thresholds", "intent-driven context filtering" ], "cost_reduction_reported": "67% cache hit rate on production queries; 40-60% token reduction from table-aware context filtering; >10,000 production queries at 8.2 s average latency and 94.3% semantic accuracy", "benchmarks": [ "enterprise inventory-management NL-to-code queries" ], "models_studied": [], "production_evidence": true, "summary": "Production multi-agent natural-language-to-code analytics system that combines semantic caching (LLM-judged equivalence with adaptation hints), a dual threshold separating exact reuse from reference-guided generation, and intent-driven prompt assembly that filters table context.", "key_idea": "Pair semantic caching with intent-driven context pruning in a deployed multi-agent system.", "relevance_score": 4, "relevance_rationale": "Production evidence for caching and context pruning in an enterprise multi-agent system.", "found_via": "MeanCache user-centric semantic cache LLM", "cites_or_cited_by": [] }, { "id": "srivastava-2026-effgen", "title": "EffGen: Enabling Small Language Models as Capable Autonomous Agents", "authors": [ "Gaurav Srivastava", "Aafiya Hussain", "Chi Wang", "Yingyan Celine Lin", "Xuan Wang" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.00887", "code_url": "https://github.com/ctrl-gaurav/effGen", "topic_clusters": [ "C3", "C8", "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "five-factor complexity score" ], "cost_reduction_reported": "Prompt compression of 70-80% (57% average); outperforms LangChain, AutoGen and Smolagents on 13 benchmarks", "benchmarks": [ "13 benchmarks" ], "models_studied": [ "1.5B-32B SLMs" ], "production_evidence": false, "summary": "EffGen is an open-source agent framework tuned for small models, combining prompt compression for tool calling, dependency-aware task decomposition, complexity-based routing and a unified memory system. Prompt optimization helps small models more while routing helps larger ones more.", "key_idea": "Agent framework engineered for SLMs with complexity routing.", "relevance_score": 4, "relevance_rationale": "Practical SLM-first agent stack.", "found_via": "arXiv title search: small language models agentic", "cites_or_cited_by": [] }, { "id": "sun-2026-collaboration-tax", "title": "The Collaboration Tax: How Much LLM Multi-Agent Systems Pay to Coordinate", "authors": [ "Weixiang Sun", "Zehong Wang", "Hong Huang", "Colby Nelson", "Yijun Ma", "Yanfang Ye" ], "year": 2026, "venue_or_source": "EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2608.22152", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "conversation features" ], "cost_reduction_reported": null, "benchmarks": [ "32 solo-tractable tasks" ], "models_studied": [ "11 models from 7 providers" ], "production_evidence": false, "summary": "Defines the collaboration tax as the loss when two LLMs must coordinate under private information instead of acting alone, and measures it across 11 models. The tax falls with capability and stems from a four-stage conversational failure cascade; heterogeneous pairs drift toward the stronger partner.", "key_idea": "Coordination itself has a measurable, predictable performance cost.", "relevance_score": 4, "relevance_rationale": "Quantifies MAS coordination overhead.", "found_via": "arXiv search: heterogeneous LLM multi-agent cost", "cites_or_cited_by": [] }, { "id": "taheri-2026-iffc-function-calling", "title": "Small Reasoning Models are Instruction Followers in Function Calling", "authors": [ "Yalda Taheri", "Mohammad Hassan Heydari", "Erfan Naaman", "Afsaneh Fatemi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.22472", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "IFFC moves function-calling logic out of the main LLM into a dedicated smaller model that works in a plain instruction-following format. It beats native and prompt-based function calling, especially for reasoning models, and stays robust under aggressive quantization.", "key_idea": "Delegate tool-call formatting to a small instruction-following model.", "relevance_score": 4, "relevance_rationale": "A function-calling small model inside a larger agent (sub-theme d).", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "tang-2026-sakana-fugu", "title": "Sakana Fugu Technical Report", "authors": [ "Yu-Jin Tang", "Edoardo Cetin", "Jinglue Xu", "Qi Sun", "Stefan Nielsen", "Vincent Richard", "Haruto Goda", "I. Tymchenko", "N. Nguyen", "Hyunin Lee", "Mari Ashiga", "Shashank Kotyan", "So Kuroki", "Tarin Clanuwat" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.21228", "code_url": null, "topic_clusters": [ "C3", "C14" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "SWE-Bench Pro", "Terminal Bench", "LiveCodeBench", "GPQA-Diamond", "Humanity's Last Exam", "CharXiv Reasoning" ], "models_studied": [], "production_evidence": false, "summary": "Sakana Fugu is a family of orchestrator LMs trained to read a query and devise an agentic scaffold over a team of LLM agents. Training combines fine-tuning, evolutionary search and RL, and the report describes the infrastructure for running it as a production service, with a latency-balanced and a quality-maximizing variant.", "key_idea": "Productized orchestrator model that builds per-query multi-model scaffolds.", "relevance_score": 4, "relevance_rationale": "An industry technical report on learned multi-model orchestration; a cost-quality variant split but no cost numbers in the abstract.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "together-2026-deepswe-cascade-deepseek-gpt", "title": "DeepSeek V4 Pro 0813 vs GPT-5.6 Sol on DeepSWE: Cost, Coding, and Routing", "authors": [ "Zain Hasan", "Shobhit Dixit" ], "year": 2026, "venue_or_source": "Together AI blog", "source_type": "blog", "url": "https://www.together.ai/blog/deepseek-v4-pro-0813-vs-gpt-5-6-sol-on-deepswe-cost-coding-and-routing", "code_url": null, "topic_clusters": [ "C2", "C3", "C7", "C13" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "unit-test pass/fail" ], "cost_reduction_reported": "Cascade (DeepSeek V4 Pro first, escalate to GPT-5.6 Sol on test failure): 83.0% of DeepSWE tasks at $3.35/task vs Sol alone 72.7% at $8.37 (10 pts better, 60% cheaper); 904 rollouts (vendor-reported)", "benchmarks": [ "DeepSWE" ], "models_studied": [ "DeepSeek V4 Pro 0813", "GPT-5.6 Sol" ], "production_evidence": false, "summary": "Compares a cheap open model and a frontier closed model over 904 DeepSWE agent rollouts, reporting pass@k, cost, steps and regressions. The cheap model's pass@4 exceeds the frontier model's, and a test-gated cascade that escalates only on failing tests beats both in accuracy and cost; the frontier model's failures more often break already-passing tests.", "key_idea": "For agentic coding, a verifier-gated cascade (cheap model, escalate on failing tests) dominates either model alone.", "relevance_score": 4, "relevance_rationale": "Quantified cascade on a long-horizon agent benchmark using an execution signal for deferral.", "found_via": "direct: together.ai/blog index", "cites_or_cited_by": [ "fireworks-2026-frontier-is-a-router", "together-2026-deepswe-cascade-glm-flash" ] }, { "id": "together-2026-deepswe-cascade-glm-flash", "title": "GLM-5.3 vs. GLM-5.3 Flash on DeepSWE: Cost, Coding, and Routing", "authors": [ "Zain Hasan", "Shobhit Dixit" ], "year": 2026, "venue_or_source": "Together AI blog", "source_type": "blog", "url": "https://www.together.ai/blog/glm-5-3-vs-glm-5-3-flash-on-deepswe-cost-coding-and-routing", "code_url": null, "topic_clusters": [ "C2", "C8", "C13" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "unit-test pass/fail" ], "cost_reduction_reported": "Cascade (GLM-5.3 Flash first, escalate to GLM-5.3 when tests reject): 80.9% of DeepSWE tasks at $1.70 vs GLM-5.3 alone 69.0% at $3.99 (57% lower cost); Flash 17x cheaper per rollout (vendor-reported)", "benchmarks": [ "DeepSWE" ], "models_studied": [ "GLM-5.3", "GLM-5.3 Flash" ], "production_evidence": false, "summary": "Evaluates a distilled Flash model against its parent on DeepSWE: the gap is mainly first-try reliability (5.6 pts at pass@1, 2.6 pts at pass@4), the Flash solves 93 of the parent's 99 solved tasks, but breaks passing tests slightly more often. A test-gated cascade from Flash to the parent beats both.", "key_idea": "Distillation mostly costs consistency, which retries and verifier-gated escalation can buy back cheaply.", "relevance_score": 4, "relevance_rationale": "Cascade plus distillation evidence on an agentic benchmark.", "found_via": "direct: together.ai/blog index", "cites_or_cited_by": [ "together-2026-deepswe-cascade-deepseek-gpt" ] }, { "id": "tsai-2026-neuralucb-routing", "title": "Reward-Based Online LLM Routing via NeuralUCB", "authors": [ "Ming-Hua Tsai", "Phat Cao Tran" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.30035", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query embedding", "utility reward (quality minus cost)" ], "cost_reduction_reported": "substantially lower inference cost than max-quality reference with competitive reward (RouterBench, simulated online)", "benchmarks": [ "RouterBench" ], "models_studied": [ "RouterBench model pool" ], "production_evidence": false, "summary": "Implements NeuralUCB for cost-aware routing and evaluates it in a simulated online setting on RouterBench, beating random and min-cost baselines on utility.", "key_idea": "NeuralUCB applied to cost-aware LLM routing.", "relevance_score": 4, "relevance_rationale": "Simple empirical baseline for neural bandit routing.", "found_via": "OpenAlex title: LLM routing online", "cites_or_cited_by": [ "zhou-2019-neuralucb" ] }, { "id": "tsang-2026-orchrm", "title": "Reward Modeling for Multi-Agent Orchestration", "authors": [ "King Yeung Tsang", "Z. Zhao", "Vishal Venkataramani", "Haizhou Shi", "Zixuan Ke", "Semih Yavuz", "Shafiq Joty", "Hao Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.13598", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "orchestration reward model" ], "cost_reduction_reported": "Up to 10x training efficiency in token usage and up to 8% higher MAS test-time scaling accuracy", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "OrchRM is a self-supervised reward model that scores orchestration decisions in multi-agent systems directly, avoiding costly sub-agent rollouts for orchestrator training and test-time scaling.", "key_idea": "Reward model at the orchestrator level.", "relevance_score": 4, "relevance_rationale": "Cheap signal for learning MAS orchestration/configuration.", "found_via": "arXiv API title search: agentic reward modeling", "cites_or_cited_by": [] }, { "id": "tsymboi-2026-production-traffic-post-training", "title": "From Production Traffic to Post-Training: Building a Self-Hosted LLM That Covers the Corporate Request Mix", "authors": [ "Olga Tsymboi", "Dmitrii Stoianov", "Ramil Latypov", "Danil Taranets", "Daniil Dryabin", "Mikhail Gashkov", "Viktor Zelenkovskiy", "Aleksandr Fida", "Gleb Alektorov", "Nikita Gulyakov", "Arthur Babkin", "Aleksandr Medvedev", "Pavel Gein", "Anatolii Potapov" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.01572", "code_url": null, "topic_clusters": [ "C6", "C8", "C14" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "production error analysis", "verifiers", "LLM judges" ], "cost_reduction_reported": "model absorbs 50% of platform traffic (116M requests/month) at a fraction of the serving cost", "benchmarks": [ "in-house Arena", "instruction following", "function-calling" ], "models_studied": [ "self-hosted LLM vs ~7x larger baseline" ], "production_evidence": true, "summary": "Consolidates traffic from over 200 internal applications onto one self-hosted model by analyzing production errors, training per-axis GRPO experts and merging them; benchmarks are stratified to production traffic.", "key_idea": "Production-traffic-driven post-training to replace a larger model fleet.", "relevance_score": 4, "relevance_rationale": "Rare production evidence of using logs to cut serving cost.", "found_via": "OpenAlex title: production traffic LLM", "cites_or_cited_by": [] }, { "id": "unell-2026-deployment-centered-rejection-risk", "title": "Deployment-Centered Evaluation: Predicting Query-Level Rejection Risk in a Clinical LLM System", "authors": [ "Alyssa Unell", "Miguel Fuentes", "Brenna Li", "Bridget Lin", "Meena Jagadeesan", "Sanmi Koyejo", "Nigam Shah" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.12702", "code_url": null, "topic_clusters": [ "C6", "C7", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query content", "deployment context (department, provider type, model used)", "user rejection feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Trains a pre-response classifier, inside an EHR-embedded clinical LLM system, that predicts before generation whether the user will reject the response, using query content and deployment context. A prospective 4.5-month evaluation gives AUROC 0.719, and the authors estimate value for guardrail triggering and abstention.", "key_idea": "Pre-generation predictor of user rejection trained on sparse real deployment feedback.", "relevance_score": 4, "relevance_rationale": "Real production quality predictor that runs before generation; the same design can drive routing or escalation.", "found_via": "query: \"user feedback\" \"rejection\" LLM deployment prediction", "cites_or_cited_by": [] }, { "id": "varshney-2026-prefill-activation-router", "title": "LLM Router: Rethinking Routing with Prefill Activations", "authors": [ "Tanay Varshney", "Annie Surla", "Michelle Xu", "Gomathy Venkata Krishnan", "Maximilian Jeblick", "David Austin", "Neal Vaidya", "Davide Onofrio" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.20895", "code_url": null, "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "prefill residual-stream activations of an open encoder model", "prefill activations", "residual stream" ], "cost_reduction_reported": "Closes 45.58% of the gap between best single model and oracle while saving 74.31% cost relative to the most expensive model", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Routes using internal activations of an open-weight encoder model to predict correctness of target models including closed ones, with a shared multi-output MLP (NVIDIA authors).", "key_idea": "Prefill activations as routing features.", "relevance_score": 4, "relevance_rationale": "New signal type.", "found_via": "search: query difficulty LLM routing | WebSearch: probes hidden states LLM correctness prediction routing escalation", "cites_or_cited_by": [] }, { "id": "wang-2026-agentconductor", "title": "AgentConductor: Topology Evolution for Multi-Agent Competition-Level Code Generation", "authors": [ "Siyu Wang", "Ruotian Lu", "Zhihao Yang", "Yuchao Wang", "Yanzhou Zhang", "Lei Xu", "Qimin Xu", "Guojun Yin", "Cailian Chen", "Xinping Guan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.17100", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "task difficulty", "execution feedback" ], "cost_reduction_reported": "Up to 14.6% higher pass@1, 13% density reduction and 68% token-cost reduction vs strongest baseline", "benchmarks": [ "three competition-level and two foundational code datasets" ], "models_studied": [], "production_evidence": false, "summary": "AgentConductor trains an LLM orchestrator with RL to infer roles and difficulty per query and build a density-controlled layered DAG topology, refining it with execution feedback. It improves competition-level code generation while cutting token cost.", "key_idea": "Difficulty-aware topology density control by an RL orchestrator.", "relevance_score": 4, "relevance_rationale": "Adaptive MAS structure for cost.", "found_via": "web search: G-Designer", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "zhang-2024-agentprune", "zhang-2025-maas-agentic-supernet" ] }, { "id": "wang-2026-beyond-tier-labels", "title": "Beyond Tier Labels: Role- and Deployment-Dependent Model Substitution in Multi-Call LLM Workflows", "authors": [ "Renxiang Wang", "Jiaming Cui" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.09155", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "role", "deployment context" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Separates where to use a stronger model from the value of the substitution in multi-call workflows, showing the benefit of upgrading a call depends on its role and surrounding deployment and can reverse across tasks.", "key_idea": "Model substitution value is role- and deployment-dependent.", "relevance_score": 4, "relevance_rationale": "Informs per-role model assignment decisions.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "wang-2026-conformal-thinking", "title": "Conformal Thinking: Risk Control for Reasoning on a Compute Budget", "authors": [ "Xi Wang", "Anushri Suresh", "Alvin Zhang", "Rishi More", "William Jurayj", "Benjamin Van Durme", "Mehrdad Farajtabar", "Daniel Khashabi", "Eric Nalisnick" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.03814", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "confidence upper threshold", "parametric lower threshold for unsolvable instances" ], "cost_reduction_reported": null, "benchmarks": [ "diverse reasoning tasks" ], "models_studied": [], "production_evidence": false, "summary": "Casts reasoning budget setting as risk control: stop when confident, and preemptively stop likely unsolvable instances, with thresholds set by distribution-free risk control on a validation set.", "key_idea": "Risk-controlled early stopping in both directions.", "relevance_score": 4, "relevance_rationale": "Guaranteed reasoning budget policies.", "found_via": "arXiv title search: thinking budget", "cites_or_cited_by": [] }, { "id": "wang-2026-mcpp-budget-deadline-allocation", "title": "On Time, Within Budget: Constraint-Driven Online Resource Allocation for Agentic Workflows", "authors": [ "Xinglin Wang", "Zishen Liu", "Shaoxiong Feng", "Peiwen Yuan", "Yiwei Li", "Jiayi Shi", "Yueqi Zhang", "Chuyi Tan", "Ji Zhang", "Boyuan Pan", "Yao Hu", "Kan Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06110", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "subtask-model success-rate and length estimates", "remaining budget/time" ], "cost_reduction_reported": "Consistently improves constrained completion probability over strong baselines", "benchmarks": [ "CodeFlow", "ProofFlow" ], "models_studied": [], "production_evidence": false, "summary": "Studies allocating models and parallel samples to subtasks of a dependency-structured agent workflow so it finishes within a budget and deadline, via Monte Carlo Portfolio Planning that simulates executions and replans after outcomes.", "key_idea": "Online subtask-to-model allocation maximizing on-time, in-budget completion.", "relevance_score": 4, "relevance_rationale": "Per-subtask model assignment under hard budgets.", "found_via": "snowball: cited-by zhang-2025-maas-agentic-supernet", "cites_or_cited_by": [ "zhang-2025-maas-agentic-supernet", "zhang-2026-budget-aware-agentic-routing", "zhang-2026-evoroute" ] }, { "id": "wang-2026-rethinking-scale-slm-agent-paradigms", "title": "Rethinking Scale: Deployment Trade-offs of Small Language Models under Agent Paradigms", "authors": [ "Xinlin Wang", "Mats Brorsson" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.19299", "code_url": null, "topic_clusters": [ "C8", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "open-source models under 10B", "<10B open models" ], "production_evidence": false, "summary": "A large study of sub-10B open models as base models, single tool-using agents and multi-agent systems. Single agents give the best performance-cost balance, while multi-agent setups add overhead with limited gains.", "key_idea": "For small models, single tool agents beat multi-agent setups on cost-effectiveness.", "relevance_score": 4, "relevance_rationale": "Evidence on when MAS is worth it at small scale.", "found_via": "arXiv title search: small language models agentic | hf-papers search: enterprise agent fine-tuning small model", "cites_or_cited_by": [] }, { "id": "wang-2026-routemoa", "title": "RouteMoA: Dynamic Routing without Pre-Inference Boosts Efficient Mixture-of-Agents", "authors": [ "Jize Wang", "Han Wu", "Zhiyuan You", "Yiming Song", "Yijun Wang", "Zifei Shan", "Yining Li", "Songyang Zhang", "Xinyi Le", "Cailian Chen", "Xinping Guan", "Dacheng Tao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.18130", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "query-based performance prediction", "judge self/cross assessment" ], "cost_reduction_reported": "89.8% cost reduction and 63.6% latency reduction in large-scale model pool vs MoA", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RouteMoA screens candidate models with a lightweight scorer before any inference, refines scores with judges using existing outputs, and ranks models by performance, cost and latency. It outperforms MoA while cutting cost by about 90% in large pools.", "key_idea": "Pre-inference routing to pick which models join each MoA layer.", "relevance_score": 4, "relevance_rationale": "Router-driven sparsification of multi-model ensembles.", "found_via": "web search: MoA cost", "cites_or_cited_by": [] }, { "id": "wei-2026-llm-experts-limited-feedback", "title": "Online Learning with LLM Experts from Limited Feedback", "authors": [ "Wang Wei", "Soumyabrata Pal", "Koyel Mukherjee", "Franck Dernoncourt", "Ryan A. Rossi", "Branislav Kveton", "Hoda Eldardiry" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.05820", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "prompt features", "feedback budget" ], "cost_reduction_reported": null, "benchmarks": [ "diverse LLMs" ], "models_studied": [], "production_evidence": false, "summary": "Online routing of prompts to LLM experts when only m << T feedback observations are allowed; gives regret bounds for full-information and bandit settings and learns strong policies from limited feedback.", "key_idea": "Bandit routing with a feedback (labeling) budget.", "relevance_score": 4, "relevance_rationale": "Models the cost of acquiring feedback in production.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "wei-2025-barp" ] }, { "id": "wei-2026-routing-least-learnable", "title": "Routing Is Least Learnable Where It Is Most Valuable: Bounds on Representation Routing for Web Agents", "authors": [ "Jiaming Wei", "Zekun Wu", "Adriano Koshiyama", "Maria Perez-Ortiz" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.06171", "code_url": null, "topic_clusters": [ "C3", "C6", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "observation mode" ], "cost_reduction_reported": "Sending only tasks no mode solves to the cheapest mode cuts cost 9.5-30.6% at unchanged success; no tested routing policy robustly beats one well-chosen fixed mode", "benchmarks": [ "VisualWebArena", "WebArena" ], "models_studied": [], "production_evidence": false, "summary": "Measures per-task routing among six observation modes for web agents and finds oracle gains are inflated by 12-14% run-to-run noise; five routing policies fail to beat a fixed good mode. Routing supervision is scarcest exactly where routing would help most.", "key_idea": "Router labels are scarce where routing matters; noise inflates oracle gains.", "relevance_score": 4, "relevance_rationale": "Important negative result for learning agent routers.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "liu-2026-adaptive-vlm-routing-cua" ] }, { "id": "weinberger-2026-prompt-induced-waste-coding-agents", "title": "Prompt-Induced Waste in Coding Agents: Reasoning, Effort, Harness Design, and End-to-End Cost", "authors": [ "Sarel Weinberger", "Amir Hozez" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.01347", "code_url": null, "topic_clusters": [ "C9", "C12", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "prompt wording", "inference effort", "harness policy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Controlled experiments showing prompt wording, reasoning effort and harness design interact to determine coding-agent cost and success; argues efficiency should be measured as cost per successful task, not tokens.", "key_idea": "Measure cost per successful task; prompt, effort and harness interact.", "relevance_score": 4, "relevance_rationale": "Methodology for evaluating agent cost levers.", "found_via": "arXiv title search: reasoning effort", "cites_or_cited_by": [] }, { "id": "weinberger-2026-token-reduction-not-cost", "title": "Token Reduction Is Not Cost Reduction", "authors": [ "Sarel Weinberger", "Amir Hozez" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.12161", "code_url": null, "topic_clusters": [ "C10", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "billed cost", "cache traffic" ], "cost_reduction_reported": "Largest compression cut tool-output tokens by 38.4% but increased billed cost by 6.8%; token reduction correlated weakly with cost reduction (r=0.15)", "benchmarks": [ "SWE-bench Go subset", "controlled coding tasks" ], "models_studied": [ "Claude Code" ], "production_evidence": false, "summary": "Shows that compressing context in tool-heavy coding agents can trigger extra retrieval, diagnosis, testing and turns that offset token savings, and aggressive compression reduced successful patches. Argues for evaluating cost per successful task including cache behaviour.", "key_idea": "Measure cost per solved task, not tokens removed.", "relevance_score": 4, "relevance_rationale": "Important caveat for any context-compression cost claim.", "found_via": "SWE-agent context management trajectory reduction cost coding agents arXiv | web search: SWE-bench cost per task", "cites_or_cited_by": [ "satish-2026-beyond-token-savings", "xiao-2025-agentdiet-trajectory-reduction" ] }, { "id": "xiong-2026-clawarena-team", "title": "ClawArena-Team: Benchmarking Subagent Orchestration and Dynamic Workflows in Language-Model Agents", "authors": [ "Kaiwen Xiong", "Haonian Ji", "Shi Qiu", "Zeyu Zheng", "Cihang Xie", "Xinyu Ye", "Huaxiu Yao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.31174", "code_url": null, "topic_clusters": [ "C3", "C12", "C13" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "subagent management score" ], "cost_reduction_reported": "API cost spans over 100x across leader models while overall score spans under 4x", "benchmarks": [ "ClawArena-Team (41 scenarios, 258 rounds)" ], "models_studied": [ "twelve proprietary, community-hosted and self-hosted models" ], "production_evidence": false, "summary": "Benchmarks a single LLM's ability to act as a manager that spawns and orchestrates a fixed pool of local subagents, scored by execution with no LLM judge. The bottleneck is privilege granting, and management quality is largely decoupled from cost, with cheap open models competitive.", "key_idea": "Benchmark for the leader role in subagent orchestration.", "relevance_score": 4, "relevance_rationale": "Evidence on which role needs the expensive model.", "found_via": "arXiv search: orchestrator subagents cost", "cites_or_cited_by": [] }, { "id": "xu-2026-hyevo", "title": "HyEvo: Self-Evolving Hybrid Agentic Workflows for Efficient Reasoning", "authors": [ "Beibei Xu", "Yutong Ye", "Chuyun Shen", "Yingbo Zhou", "Cheng Chen", "Mingsong Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.19639", "code_url": null, "topic_clusters": [ "C4", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "execution feedback" ], "cost_reduction_reported": "Up to 19x lower inference cost and 16x lower latency vs state-of-the-art open-source baseline", "benchmarks": [ "reasoning and coding benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "HyEvo evolves agentic workflows that mix LLM nodes with deterministic code nodes, offloading predictable operations from LLM inference, using a multi-island evolutionary search with reflection. It outperforms prior workflow generators while cutting cost and latency sharply.", "key_idea": "Replace predictable LLM steps with code nodes during workflow search.", "relevance_score": 4, "relevance_rationale": "Architecture-level cost lever: fewer LLM calls.", "found_via": "OpenAlex search: heterogeneous LLMs multi-agent", "cites_or_cited_by": [ "zhang-2025-maas-agentic-supernet" ] }, { "id": "xu-2026-learning-what-to-skip", "title": "Learning What to Skip: Counterfactual Credit Assignment for Efficient Multi-Agent LLM Workflows", "authors": [ "Jinfeng Xu", "Zheyu Chen", "Ziyue Peng", "Zheng Lin", "Shuo Yang", "Jinze Li", "Zheng Xing", "Mengran Li", "Victor C. M. Leung" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.30734", "code_url": null, "topic_clusters": [ "C3", "C6", "C12" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "counterfactual skip interventions", "calibrated safety models" ], "cost_reduction_reported": "Reduces recorded token cost while matching or improving full-workflow accuracy (no single number in abstract)", "benchmarks": [ "math reasoning", "multiple-choice QA", "code generation" ], "models_studied": [ "two instruction-model families" ], "production_evidence": false, "summary": "LW2S learns from controlled skip interventions on full-workflow logs which planning, verification or summarization components can be safely omitted given the current state, with held-out calibration and guards. It lowers token cost at equal accuracy.", "key_idea": "Counterfactual credit assignment to skip unneeded workflow components.", "relevance_score": 4, "relevance_rationale": "Learning component-level omission from logs.", "found_via": "arXiv search: cost-efficient LLM agent model selection", "cites_or_cited_by": [ "feng-2026-graphplanner", "yue-2025-masrouter" ] }, { "id": "xu-2026-low-effort-office-agents", "title": "Low Reasoning Effort Is Enough for Routine Office Work by Language-Model Agents", "authors": [ "Xiaonan Xu", "Wenjing Wu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.03169", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "reasoning level setting" ], "cost_reduction_reported": "Low reasoning level used about 43% fewer output tokens and 20% less time than max level, with no rule violations across 840 runs of 14 office tasks", "benchmarks": [ "14 routine office tasks (840 runs)" ], "models_studied": [ "two tiers of GPT-5.6" ], "production_evidence": false, "summary": "Controlled test of low vs max reasoning effort for agents doing routine office tasks with a forbidden-tool temptation; low effort was sufficient for compliance and completion.", "key_idea": "Routine agent work does not need high reasoning effort.", "relevance_score": 4, "relevance_rationale": "Practical evidence for downscaling effort in enterprise agents.", "found_via": "arXiv title search: reasoning effort", "cites_or_cited_by": [] }, { "id": "xu-2026-oneflow-single-agent-baseline", "title": "Rethinking the Value of Multi-Agent Workflow: A Strong Single Agent Baseline", "authors": [ "Jiawei Xu", "Arief Koesdwiady", "Sisong Bei", "Yan Han", "Baixiang Huang", "Dakuo Wang", "Yutong Chen", "Zheshen Wang", "Peihao Wang", "Pan Li", "Ying Ding" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.12307", "code_url": null, "topic_clusters": [ "C3", "C10", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "Single-agent execution matches homogeneous workflows with KV-cache reuse efficiency; OneFlow reduces inference costs vs automatic multi-agent design frameworks", "benchmarks": [ "seven benchmarks (coding, math, QA, domain reasoning, planning/tool use)" ], "models_studied": [], "production_evidence": false, "summary": "Shows that homogeneous multi-agent workflows, where all agents share one base LLM, can be simulated by a single agent in multi-turn conversation at equal accuracy while reusing the KV cache. The proposed OneFlow tailors workflows for single-agent execution; the authors note that truly heterogeneous workflows are where multi-agent value may remain.", "key_idea": "Homogeneous MAS is largely reducible to one agent; heterogeneity is what matters.", "relevance_score": 4, "relevance_rationale": "Key baseline for deciding when multi-agent or multi-model splits are worth it.", "found_via": "snowball: cited-by ye-2025-x-mas / yue-2025-masrouter / belcak-2025", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai", "ye-2025-x-mas", "yue-2025-masrouter" ] }, { "id": "xu-2026-patching-architectures-study", "title": "A Systematic Study of LLM-Based Architectures for Automated Patching", "authors": [ "Qingxiao Xu", "Ze Sheng", "Zhicheng Chen", "Jeff Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.01257", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "token usage", "execution time" ], "cost_reduction_reported": null, "benchmarks": [ "real-world vulnerability tasks" ], "models_studied": [], "production_evidence": false, "summary": "A controlled comparison of fixed workflows, single agents, multi-agent systems and general-purpose code agents for automated vulnerability patching. Workflows are efficient but brittle, single agents balance flexibility and cost, multi-agent designs add heavy overhead and drift, and general code agents perform best.", "key_idea": "Architecture and iteration depth, not just model, drive patching cost and reliability.", "relevance_score": 4, "relevance_rationale": "Controlled architecture cost comparison.", "found_via": "arXiv API search", "cites_or_cited_by": [] }, { "id": "xu-2026-tokenpilot", "title": "TokenPilot: Cache-Efficient Context Management for LLM Agents", "authors": [ "Buqiang Xu", "Zirui Xue", "Dianmou Chen", "Chenyang Fu", "Chiyu Wu", "Caiying Huang", "Chen Jiang", "Jizhan Fang", "Xinle Deng", "Yijun Chen", "Yunzhi Yao", "Xuehai Wang", "Jin Shang", "Gong Yu", "Ningyu Zhang" ], "year": 2026, "venue_or_source": "Findings of EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2606.17016", "code_url": "https://github.com/zjunlp/RSI", "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "ingestion-aware compaction", "residual-utility eviction" ], "cost_reduction_reported": "costs reduced 61% and 56% in isolated mode and 61% and 87% in continuous mode on PinchBench and Claw-Eval with competitive performance", "benchmarks": [ "PinchBench", "Claw-Eval" ], "models_studied": [], "production_evidence": false, "summary": "Context management for agents designed to keep prompt caches valid: volatile runtime values in tool responses are replaced by stable placeholders before ingestion, and segments are evicted only when their relevance expires, in batched turns to limit cache invalidation.", "key_idea": "Compact context in a cache-friendly way so savings compound with prompt caching.", "relevance_score": 4, "relevance_rationale": "Explicitly co-designs compaction with prompt caching for cost.", "found_via": "tool output observation compression LLM agent tokens arXiv", "cites_or_cited_by": [ "lumer-2026-dont-break-the-cache" ] }, { "id": "xue-2026-r2-router", "title": "R2-Router: A New Paradigm for LLM Routing with Reasoning", "authors": [ "Jiaqi Xue", "Qian Lou", "Jiarong Xing", "Heng Huang" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.02823", "code_url": "https://github.com/UCF-ML-Research/R2-Router", "topic_clusters": [ "C1", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted quality per (model, length budget)" ], "cost_reduction_reported": "State-of-the-art performance at 4-5x lower cost than existing routers", "benchmarks": [ "R2-Bench" ], "models_studied": [], "production_evidence": false, "summary": "Routes jointly over models and output-length budgets enforced by instructions, finding that a strong model with a constrained length can beat a weaker model at similar cost. Introduces R2-Bench.", "key_idea": "Route over (model, output length budget).", "relevance_score": 4, "relevance_rationale": "Joins routing and token budget control.", "found_via": "arXiv title search: reasoning router", "cites_or_cited_by": [] }, { "id": "yang-2026-cogrouter", "title": "Think Fast and Slow: Step-Level Cognitive Depth Adaptation for LLM Agents", "authors": [ "Ruihan Yang", "Fanghua Ye", "Xiang We", "Ruoqing Zhao", "Kang Luo", "Xinbo Xu", "Bo Zhao", "Ruotian Ma", "Shanyi Wang", "Zhaopeng Tu", "Xiaolong Li", "Deqing Yang", " Linus" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.12662", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "action confidence", "cognitive level" ], "cost_reduction_reported": "With Qwen2.5-7B, 82.3% success on ALFWorld/ScienceWorld, beating GPT-4o (+40.3%) and o3 (+18.3%) while using 62% fewer tokens", "benchmarks": [ "ALFWorld", "ScienceWorld" ], "models_studied": [ "Qwen2.5-7B", "GPT-4o", "OpenAI-o3" ], "production_evidence": false, "summary": "CogRouter trains an agent to choose among four levels of cognitive depth at each step, from instinctive responses to strategic planning. Training combines cognition-aware SFT with an RL stage that does step-level credit assignment via confidence-weighted advantages.", "key_idea": "Per-step choice of reasoning depth learned with step-level credit assignment.", "relevance_score": 4, "relevance_rationale": "Step-level compute allocation inside agents, which is analogous to step-level model routing; covers sub-themes (a) and (f).", "found_via": "arXiv query ti:\"fast and slow\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "yang-2026-protocol-routing-cost-aware", "title": "LLMs Can Predict Failure Risk, But Struggle to Predict Which Collaboration Protocol Pays Off: Cost-Aware Protocol Routing Across Reasoning Tasks", "authors": [ "Chih-Hsuan Yang", "Jingyan Jiang", "Cheng-Hau Yang", "Vikram Vasudevan", "Huihuo Zheng", "Venkatram Vishwanath", "Rajeev Thakur" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.14927", "code_url": null, "topic_clusters": [ "C2", "C7", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "post-answer failure probe", "self-confidence gate", "failure-risk probe" ], "cost_reduction_reported": "Self-confidence gate reaches 78.0% solve at 45K tokens vs 73.8% at 71.3K for a frozen gpt-oss-120b router", "benchmarks": [ "4,181 competition math problems", "biology and science benchmarks" ], "models_studied": [ "gpt-oss-120b", "two solver families" ], "production_evidence": false, "summary": "Runs every problem under four protocols (direct, self-correction, planner-executor-reviewer, multi-agent deliberation) to study when to pay for collaboration. LLM probes predict baseline failure well (0.88 AUROC) but struggle to predict which collaboration protocol helps; routers tend to under- or over-escalate.", "key_idea": "Predicting failure is easy; predicting which collaboration pays off is hard.", "relevance_score": 4, "relevance_rationale": "Escalation to multi-agent protocols as a routing decision.", "found_via": "arXiv search: planner executor cost | hf-papers search: confidence estimation routing escalation", "cites_or_cited_by": [] }, { "id": "yao-2026-hieramas", "title": "HieraMAS: Optimizing Intra-Node LLM Mixtures and Inter-Node Topology for Multi-Agent Systems", "authors": [ "Tianjun Yao", "Zhaoyi Li", "Zhiqiang Shen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.20229", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "two-stage credit assignment" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "HieraMAS implements each MAS role as a supernode mixing several heterogeneous LLMs in a propose-synthesize structure and jointly optimizes intra-node mixtures and inter-node topology, addressing credit assignment when outcomes depend on backbone capability.", "key_idea": "Optimize LLM mixtures inside each role plus topology.", "relevance_score": 4, "relevance_rationale": "Role-level heterogeneous model assignment.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [ "zhang-2024-agentprune" ] }, { "id": "ye-2026-roro-process-reward-routing", "title": "Rubric-Guided Process Reward for Stepwise Model Routing", "authors": [ "Shenghao Ye", "Yu Guo", "Zhengheng Li", "Shuangwu Chen", "Jian Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.29310", "code_url": null, "topic_clusters": [ "C3", "C7", "C9" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "rubric-guided process reward", "cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RoRo supervises RL-trained stepwise routers with process rewards instead of only outcome rewards. A Rubricor writes a query-specific rubric and a Judge scores routing trajectories, both trained on preference pairs built from outcome, cost and process quality.", "key_idea": "Process-level rewards for training stepwise model routers.", "relevance_score": 4, "relevance_rationale": "Credit assignment for step routing decisions (sub-theme f).", "found_via": "snowball: cited-by lee-2025-steer / kapoor-2026-trim-stepwise-routing", "cites_or_cited_by": [ "kapoor-2026-trim-stepwise-routing", "lee-2025-steer" ] }, { "id": "yuan-2026-clawtrace", "title": "ClawTrace: Cost-Aware Tracing for LLM Agent Skill Distillation", "authors": [ "Boqin Yuan", "Renchu Song", "Yue Su", "Sen Yang", "Jing Qin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.23853", "code_url": null, "topic_clusters": [ "C6", "C8" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "per-step USD cost", "redundancy flags" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Tracing platform recording every LLM call, tool use and sub-agent spawn with per-step cost, compiling sessions into TraceCards that let skill-distillation pipelines prune expensive steps that did not matter.", "key_idea": "Cost-aware traces for skill distillation.", "relevance_score": 4, "relevance_rationale": "Uses production traces for cost-aware distillation.", "found_via": "hf-papers search: SFT agent traces cost", "cites_or_cited_by": [] }, { "id": "yuan-2026-paramanager-small-orchestrator", "title": "Small Model as Master Orchestrator: Learning Unified Agent-Tool Orchestration with Parallel Subtask Decomposition", "authors": [ "Wenzhen Yuan", "Wutao Xiong", "Fanchen Yu", "Shengji Tang", "Ting Liu", "Tao Chen", "Peng Ye", "Yuzhuo Fu", "Wanli Ouyang", "Lei Bai" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.17009", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "state feedback", "SFT with recovery", "RL multi-objective" ], "cost_reduction_reported": null, "benchmarks": [ "multiple benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Agent-as-Tool unifies agents and tools into one learnable action space, and a lightweight orchestrator (ParaManager) trained with SFT and RL decomposes tasks into parallel subtasks and delegates them. It generalizes to unseen model pools.", "key_idea": "A small trained orchestrator delegates parallel subtasks to agents and tools.", "relevance_score": 4, "relevance_rationale": "Small-model orchestration of larger workers.", "found_via": "web search: Puppeteer | hf-papers search: small model orchestrator", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration" ] }, { "id": "zeng-2026-learning-to-seek-help", "title": "Learning to Seek Help: Dynamic Collaboration Between Small and Large Language Models", "authors": [ "Hang Zeng", "Xiangyu Liu", "Yong Hu", "Chaoyue Niu", "Jiarui Zhang", "Shaojie Tang", "Fan Wu", "Guihai Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.17827", "code_url": null, "topic_clusters": [ "C2", "C3" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "SLM learned help-request policy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "An SLM learns when and how to request help from an LLM during multi-step reasoning, and the LLM gives adaptive feedback. Stronger SLMs become more self-reliant and the learned strategies transfer to unseen LLMs.", "key_idea": "Learned mid-reasoning help requests from small to large model.", "relevance_score": 4, "relevance_rationale": "Step-level escalation in multi-step reasoning.", "found_via": "snowball: cited-by AutoMix (2310.12963)", "cites_or_cited_by": [ "aggarwal-2023-automix" ] }, { "id": "zeng-2026-metarouter", "title": "Learning to Route LLMs from Implicit Cost-Performance Preferences via Meta-Learning", "authors": [ "Jiahao Zeng", "Ming Tang", "Ningning Ding" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.06178", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "implicit user cost-performance preference" ], "cost_reduction_reported": null, "benchmarks": [ "ID and OOD tasks" ], "models_studied": [], "production_evidence": false, "summary": "MetaRouter learns each user's implicit cost-performance preference from a few interactions by treating preference profiles as tasks in a meta-learned contextual bandit.", "key_idea": "Meta-learned bandit router for per-user preferences.", "relevance_score": 4, "relevance_rationale": "Online learning of user-specific cost trade-offs.", "found_via": "arXiv: abs:\"LLM routing\" AND abs:bandit", "cites_or_cited_by": [ "li-2025-llm-bandit", "wei-2025-barp" ] }, { "id": "zeng-2026-mixture-of-roles-single-agent", "title": "One Model, Many Minds: Unlocking Multi-Agent Synergy in a Single Agent via Mixture of Roles", "authors": [ "Zhichen Zeng", "Huiyuan Chen", "Jingru Cheng", "Juan Zha", "Ming Liu", "Ying Chen", "Xiyuan Yang", "Chaosheng Dong", "Haiyang Zhang", "Hanghang Tong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.27338", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "query-aware router over steering-vector codebook" ], "cost_reduction_reported": "On par with MAS while reducing token cost by 20x; +2.2% over single-agent baselines", "benchmarks": [ "reasoning and personality benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "MoRe composes several learned role steering vectors into one per query so a single frozen LLM gets multi-perspective specialization in one turn, matching multi-agent systems at a twentieth of the tokens.", "key_idea": "Compress multi-agent role diversity into a single steered forward pass.", "relevance_score": 4, "relevance_rationale": "Architecture-level alternative to MAS with large token savings.", "found_via": "arXiv API search", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems", "tran-2026-single-vs-multi-agent-equal-budget" ] }, { "id": "zha-2026-cacherouter", "title": "CacheRouter: A Dual-Path Tool Routing Architecture with Cache-Preserving Main-Model Isolation for Long-Tail Tool Discovery", "authors": [ "Donghui Zha", "Lingwei Xu", "Linxiao Wu", "Yixue Dong", "Haochen Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.22708", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "tool search by router sub-model" ], "cost_reduction_reported": "Cache hit rates 90.99% and 95.2%, cutting input cost to ~12.0% and ~8.0% of a no-cache baseline under DeepSeek pricing (55 queries and a 30-turn dialogue)", "benchmarks": [], "models_studied": [ "DeepSeek" ], "production_evidence": false, "summary": "CacheRouter keeps the main model's tool list small and fixed so its prompt prefix stays cacheable, while a separate router sub-model searches the full tool catalog, runs the chosen tool and returns the result.", "key_idea": "Offload long-tail tool selection to a router sub-model to keep the main model's cache prefix stable.", "relevance_score": 4, "relevance_rationale": "Subagent offloading as a cost lever that interacts with prompt caching; prototype-scale evaluation.", "found_via": "arXiv query ti:\"routing\" AND ti:\"tool\"", "cites_or_cited_by": [] }, { "id": "zhang-2026-agentfactory", "title": "AgentFactory: Towards Automated Agentic System Design and Optimization", "authors": [ "Enci Zhang", "Haofeng Wang", "Yuesheng Zhu", "Xiaole Cui", "Guibo Luo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.01045", "code_url": null, "topic_clusters": [ "C4", "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "performance", "cost", "efficiency" ], "cost_reduction_reported": null, "benchmarks": [ "MedQA", "FinEval" ], "models_studied": [], "production_evidence": false, "summary": "AgentFactory jointly optimizes foundation models (including fine-tuned ones) and workflow structures with LLM optimizers over multiple objectives including cost, gaining 9.1% on average over eight benchmarks.", "key_idea": "Joint model and workflow optimization with multiple objectives.", "relevance_score": 4, "relevance_rationale": "Model choice as a variable with cost among objectives.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "zhang-2026-agentforesight", "title": "AgentForesight: Online Auditing for Early Failure Prediction in Multi-Agent Systems", "authors": [ "Boxuan Zhang", "Jianing Zhu", "Zeru Shi", "Dongfang Liu", "Ruixiang Tang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.08715", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "online auditor", "risk anticipation" ], "cost_reduction_reported": null, "benchmarks": [ "AFTraj-2K" ], "models_studied": [ "AgentForesight-7B" ], "production_evidence": false, "summary": "Reframes failure attribution as online early failure prediction and trains a compact 7B auditor with RL on the AFTraj-2K corpus to flag decisive errors during multi-agent execution.", "key_idea": "Small online auditor predicts failures before they cascade.", "relevance_score": 4, "relevance_rationale": "Enables in-flight escalation in MAS.", "found_via": "hf-papers search: agent failure detection", "cites_or_cited_by": [ "zhang-2025-which-agent-when" ] }, { "id": "zhang-2026-cachescout", "title": "Learning Agent Execution for KV-Cache Management in Agentic Serving", "authors": [ "Rui Zhang", "Chaeeun Kim", "Shaoting Feng", "Kuntai Du", "Yuhan Liu", "Yi Zhong", "Cheng-Wei Ching", "Junchen Jiang", "Liting Hu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.14624", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "subagent", "learning_regime": "bandit-online", "signals_used": [ "online-learned agent execution transitions" ], "cost_reduction_reported": "KV hit rate +10-18 points, mean TTFT -18-45%, per-turn latency -29-38%, peak throughput up to +57% (TTFT up to -54% on larger models)", "benchmarks": [ "real-world multi-agent workloads" ], "models_studied": [], "production_evidence": false, "summary": "CacheScout learns agent execution transitions online, without predefined workflow graphs, and uses them to guide KV eviction and prefetching in vLLM off the critical path.", "key_idea": "Learn which agent runs next to decide what KV to keep.", "relevance_score": 4, "relevance_rationale": "Online learning of agent execution semantics for cache management.", "found_via": "KV cache reuse agentic workloads prefix caching hit rate production trace", "cites_or_cited_by": [ "pan-2025-kvflow", "zheng-2026-pbkv" ] }, { "id": "zhang-2026-cauc", "title": "Calibration-Aware Uncertainty Cascades for Efficient Heterogeneous Model Collaboration", "authors": [ "Yilin Zhang", "Han Jiang", "Cai Xu", "Ying Liu", "Wei Zhao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.11446", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "per-model calibrated confidence" ], "cost_reduction_reported": "Average relative accuracy +1.9% over strong-model-only inference while avoiding ~47% of strong-model calls across six language benchmarks", "benchmarks": [ "six language benchmarks", "image classification benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Calibrates each model's confidence separately so that one threshold has a selective-risk meaning across heterogeneous models, then chooses accept, escalate or combine policies on validation data.", "key_idea": "Per-model calibration gives cascade thresholds a common reliability scale.", "relevance_score": 4, "relevance_rationale": "Portable thresholds across changing model pools.", "found_via": "arXiv title search: model cascading", "cites_or_cited_by": [] }, { "id": "zhang-2026-corea", "title": "Confidence-Calibrated Small-Large Language Model Collaboration for Cost-Efficient Reasoning", "authors": [ "Chuang Zhang", "Zizhen Zhu", "Yihao Wei", "Bing Tian", "Junyi Liu", "Henan Wang", "Xavier Wang", "Yaxiao Liu" ], "year": 2026, "venue_or_source": "EACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2603.03752", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "verbalized confidence (RL-calibrated)" ], "cost_reduction_reported": "Reduces cost by 21.5% and 16.8% on out-of-domain math and non-math datasets vs LLM alone, with absolute pass@1 drop within 2%", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Cascades a small model that outputs an answer plus verbalised confidence with a large model, deferring low-confidence questions. RL with a calibration reward improves both reasoning and confidence calibration of the small model.", "key_idea": "RL-calibrated verbalised confidence for SLM-to-LLM deferral.", "relevance_score": 4, "relevance_rationale": "Training the cheap model to know when to escalate.", "found_via": "arXiv title search: confidence small large language model collaboration", "cites_or_cited_by": [] }, { "id": "zhang-2026-dialrouter", "title": "From Myopic Selection to Long-Horizon Awareness: Sequential LLM Routing for Multi-Turn Dialogue", "authors": [ "Jiarui Zhang", "Xiangyu Liu", "Yong Hu", "Chaoyue Niu", "Hang Zeng", "Shaojie Tang", "Fan Wu", "Guihai Chen", "Jia-Rui Zhang", "Yong-Hao Hu", "Chao-Yue Niu", "H. Zeng", "Gui-Hai Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.12385", "code_url": null, "topic_clusters": [ "C1", "C3", "C6" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "MCTS-derived trajectory rewards", "retrieval-based future state approximation", "cost-aware reward", "dialogue state" ], "cost_reduction_reported": null, "benchmarks": [ "open-domain and domain-specific dialogue tasks" ], "models_studied": [ "open-source and closed-source LLMs" ], "production_evidence": false, "summary": "DialRouter treats model selection across a multi-turn dialogue as a sequential decision problem. It runs MCTS over dialogue branches induced by different model choices, then distills a lightweight routing policy from the high-reward search trajectories so that no online search is needed at inference.", "key_idea": "Long-horizon sequential routing for multi-turn interactions, distilled from MCTS search.", "relevance_score": 4, "relevance_rationale": "Per-turn routing with delayed reward; dialogue rather than tool-using agents, and no headline cost number in the abstract.", "found_via": "arXiv query ti:\"LLM routing\" AND abs:\"multi-turn\" | query: off-policy evaluation LLM routing", "cites_or_cited_by": [] }, { "id": "zhang-2026-m-cmab", "title": "Adapter-Augmented Bandits for Online Multi-Constrained Multi-Modal Inference Scheduling", "authors": [ "Xianzhi Zhang", "Yue Xu", "Yin-Lin Zhu", "Di Wu", "Yi-Peng Zhou", "Miao Hu", "Guo-Cong Quan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.06403", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "task representation", "backend cost", "budget consumption" ], "cost_reduction_reported": "Up to 14.18% higher reward than state-of-the-art baselines across budget regimes on a composite multimodal benchmark", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Schedules multimodal LLM requests across heterogeneous backends whose costs vary over time, using an adapter-based reward predictor, a primal-dual constraint handler for multiple long-horizon budgets, and a two-phase explore-exploit scheduler. Provides regret bounds under multi-dimensional knapsack constraints.", "key_idea": "Online constrained bandit routing across backends with time-varying costs and multiple budgets.", "relevance_score": 4, "relevance_rationale": "Online budget-paced routing with non-stationary costs.", "found_via": "snowball: cited-by li-2025-llm-bandit", "cites_or_cited_by": [ "li-2025-llm-bandit" ] }, { "id": "zhang-2026-po-coin-flip", "title": "Prompt Optimization Is a Coin Flip: Diagnosing When It Helps in Compound AI Systems", "authors": [ "Xing Zhang", "Guanghui Wang", "Yanwei Cui", "Wei Qiu", "Ziyuan Li", "Bing Zhu", "Peiyang He" ], "year": 2026, "venue_or_source": "CTB@ICML 2026 Workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2604.14585", "code_url": null, "topic_clusters": [ "C4", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "zero-shot vs optimized score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude Haiku 4.5", "Amazon Nova Lite" ], "production_evidence": false, "summary": "Large-scale study (72+ optimization runs, 18,000 grid evaluations) finding that prompt optimization in compound systems is often no better than chance: 49% of runs on Claude Haiku 4.5 score below zero-shot. Agent-prompt interaction effects were never significant; gains appear only when a task has exploitable output structure.", "key_idea": "Prompt optimization in compound systems helps only with exploitable output structure; joint optimization rarely matters.", "relevance_score": 4, "relevance_rationale": "Rigorous negative result that informs whether to invest in prompt vs. model/config optimization.", "found_via": "search: joint optimization model selection prompt compound AI system cost quality arXiv", "cites_or_cited_by": [ "khattab-2023-dspy", "yuksekgonul-2024-textgrad" ] }, { "id": "zhang-2026-racer", "title": "Reasoning Is Not Free: Robust Adaptive Cost-Efficient Routing for LLM-as-a-Judge", "authors": [ "Wenbo Zhang", "Lijinghua Zhang", "Liner Xiang", "Hengrui Cai" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2605.10805", "code_url": null, "topic_clusters": [ "C1", "C7", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "judge task features", "KL uncertainty set" ], "cost_reduction_reported": "Superior accuracy-cost trade-offs under distribution shift (no single number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows reasoning judges help on math and code verification but not on simpler evaluations, and routes between reasoning and non-reasoning judges under a fixed budget via distributionally robust optimisation.", "key_idea": "Budgeted, shift-robust routing between reasoning and non-reasoning judges.", "relevance_score": 4, "relevance_rationale": "Cost control for LLM-as-judge in evaluation loops.", "found_via": "web search: route between reasoning and non-reasoning models", "cites_or_cited_by": [] }, { "id": "zhang-2026-v0-generalist-value-model", "title": "V_0: A Generalist Value Model for Any Policy at State Zero", "authors": [ "Yi-Kai Zhang", "Zhiyuan Yao", "Hongyan Hao", "Yueqing Sun", "Qi Gu", "Hui Su", "Xunliang Cai", "De-Chuan Zhan", "Han-Jia Ye" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.03584", "code_url": null, "topic_clusters": [ "C1", "C6", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "instruction-performance history", "prompt" ], "cost_reduction_reported": "Pareto-optimal performance-cost trade-off in LLM routing tasks (qualitative)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Trains a generalist value model that predicts the expected success of any model on an unseen prompt by conditioning on a history of that model's instruction-performance pairs instead of fitting parameters. Used to allocate GRPO rollout budgets and, at deployment, as a router.", "key_idea": "Predict per-prompt success of any model from its performance profile and route by it.", "relevance_score": 4, "relevance_rationale": "Performance prediction that adapts to new models without retraining; used as a router.", "found_via": "query: off-policy evaluation LLM routing", "cites_or_cited_by": [] }, { "id": "zhang-2026-xconf-experiential-confidence", "title": "Confidence Comes from Experience: Experiential Confidence Estimation from Reasoning to Agents", "authors": [ "Caiqi Zhang", "Xiaochen Zhu", "Chengzu Li", "Yulong Chen", "Dharshan Kumaran", "Nigel Collier" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.17708", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "experiential confidence" ], "cost_reduction_reported": "Matches or beats ten-sample self-consistency AUROC on 23/24 comparisons at a tenth of the generation cost", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "XConf estimates confidence using experience from past inferences rather than only the current process; needs no logits or weight updates and costs one generation, across reasoning, coding, multimodal QA and agent tasks.", "key_idea": "Confidence from past experience, one generation.", "relevance_score": 4, "relevance_rationale": "Cheap confidence for ship/escalate/retry decisions.", "found_via": "hf-papers search: confidence estimation routing escalation", "cites_or_cited_by": [] }, { "id": "zhao-2026-atom", "title": "ATOM: Instantiating Budget-Controllable Multi-Agent Collaboration via Nucleus-Electron Hierarchy", "authors": [ "Xinkui Zhao", "Sai Liu", "Yifan Zhang", "Qingyu Ma", "Zewen Lin", "Naibo Wang", "Guanjie Cheng", "Chang Liu", "Yueshen Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.26178", "code_url": null, "topic_clusters": [ "C4", "C12" ], "routing_granularity": "query", "learning_regime": "RL-online", "signals_used": [ "query difficulty", "budget" ], "cost_reduction_reported": "Improves token efficiency by up to 30% vs strong baselines across six benchmarks", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ATOM generates budget-controllable multi-agent collaboration graphs, keeping an offline-learned backbone and activating query-conditioned agents according to estimated difficulty.", "key_idea": "Difficulty-aware budget control over which agents are instantiated.", "relevance_score": 4, "relevance_rationale": "Budget-controlled MAS construction.", "found_via": "search: BAMAS budget-aware multi-agent systems LLM provisioning ILP arXiv", "cites_or_cited_by": [] }, { "id": "zhong-2026-dualspec", "title": "DualSpec: Accelerating Deep Research Agents via Dual-Process Action Speculation", "authors": [ "Shuzhang Zhong", "Baotong Lu", "Qi Chen", "Chuanjie Liu", "Fan Yang", "Meng Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.07416", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "action-type entropy", "confidence-based semantic verifier" ], "cost_reduction_reported": "Up to 3.28x end-to-end speedup with accuracy comparable to fully reasoning agents", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "DualSpec observes that in deep-research agents Search actions are high-entropy and need explicit reasoning, whereas Visit actions are low-entropy and depend mainly on model capacity. It speculates actions heterogeneously by action type and verifies them with a lightweight confidence-based semantic checker.", "key_idea": "Assign different speculation/reasoning resources by action type in agent loops.", "relevance_score": 4, "relevance_rationale": "Shows action-type heterogeneity in capacity needs, which is direct evidence for step-type routing; the gain reported is latency.", "found_via": "arXiv query ti:\"speculative\" AND ti:\"actions\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "zhong-2026-enterpriseclawbench", "title": "EnterpriseClawBench: Benchmarking Agents from Real Workplace Sessions", "authors": [ "Jincheng Zhong", "Weizhi Wang", "Che Jiang", "Kai Tian", "Zhenzhao Yuan", "Junlin Yang", "Dianqiao Lei", "Kaiyan Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.23654", "code_url": "https://github.com/FrontisAI/EnterpriseClawBench", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "rubrics", "hard rules" ], "cost_reduction_reported": null, "benchmarks": [ "EnterpriseClawBench (852 tasks from real workplace sessions)" ], "models_studied": [ "Codex with GPT-5.5", "other harness-model combinations" ], "production_evidence": false, "summary": "Builds 852 reproducible enterprise tasks from proprietary real workplace agent sessions, releasing the construction and evaluation protocol but not the data. The best configuration scores 0.663, and the authors argue reports must include harness-model pairing, cost and runtime.", "key_idea": "Benchmark mined from real enterprise agent logs.", "relevance_score": 4, "relevance_rationale": "Shows how to turn production sessions into an offline benchmark (relevant to C6).", "found_via": "web search: enterprise agent benchmark cost efficiency", "cites_or_cited_by": [] }, { "id": "zhou-2026-select-then-solve", "title": "Select-then-Solve: Paradigm Routing as Inference-Time Optimization for LLM Agents", "authors": [ "Heng Zhou", "Zelin Tan", "Zhemeng Zhang", "Yutao Fan", "Yibing Lin", "Li Kang", "Xiufeng Song", "Rui Li", "Songtao Huang", "Ao Yu", "Yuchen Fan", "Yanxu Chen", "Kaixin Xu", "Xiaohong Liu", "Yiran Qin", "Philip Torr", "Chen Zhang", "Zhenfei Yin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.06753", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding" ], "cost_reduction_reported": null, "benchmarks": [ "ten benchmarks incl. GAIA", "HumanEval" ], "models_studied": [ "four frontier LLMs incl. GPT-5" ], "production_evidence": false, "summary": "Compares six reasoning paradigms (Direct, CoT, ReAct, Plan-Execute, Reflection, ReCode) over four LLMs and ten benchmarks (~18,000 runs) and finds no paradigm dominates. A light embedding router that picks the paradigm per task raises average accuracy from 47.6% to 53.1%.", "key_idea": "Route each task to the best agent paradigm, not just the best model.", "relevance_score": 4, "relevance_rationale": "Architecture/paradigm routing per query.", "found_via": "web search: routing among agents", "cites_or_cited_by": [] }, { "id": "ziller-2026-greenserv", "title": "GreenServ: Energy-Efficient Context-Aware Dynamic Routing for Multi-Model LLM Inference", "authors": [ "Thomas Ziller", "Shashikant Ilager", "Alessandro Tundo", "Ezio Bartocci", "Leonardo Mariani", "Ivona Brandic" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.17551", "code_url": "https://github.com/TZData1/llm-inference-router", "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "task type", "semantic cluster", "text complexity", "observed accuracy", "energy" ], "cost_reduction_reported": "Compared with random routing, 22% higher accuracy with 31% lower cumulative energy across five tasks and 16 open LLMs; 71.7% average accuracy on RouterBench", "benchmarks": [ "RouterBench" ], "models_studied": [ "16 open-access LLMs" ], "production_evidence": false, "summary": "Routes each query among 16 open LLMs with a contextual multi-armed bandit that uses lightweight query features and learns online from partial feedback on accuracy and energy. It needs no offline calibration and new models can be added without retraining.", "key_idea": "Contextual bandit routing that optimizes accuracy against energy and onboards new models online.", "relevance_score": 4, "relevance_rationale": "Online partial-feedback routing with cold-start for new models; code released.", "found_via": "query: abs:\"bandit\" AND abs:\"LLM\" AND abs:\"routing\"", "cites_or_cited_by": [] }, { "id": "zu-2026-barouter", "title": "BARouter: A Budget-adaptive Online Large Language Model Router Framework", "authors": [ "Ling-Kai Zu", "Xiyue Peng", "Xin Liu" ], "year": 2026, "venue_or_source": "The Web Conference (WWW) 2026", "source_type": "paper", "url": "https://doi.org/10.1145/3774904.3792725", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "estimated response quality", "query cost", "real-time budget consumption" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "An online router for LLM serving that adjusts its policy per query from estimated quality, query cost and live budget consumption, so it adapts to different initial budgets and shifting query distributions without manual tuning. Proves sublinear regret and reports outperforming baselines across budget levels.", "key_idea": "Budget-adaptive online routing with regret guarantees under shifting query distributions.", "relevance_score": 4, "relevance_rationale": "Online budget-paced routing robust to distribution shift; same problem as ParetoBandit.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "aggarwal-2025-optimalthinkingbench", "title": "OptimalThinkingBench: Evaluating Over and Underthinking in LLMs", "authors": [ "Pranjal Aggarwal", "Seungone Kim", "Jack Lanchantin", "Sean Welleck", "Jason Weston", "Ilia Kulikov", "Swarnadeep Saha" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.13141", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "thinking-adjusted accuracy" ], "cost_reduction_reported": null, "benchmarks": [ "OverthinkingBench", "UnderthinkingBench" ], "models_studied": [ "33 thinking and non-thinking models" ], "production_evidence": false, "summary": "Benchmark that jointly scores overthinking on simple queries across 72 domains and underthinking on hard reasoning tasks. None of 33 models thinks optimally; methods that help one side hurt the other.", "key_idea": "Unified over/underthinking benchmark.", "relevance_score": 4, "relevance_rationale": "Evaluation target for reasoning-mode routers.", "found_via": "arXiv search: thinking vs non-thinking (related)", "cites_or_cited_by": [] }, { "id": "agrawal-2025-gepa", "title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning", "authors": [ "Lakshya A Agrawal", "Shangyin Tan", "Dilara Soylu", "Noah Ziems", "Rishi Khare", "Krista Opsahl-Ong", "Arnav Singhvi", "Herumb Shandilya", "Michael J Ryan", "Meng Jiang", "Christopher Potts", "Koushik Sen", "Alexandros G. Dimakis", "Ion Stoica", "Dan Klein", "et al." ], "year": 2025, "venue_or_source": "ICLR 2026 (Oral)", "source_type": "paper", "url": "https://arxiv.org/abs/2507.19457", "code_url": "https://github.com/gepa-ai/gepa", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "natural-language reflection on trajectories", "Pareto frontier of candidates" ], "cost_reduction_reported": "Outperforms GRPO by 6% on average (up to 20%) using up to 35x fewer rollouts; beats MIPROv2 by over 10% (+12% on AIME-2025)", "benchmarks": [ "AIME-2025" ], "models_studied": [], "production_evidence": false, "summary": "GEPA is a reflective prompt optimizer for any system with one or more LLM prompts: it samples trajectories, reflects on them in natural language to diagnose failures, proposes prompt updates, and merges complementary lessons from a Pareto frontier of candidates. It is far more sample-efficient than RL fine-tuning.", "key_idea": "Genetic-Pareto reflective prompt evolution for compound AI systems.", "relevance_score": 4, "relevance_rationale": "State-of-the-art compound-system prompt optimizer; industry reports use it to let cheap open models match frontier models.", "found_via": "seed list: GEPA", "cites_or_cited_by": [ "opsahlong-2024-mipro", "singhvi-2025-databricks-gepa-90x", "wang-2026-dropbox-dash-dspy" ] }, { "id": "aws-2025-bedrock-intelligent-prompt-routing", "title": "Use Amazon Bedrock Intelligent Prompt Routing for cost and latency benefits", "authors": [ "Shreyas Subramanian", "Haibo Ding", "Balasubramaniam Srinivasan", "Yun Zhou" ], "year": 2025, "venue_or_source": "AWS Machine Learning blog", "source_type": "blog", "url": "https://aws.amazon.com/blogs/machine-learning/use-amazon-bedrock-intelligent-prompt-routing-for-cost-and-latency-benefits/", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted response quality per model" ], "cost_reduction_reported": "(vendor-reported) Internal tests: 56% savings for Anthropic family (Haiku 3.5 + Sonnet 3.5 v2) at Sonnet quality, 35% Nova, 16% Llama; 63.6% average savings on RAG datasets; ~85 ms P90 router overhead", "benchmarks": [ "internal AWS datasets incl. RAG" ], "models_studied": [ "Claude 3/3.5 Haiku", "Claude 3.5 Sonnet", "Llama 3.x", "Amazon Nova Pro/Lite" ], "production_evidence": false, "summary": "GA announcement of Bedrock's managed router, which predicts per-request quality for two models in the same family and routes on a cost-quality criterion. Caveats: English and chat-optimized, pairwise only, not adaptable to application data, may underperform on specialized tasks. Docs: docs.aws.amazon.com/bedrock/latest/userguide/prompt-routing.html.", "key_idea": "Managed pairwise quality-prediction router inside a cloud platform.", "relevance_score": 4, "relevance_rationale": "Seed; major cloud router product with stated caveats.", "found_via": "seed: AWS Bedrock intelligent prompt routing", "cites_or_cited_by": [] }, { "id": "ayala-2025-slm-vs-llm-low-code-workflows", "title": "Fine-Tune an SLM or Prompt an LLM? The Case of Generating Low-Code Workflows", "authors": [ "Orlando Marquez Ayala", "Patrice Bechard", "Emily Chen", "Maggie Baird", "Jingfei Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.24189", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "task fine-tuning" ], "cost_reduction_reported": "Fine-tuning an SLM improves quality by 10% on average over prompting GPT-4o", "benchmarks": [ "low-code workflow generation (JSON)" ], "models_studied": [ "GPT-4o", "SLMs" ], "production_evidence": true, "summary": "ServiceNow study on generating structured low-code workflows: fine-tuned small models beat well-prompted GPT-4o by about 10% on average, with error analysis of remaining limits.", "key_idea": "For structured enterprise outputs, fine-tuned SLMs beat prompted frontier LLMs.", "relevance_score": 4, "relevance_rationale": "Enterprise evidence directly relevant to this project.", "found_via": "hf-papers search: customer service agent small model fine-tune", "cites_or_cited_by": [] }, { "id": "barres-2025-tau2-bench", "title": "τ²-Bench: Evaluating Conversational Agents in a Dual-Control Environment", "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.07982", "code_url": "https://github.com/sierra-research/tau2-bench", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "environment state" ], "cost_reduction_reported": null, "benchmarks": [ "τ²-bench telecom", "retail", "airline" ], "models_studied": [ "frontier LLM agents" ], "production_evidence": false, "summary": "Extends τ-bench to dual-control settings where the simulated user also acts on a shared environment, modeled as a Dec-POMDP, with a new telecom technical-support domain, a compositional task generator, and a tool-constrained user simulator. Agents drop sharply when moving from no-user to dual-control modes.", "key_idea": "Dual-control user-agent tasks for technical support.", "relevance_score": 4, "relevance_rationale": "Directly relevant to IT/tech-support agents; used by later routing evaluations.", "found_via": "web search: tau2-bench dual-control", "cites_or_cited_by": [ "kumar-2026-task-session-routing-eval", "yao-2024-tau-bench" ] }, { "id": "bian-2025-tokencake", "title": "TokenCake: A KV-Cache-centric Serving Framework for LLM-based Multi-Agent Applications", "authors": [ "Zhuohang Bian", "Feiyang Wu", "Zhuoran Li", "Teng Ma", "Youwei Zhuo" ], "year": 2025, "venue_or_source": "EuroSys 2027", "source_type": "paper", "url": "https://arxiv.org/abs/2510.18586", "code_url": "https://github.com/pku-lemonade/TokenCake", "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "agent graph structure", "function-call idle state" ], "cost_reduction_reported": "end-to-end latency reduced by up to 47.06% in memory-constrained settings and effective GPU utilization improved 16.9 points vs vLLM", "benchmarks": [ "multi-agent benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "KV-centric serving for multi-agent apps: a temporal scheduler offloads KV caches of agents stalled on function calls and prefetches them back, and a spatial scheduler reserves GPU memory for critical-path agents.", "key_idea": "Offload idle agents' KV during function calls; prioritize critical-path agents.", "relevance_score": 4, "relevance_rationale": "Multi-agent-aware memory management.", "found_via": "Tokencake KV cache multi-agent function call serving", "cites_or_cited_by": [ "bian-2026-tokendance", "pan-2025-kvflow", "xia-2026-mori" ] }, { "id": "bijoy-2025-prost-slm-multi-agent", "title": "ProST: Progressive Sub-task Training for Pareto-Optimal Multi-agent Systems Using Small Language Models", "authors": [ "Biddut Sarker Bijoy", "Mohammad Saqib Hasan", "Pegah Alipoormolabashi", "Avirup Sil", "Aruna Balasubramanian", "Niranjan Balasubramanian" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.04508", "code_url": null, "topic_clusters": [ "C3", "C8", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [], "cost_reduction_reported": "Progressive sub-task training improves effectiveness of SLM multi-agent systems at all configurations (Pareto analysis in paper)", "benchmarks": [ "AppWorld" ], "models_studied": [ "various-size LMs" ], "production_evidence": false, "summary": "Compares single-agent LLM systems with multi-agent SLM systems on AppWorld for effectiveness and efficiency, finding SLMs struggle with long trajectories even with role specialization. Progressive sub-task training, a curriculum over subtasks, improves SLM multi-agent results.", "key_idea": "Curriculum over subtasks to make SLM role agents Pareto-competitive.", "relevance_score": 4, "relevance_rationale": "Single-LLM vs multi-SLM trade-off with training fix.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "cemri-2025-mast-why-mas-fail", "title": "Why Do Multi-Agent LLM Systems Fail?", "authors": [ "Mert Cemri", "Melissa Z. Pan", "Shuyi Yang", "Lakshya A. Agrawal", "Bhavya Chopra", "Rishabh Tiwari", "Kurt Keutzer", "Aditya Parameswaran", "Dan Klein", "Kannan Ramchandran", "Matei Zaharia", "Joseph E. Gonzalez", "Ion Stoica" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.13657", "code_url": "https://github.com/multi-agent-systems-failure-taxonomy/MAST", "topic_clusters": [ "C7", "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM-as-judge failure annotation", "failure taxonomy", "LLM judge" ], "cost_reduction_reported": null, "benchmarks": [ "MAST-Data (1600+ traces, 7 MAS frameworks)", "MAST annotated traces (5 MAS frameworks, 150+ tasks)" ], "models_studied": [ "GPT-4", "Claude 3", "Qwen2.5", "CodeLlama" ], "production_evidence": false, "summary": "Builds MAST, a taxonomy of 14 multi-agent failure modes in three categories (system design, inter-agent misalignment, task verification) from expert annotation of traces, plus an LLM-judge annotator and a 1600+ trace dataset. Shows that MAS gains on benchmarks are often minimal and that simple fixes give limited improvement.", "key_idea": "Taxonomy and dataset of why multi-agent systems fail.", "relevance_score": 4, "relevance_rationale": "Seed; evidence on when MAS is worth it and a failure-detection signal source.", "found_via": "web search: seed | seed", "cites_or_cited_by": [ "chen-2024-optima", "chen-2026-signals-trajectory-triage" ] }, { "id": "cencerrado-2025-no-answer-needed", "title": "No Answer Needed: Predicting LLM Answer Accuracy from Question-Only Linear Probes", "authors": [ "Iván Vicente Moreno Cencerrado", "Arnau Padrés Masdemont", "Anton Gonzalvez Hawthorne", "David Demitri Africa", "Lorenzo Pacchiardi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.10625", "code_url": null, "topic_clusters": [ "C1", "C6", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "pre-generation activations (linear probe)", "pre-generation linear probe" ], "cost_reduction_reported": null, "benchmarks": [ "trivia and OOD knowledge datasets", "TriviaQA", "OOD knowledge datasets" ], "models_studied": [ "open models 7B-70B", "7B-70B open models" ], "production_evidence": false, "summary": "Linear probes on activations after reading a question, before generation, predict whether the model's answer will be correct, generalizing out of distribution except on math.", "key_idea": "Pre-generation correctness probes.", "relevance_score": 4, "relevance_rationale": "Cheap pre-generation success signal for routing/cascades.", "found_via": "snowball: cited-by pacchiardi-2025-predictaboard | hf-papers search: probe predict success before generation", "cites_or_cited_by": [ "lugoloobi-2026-llms-encode-failures", "pacchiardi-2025-predictaboard" ] }, { "id": "chae-2025-web-shepherd", "title": "Web-Shepherd: Advancing PRMs for Reinforcing Web Agents", "authors": [ "Hyungjoo Chae", "Sunghwan Kim", "Junhee Cho", "Seungone Kim", "Seungjun Moon", "Gyeom Hwangbo", "Dongha Lim", "Minjin Kim", "Yeonjun Hwang", "Minju Gwak", "Dongwook Choi", "Minseok Kang", "Gwanhoon Im", "ByeongUng Cho", "Hyojun Kim", "Jun Hee Han", "Taeyoon Kwon", "Minju Kim", "Beong-woo Kwak", "Dongjin Kang", "Jinyoung Yeo" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.15277", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward", "checklists" ], "cost_reduction_reported": "About 30 points better accuracy than GPT-4o on WebRewardBench; as verifier on WebArena-lite with GPT-4o-mini policy, 10.9 points better at roughly 10x less cost than GPT-4o-mini as verifier", "benchmarks": [ "WebRewardBench", "WebArena-lite" ], "models_studied": [ "GPT-4o", "GPT-4o-mini" ], "production_evidence": false, "summary": "First process reward model for web navigation, trained on 40K step-level preference pairs with checklists; outperforms GPT-4o as a step-level judge while being far cheaper.", "key_idea": "Small dedicated web-agent PRM replaces expensive MLLM judges.", "relevance_score": 4, "relevance_rationale": "Cheap step-level signal for web agents.", "found_via": "keyword: step-level reward web agent", "cites_or_cited_by": [ "lightman-2023-lets-verify-step-by-step" ] }, { "id": "chakraborty-2025-gencache", "title": "Generative Caching for Structurally Similar Prompts and Responses", "authors": [ "Sarthak Chakraborty", "Suman Nath", "Xuchao Zhang", "Chetan Bansal", "Indranil Gupta" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2511.17565", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "prompt structure similarity", "response pattern templates" ], "cost_reduction_reported": "83% cache hit rate with minimal incorrect hits on datasets without prompt repetition; in agentic workflows ~20% higher hit rate and ~34% lower end-to-end latency vs standard prompt matching", "benchmarks": [ "agentic workflow traces", "structurally similar prompt datasets" ], "models_studied": [], "production_evidence": false, "summary": "A generative cache for prompts that share structure but differ in details (common in agent steps): it learns reusable response patterns and synthesizes a customized response for a new prompt instead of returning a stale exact answer.", "key_idea": "Cache response templates for structurally similar agent prompts and fill in the differences.", "relevance_score": 4, "relevance_rationale": "Directly targets repeated, structured LLM calls inside agent workflows.", "found_via": "\"Don't Break the Cache\" prompt caching agentic tasks evaluation", "cites_or_cited_by": [ "zhang-2025-agentic-plan-caching" ] }, { "id": "cheema-2025-tweakllm", "title": "TweakLLM: A Routing Architecture for Dynamic Tailoring of Cached Responses", "authors": [ "Muhammad Taha Cheema", "Abeer Aamir", "Khawaja Gul Muhammad", "Naveed Anwar Bhatti", "Ihsan Ayyub Qazi", "Zafar Ayyub Qazi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.23674", "code_url": null, "topic_clusters": [ "C1", "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "semantic cache hit", "lightweight LLM adaptation" ], "cost_reduction_reported": null, "benchmarks": [ "real-world query datasets", "user studies" ], "models_studied": [ "lightweight LLM", "frontier LLMs" ], "production_evidence": false, "summary": "Routes semantically matched queries to a small LLM that edits the cached response to fit the new prompt, instead of returning it verbatim or calling the frontier model. User studies and LLM debates suggest quality comparable to frontier models with better cache effectiveness.", "key_idea": "Cache hit + small-model rewrite as a cheap route.", "relevance_score": 4, "relevance_rationale": "Combines caching with routing to a small model: a direct cost-quality lever.", "found_via": "semantic caching LLM false positive hits correctness study", "cites_or_cited_by": [ "bang-2023-gptcache", "koster-2026-memboost" ] }, { "id": "chen-2025-concur", "title": "CONCUR: A Framework for Continual Constrained and Unconstrained Routing", "authors": [ "Peter Baile Chen", "Weiyue Li", "Dan Roth", "Michael Cafarella", "Samuel Madden", "Jacob Andreas" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.09386", "code_url": null, "topic_clusters": [ "C1", "C4", "C5", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "per-strategy predictors", "multiple task representations", "task representations", "strategy representations", "budget" ], "cost_reduction_reported": null, "benchmarks": [ "knowledge- and reasoning-intensive tasks (ID and OOD)" ], "models_studied": [], "production_evidence": false, "summary": "CONCUR routes tasks to computation strategies (model plus decoding method) with separate per-strategy predictors, supporting budgeted and unbudgeted routing and continual addition of strategies.", "key_idea": "Modular continual routing over strategies.", "relevance_score": 4, "relevance_rationale": "Strategy-level routing.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: cited-by nguyen-2024-metallm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "chen-2025-mao-arag", "title": "MAO-ARAG: Multi-Agent Orchestration for Adaptive Retrieval-Augmented Generation", "authors": [ "Yiqun Chen", "Erhan Zhang", "Lingyong Yan", "Shuaiqiang Wang", "Jizhou Huang", "Dawei Yin", "Jiaxin Mao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.01005", "code_url": "https://github.com/chenyiqun/Agentic-RAG", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "RL-online", "signals_used": [ "F1 reward", "cost penalty" ], "cost_reduction_reported": "Maintains cost and latency within acceptable limits while achieving high answer quality", "benchmarks": [ "multiple QA datasets" ], "models_studied": [], "production_evidence": false, "summary": "MAO-ARAG uses an RL-trained planner agent that assembles a per-query RAG workflow from executor agents (reformulation, selection, generation), rewarded on F1 minus a cost penalty. It adapts pipeline complexity to query difficulty.", "key_idea": "RL planner assembles per-query agent workflows with a cost penalty.", "relevance_score": 4, "relevance_rationale": "Cost-penalized per-query agent orchestration.", "found_via": "web search: planner executor", "cites_or_cited_by": [] }, { "id": "chen-2025-netgpt-routing-synergy", "title": "Optimizing NetGPT via Routing-Based Synergy and Reinforcement Learning", "authors": [ "Yuxuan Chen", "Rong-Peng Li", "Xian-Fu Chen", "Celimuge Wu", "Chenghui Peng", "Zhi-Feng Zhao", "Hong-Gang Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.22217", "code_url": null, "topic_clusters": [ "C3", "C5", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "bandwidth", "RTT", "scoring policy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "For edge-deployed network agents, structured tool-calling requests are routed to a cloud or edge agent by a scoring policy that provably reduces to a monotone fallback threshold in bandwidth and RTT. Cloud-routed data is then used for schema-preserving RL to improve the edge agent over time.", "key_idea": "Edge/cloud routing of tool calls with self-improvement of the small edge agent from cloud traces.", "relevance_score": 4, "relevance_rationale": "Combines routing of tool calls with distillation from routed traffic (sub-themes c and d).", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "chen-2025-tagrouter", "title": "TagRouter: Learning Route to LLMs through Tags for Open-Domain Text Generation Tasks", "authors": [ "Zhou Chen", "Zhiqiang Wei", "Yuqi Bai", "Xue Xiong", "Jianmin Wu" ], "year": 2025, "venue_or_source": "ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.12473", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query tags", "tag-model performance statistics" ], "cost_reduction_reported": "Increases system accept rate by 6.15% and reduces costs by 17.20% vs 13 baselines (abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "TagRouter is a training-free routing method for open-domain text generation that routes via tags derived from queries and model performance per tag. It outperforms 13 baselines on accept rate and cost.", "key_idea": "Tag-based training-free routing for open-ended generation.", "relevance_score": 4, "relevance_rationale": "Production-oriented (industry authors).", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "chiang-2025-dueling-routing", "title": "LLM Routing with Dueling Feedback", "authors": [ "Chao-Kai Chiang", "Takashi Ishida", "Masashi Sugiyama" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.00841", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "pairwise preference feedback", "category-calibrated model embeddings", "model embeddings" ], "cost_reduction_reported": null, "benchmarks": [ "RouterBench", "MixInstruct" ], "models_studied": [], "production_evidence": false, "summary": "Formulates routing as contextual dueling bandits learning from pairwise preferences, with model embeddings derived by category-calibrated contrastive fine-tuning and Feel-Good Thompson Sampling. Achieves lower regret and better cost-performance balance.", "key_idea": "Contextual dueling bandits for preference-feedback routing.", "relevance_score": 4, "relevance_rationale": "Online learning from pairwise feedback.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm", "panda-2025-pilot" ] }, { "id": "choudhury-2025-agentprm-choudhury", "title": "Process Reward Models for LLM Agents: Practical Framework and Directions", "authors": [ "Sanjiban Choudhury" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.10325", "code_url": "https://github.com/sanjibanc/agent_prm", "topic_clusters": [ "C7", "C8" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "process reward", "Monte Carlo rollouts", "inverse PRM" ], "cost_reduction_reported": "3B models trained with AgentPRM/InversePRM outperform GPT-4o baselines on ALFWorld", "benchmarks": [ "ALFWorld" ], "models_studied": [ "3B models", "GPT-4o" ], "production_evidence": false, "summary": "Presents AgentPRM, an actor-critic style framework where Monte Carlo rollouts produce process-reward targets used to train and improve LLM agents, plus InversePRM that learns process rewards from demonstrations. Small 3B models trained this way beat GPT-4o baselines on ALFWorld; the paper also analyzes test-time scaling and reward hacking.", "key_idea": "Lightweight process reward models for agents, learned from rollouts or demonstrations.", "relevance_score": 4, "relevance_rationale": "Directly usable step-level signal and a path to cheap small agents.", "found_via": "seed (AgentPRM)", "cites_or_cited_by": [ "lightman-2023-lets-verify-step-by-step", "xi-2025-agentprm-xi" ] }, { "id": "chuang-2025-confident-or-seek-stronger", "title": "Confident or Seek Stronger: Exploring Uncertainty-Based On-device LLM Routing From Benchmarking to Generalization", "authors": [ "Yu-Neng Chuang", "Leisheng Yu", "Guanchu Wang", "Lizhe Zhang", "Zirui Liu", "Xuanting Cai", "Yang Sui", "Vladimir Braverman", "Xia Hu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.04428", "code_url": null, "topic_clusters": [ "C1", "C2", "C7", "C13" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "uncertainty quantification methods", "calibration data", "uncertainty quantification" ], "cost_reduction_reported": null, "benchmarks": [ "1500+ settings" ], "models_studied": [], "production_evidence": false, "summary": "Benchmarks uncertainty-based routing from on-device small models to LLMs across 1500+ settings, finding that uncertainty-correctness alignment matters most and depends on SLM and UQ method. Proposes a calibration-data pipeline for generalising routing thresholds.", "key_idea": "Large benchmark of UQ-based SLM-to-LLM escalation.", "relevance_score": 4, "relevance_rationale": "Empirical guide for picking escalation signals.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs | hf-papers search: confidence estimation routing escalation", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "databricks-2025-tao", "title": "TAO: Using test-time compute to train efficient LLMs without labeled data", "authors": [ "Databricks AI Research Team" ], "year": 2025, "venue_or_source": "Databricks blog", "source_type": "blog", "url": "https://www.databricks.com/blog/tao-using-test-time-compute-train-efficient-llms-without-labeled-data", "code_url": null, "topic_clusters": [ "C6", "C7", "C8" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "unlabeled usage inputs", "reward model (DBRM) scores" ], "cost_reduction_reported": "Brings Llama 3.1 8B and Llama 3.3 70B close to GPT-4o and o3-mini on FinanceBench, DB Enterprise Arena and BIRD-SQL without labels, at the base model's inference cost; multi-task TAO +2.4% for Llama 3.3 70B on a broad enterprise benchmark; notes o3-mini emits 5-10x more output tokens (vendor-reported)", "benchmarks": [ "FinanceBench", "DB Enterprise Arena", "BIRD-SQL (Databricks dialect)" ], "models_studied": [ "Llama 3.1 8B", "Llama 3.3 70B", "GPT-4o", "GPT-4o-mini", "o3-mini" ], "production_evidence": false, "summary": "TAO uses only unlabeled task inputs such as production prompts: it samples many candidate responses with test-time compute, scores them with an enterprise reward model, and trains the model with RL so the tuned model answers directly with no extra inference compute. It outperforms labeled fine-tuning on three enterprise tasks and scales with tuning compute budget.", "key_idea": "Convert test-time compute plus a reward model into a cheaper specialised model using only unlabeled usage data.", "relevance_score": 4, "relevance_rationale": "Directly addresses learning cheaper models from customer usage data without labels.", "found_via": "direct: databricks.com/blog", "cites_or_cited_by": [] }, { "id": "davidson-2025-collaboration-gap", "title": "The Collaboration Gap: Exploration and Benchmarking of Open-World Agentic Cooperation", "authors": [ "Tim R. Davidson", "Adam Fourney", "Saleema Amershi", "Robert West", "Eric Horvitz", "Ece Kamar" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.02687", "code_url": null, "topic_clusters": [ "C3", "C12", "C13" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "pairing type" ], "cost_reduction_reported": null, "benchmarks": [ "collaborative maze-solving benchmark" ], "models_studied": [ "32 open and closed models" ], "production_evidence": false, "summary": "A maze benchmark that isolates collaboration among heterogeneous agents shows that models strong when solo often degrade when paired. A relay in which a stronger agent starts and then hands off to a weaker one closes much of the gap.", "key_idea": "Strong-first, weak-later relay mitigates the heterogeneous collaboration gap.", "relevance_score": 4, "relevance_rationale": "Evidence on ordering big and small models across the steps of a shared task.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "ding-2025-best-route", "title": "BEST-Route: Adaptive LLM Routing with Test-Time Optimal Compute", "authors": [ "Dujian Ding", "Ankur Mallick", "Shaokun Zhang", "Chi Wang", "Daniel Madrigal", "Mirian Del Carmen Hipolito Garcia", "Menglin Xia", "Laks V. S. Lakshmanan", "Qingyun Wu", "Victor Rühle" ], "year": 2025, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.22716", "code_url": null, "topic_clusters": [ "C1", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "difficulty estimate", "quality threshold", "best-of-n sampling count", "query difficulty", "quality thresholds" ], "cost_reduction_reported": "Reduces costs by up to 60% with less than 1% performance drop on real-world datasets", "benchmarks": [ "real-world datasets" ], "models_studied": [], "production_evidence": false, "summary": "BEST-Route chooses both the model and how many responses to sample from it (best-of-n), based on query difficulty and a quality threshold. Sampling multiple answers from a small model often beats one answer from a large model at lower cost.", "key_idea": "Jointly route over model choice and test-time sample count.", "relevance_score": 4, "relevance_rationale": "Seed; bridges routing with test-time compute allocation.", "found_via": "seed | GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm", "sui-2025-stop-overthinking" ] }, { "id": "el-2025-inefficiencies-meta-agents", "title": "Inefficiencies of Meta Agents for Agent Design", "authors": [ "Batu El", "Mert Yuksekgonul", "James Zou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.06711", "code_url": null, "topic_clusters": [ "C4", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "design cost", "per-example inference cost" ], "cost_reduction_reported": "Only on two datasets does total design-plus-deployment cost of meta-designed agents fall below human-designed agents, and only when deployed on over 15,000 examples", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Examines meta-agents that design agents: expanding context with all prior designs underperforms, designed agents lack diversity, and the design cost is recouped only in a few datasets at large deployment scale.", "key_idea": "Automated agent design rarely pays back its design cost.", "relevance_score": 4, "relevance_rationale": "Cost accounting for automated MAS design.", "found_via": "snowball: cited-by zhang-2025-maas-agentic-supernet | search: Inefficiencies of Meta Agents for Agent Design cost", "cites_or_cited_by": [ "hu-2024-adas", "zhang-2025-maas-agentic-supernet" ] }, { "id": "erol-2025-cost-of-pass", "title": "Cost-of-Pass: An Economic Framework for Evaluating Language Models", "authors": [ "Mehmet Hamza Erol", "Batu El", "Mirac Suzgun", "Mert Yuksekgonul", "James Zou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.13359", "code_url": null, "topic_clusters": [ "C12", "C13", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "expected cost of a correct solution", "accuracy", "inference cost" ], "cost_reduction_reported": "Frontier cost-of-pass for complex quantitative tasks roughly halved every few months over the year studied", "benchmarks": [ "basic quantitative, knowledge-intensive and complex quantitative task suites" ], "models_studied": [ "lightweight, large and reasoning LLMs" ], "production_evidence": false, "summary": "Defines cost-of-pass, the expected monetary cost of producing a correct solution, and the frontier cost-of-pass across models and human experts. Finds lightweight models most cost-effective for basic quantitative tasks, large models for knowledge-intensive ones and reasoning models for complex math.", "key_idea": "Cost-of-pass as an economic metric for model selection.", "relevance_score": 4, "relevance_rationale": "Metric used by Efficient Agents and useful for per-step model choice.", "found_via": "snowball: cited-by wang-2025-efficient-agents | web search: Cost-of-Pass", "cites_or_cited_by": [ "kapoor-2024-ai-agents-that-matter", "wang-2025-efficient-agents" ] }, { "id": "fan-2025-cothink", "title": "The Price of a Second Thought: On the Evaluation of Reasoning Efficiency in Large Language Models", "authors": [ "Siqi Fan", "Bowen Qin", "Peng Han", "Shuo Shang", "Yequan Wang", "Aixin Sun" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.22017", "code_url": null, "topic_clusters": [ "C3", "C9", "C13" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "instruct-model outline" ], "cost_reduction_reported": "Cuts token usage by 21.1% while keeping accuracy on GSM8K, MATH500, AIME24 across four thinking models", "benchmarks": [ "GSM8K", "MATH500", "AIME24" ], "models_studied": [ "four thinking models" ], "production_evidence": false, "summary": "Measures reasoning efficiency relative to instruct models as the minimal-effort baseline, finding thinking models waste compute on easy problems. CoThink has an instruct model draft an outline that a thinking model expands.", "key_idea": "Instruct model outlines, thinking model expands.", "relevance_score": 4, "relevance_rationale": "Reasoning vs non-reasoning model collaboration.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "feng-2025-agentrr", "title": "Get Experience from Practice: LLM Agents with Record & Replay", "authors": [ "Erhu Feng", "Wenbo Zhou", "Zibin Liu", "Le Chen", "Yunpeng Dong", "Cheng Zhang", "Yisheng Zhao", "Dong Du", "Zhichao Hua", "Yubin Xia", "Haibo Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.17716", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "recorded traces", "check functions" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AgentRR records agent interaction traces, summarizes them into multi-level experiences, and replays them on similar tasks, with check functions acting as safety anchors. It discusses large-small model collaboration (a small model replays what a large model recorded) as a way to cut deployment cost.", "key_idea": "Record-and-replay of agent experiences to reuse work and allow smaller models.", "relevance_score": 4, "relevance_rationale": "Proposes replaying frontier-model traces with small models, a subagent-level cost lever.", "found_via": "snowball: related work of zhang-2025-agentic-plan-caching", "cites_or_cited_by": [ "wang-2024-agent-workflow-memory" ] }, { "id": "feng-2025-fusionfactory", "title": "FusionFactory: Fusing LLM Capabilities with Multi-LLM Log Data", "authors": [ "Tao Feng", "Haozhen Zhang", "Zijie Lei", "Pengrui Han", "Mostofa Patwary", "Mohammad Shoeybi", "Bryan Catanzaro", "Jiaxuan You", "Mostofa Ali Patwary" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.10540", "code_url": "https://github.com/ulab-uiuc/FusionFactory", "topic_clusters": [ "C1", "C6", "C8", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "multi-LLM log data", "routers", "reasoning templates", "distillation", "multi-LLM logs", "query embedding" ], "cost_reduction_reported": null, "benchmarks": [ "LLMFusionBench (14 tasks)", "LLMFusionBench (14 tasks, 20 LLMs)", "LLMFusionBench (14 tasks, 5 domains)" ], "models_studied": [ "20 open LLMs (8B-671B)", "20 open-source LLMs (8B-671B)" ], "production_evidence": false, "summary": "FusionFactory uses multi-LLM log data to fuse models at three levels: query-level routing, thought-level template reuse, and model-level distillation, outperforming the best single model on all 14 tasks.", "key_idea": "Exploit multi-LLM logs for routing and distillation.", "relevance_score": 4, "relevance_rationale": "Learning from logged data.", "found_via": "snowball: cited-by ong-2024-routellm | OpenAlex title: logged data LLM | web search: FusionBench LLM routing", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "fernandez-2025-radar", "title": "RADAR: Reasoning-Ability and Difficulty-Aware Routing for Reasoning LLMs", "authors": [ "Nigel Fernandez", "Branislav Kveton", "Ryan A. Rossi", "Andrew S. Lan", "Zichao Wang" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2509.25426", "code_url": null, "topic_clusters": [ "C1", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "IRT query difficulty", "model-budget ability" ], "cost_reduction_reported": null, "benchmarks": [ "8 reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "RADAR fits an item response model over model and reasoning-budget pairs, then routes harder queries to higher-ability configurations. It generalizes out of distribution and can add models using a small adaptive query set.", "key_idea": "Psychometric routing over (model, reasoning budget) configurations.", "relevance_score": 4, "relevance_rationale": "Relevant to reasoning-effort selection.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "frick-2025-p2l", "title": "Prompt-to-Leaderboard", "authors": [ "Evan Frick", "Connor Chen", "Joseph Tennyson", "Tianle Li", "Wei-Lin Chiang", "Anastasios N. Angelopoulos", "Ion Stoica" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.14855", "code_url": "https://github.com/lmarena/p2l", "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "prompt-conditioned Bradley-Terry coefficients", "human preference votes (Chatbot Arena)" ], "cost_reduction_reported": null, "benchmarks": [ "Chatbot Arena" ], "models_studied": [], "production_evidence": true, "summary": "Prompt-to-Leaderboard trains an LLM to output prompt-specific Bradley-Terry coefficients over models, producing a per-prompt leaderboard usable for routing and evaluation. A router built this way reached #1 on Chatbot Arena in January 2025, and performance scales with a power law.", "key_idea": "Predict a per-prompt leaderboard from preference data and route to its top model.", "relevance_score": 4, "relevance_rationale": "Uses real Arena traffic/votes; strong preference-based router from LMSYS.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "gao-2025-flowreasoner", "title": "FlowReasoner: Reinforcing Query-Level Meta-Agents", "authors": [ "Hongcheng Gao", "Yue Liu", "Yufei He", "Longxu Dou", "Chao Du", "Zhijie Deng", "Bryan Hooi", "Min Lin", "Tianyu Pang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.15257", "code_url": "https://github.com/sail-sg/FlowReasoner", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "execution feedback", "complexity/efficiency reward", "reward on performance, complexity and efficiency" ], "cost_reduction_reported": null, "benchmarks": [ "engineering and competition code benchmarks (three)" ], "models_studied": [ "DeepSeek R1 (distillation source)", "o1-mini (baseline)", "o1-mini" ], "production_evidence": false, "summary": "FlowReasoner is a reasoning meta-agent, distilled from DeepSeek-R1 and then trained with RL on execution feedback, that designs one multi-agent system per query. Its reward includes performance, complexity and efficiency terms, and it beats o1-mini by 10.52% across three code benchmarks.", "key_idea": "RL-trained meta-agent designs a per-query MAS with an efficiency term in the reward.", "relevance_score": 4, "relevance_rationale": "Query-level MAS design with cost-aware reward.", "found_via": "web search: FlowReasoner | seed list: FlowReasoner", "cites_or_cited_by": [ "yuan-2026-flowbank", "zhang-2024-aflow", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "guan-2025-dynamic-speculative-planning", "title": "Dynamic Speculative Agent Planning", "authors": [ "Yilin Guan", "Qingfeng Lan", "Sun Fei", "Dujian Ding", "Devang Acharya", "Chi Wang", "William Yang Wang", "Wenyue Hua" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.01920", "code_url": "https://github.com/guanyilin428/Dynamic-Speculative-Planning", "topic_clusters": [ "C3", "C5", "C11" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "latency", "dollar cost" ], "cost_reduction_reported": "Comparable efficiency to fastest lossless acceleration while reducing total cost by 30% and unnecessary cost by up to 60%", "benchmarks": [ "two standard agent benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Dynamic Speculative Planning uses a cheaper model to draft agent plans speculatively and an online, asynchronous RL policy to decide how far to speculate, with a single knob trading latency against dollar cost. It keeps outputs lossless while cutting cost relative to prior speculative agents.", "key_idea": "Online RL tunes speculative planning with a cheap drafter to balance latency and cost.", "relevance_score": 4, "relevance_rationale": "Big-little model use within agent planning, learned online.", "found_via": "web search: speculative actions", "cites_or_cited_by": [] }, { "id": "guo-2025-moma-generalized-routing", "title": "Towards Generalized Routing: Model and Agent Orchestration for Adaptive and Efficient Inference", "authors": [ "Xiyu Guo", "Shan Wang", "Chunfang Ji", "Xuefeng Zhao", "Wenhao Xi", "Yaoyao Liu", "Qinglan Li", "Chao Deng", "Junlan Feng" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.07571", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "intent recognition", "capability profiles" ], "cost_reduction_reported": "Superior cost-efficiency and scalability vs existing routers (no single number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MoMA routes each query either to an LLM or to a specialized agent, using profiled model capabilities for LLM routing and a context-aware state machine with dynamic masking for agent selection. It targets mixed ecosystems of models and domain agents.", "key_idea": "Unified router over both LLMs and agents.", "relevance_score": 4, "relevance_rationale": "Routing among agents as well as models.", "found_via": "arXiv title search: model routing agentic", "cites_or_cited_by": [] }, { "id": "huang-2025-crmarena-pro", "title": "CRMArena-Pro: Holistic Assessment of LLM Agents Across Diverse Business Scenarios and Interactions", "authors": [ "Kung-Hsiang Huang", "Akshara Prabhakar", "Onkar Thorat", "Divyansh Agarwal", "Prafulla Kumar Choubey", "Yixin Mao", "Silvio Savarese", "Caiming Xiong", "Chien-Sheng Wu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.18878", "code_url": "https://github.com/SalesforceAIResearch/CRMArena", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Top agents ~58% single-turn and ~35% multi-turn success", "benchmarks": [ "CRMArena-Pro (19 tasks, B2B and B2C)" ], "models_studied": [ "leading LLM agents" ], "production_evidence": false, "summary": "Expands CRMArena to 19 expert-validated sales, service and CPQ tasks in B2B and B2C settings, with persona-driven multi-turn interactions and confidentiality tests. Top agents reach about 58% single-turn and about 35% multi-turn success, and show almost no confidentiality awareness.", "key_idea": "Multi-turn enterprise CRM evaluation with confidentiality checks.", "relevance_score": 4, "relevance_rationale": "Enterprise agent benchmark; multi-turn drop matters for per-step model assignment.", "found_via": "web search: CRMArena-Pro", "cites_or_cited_by": [ "huang-2024-crmarena" ] }, { "id": "huang-2025-routereval", "title": "RouterEval: A Comprehensive Benchmark for Routing LLMs to Explore Model-level Scaling Up in LLMs", "authors": [ "Zhongzhan Huang", "Guoming Ling", "Yupei Lin", "Yandong Chen", "Shanshan Zhong", "Hefeng Wu", "Liang Lin" ], "year": 2025, "venue_or_source": "EMNLP 2025", "source_type": "benchmark", "url": "https://arxiv.org/abs/2503.10657", "code_url": "https://github.com/MilkThink-Lab/RouterEval", "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "precomputed performance records", "per-query model outcomes" ], "cost_reduction_reported": null, "benchmarks": [ "RouterEval (12 LLM evaluations)", "ARC", "HellaSwag", "MMLU", "TruthfulQA", "GSM8K", "MMLU-PRO", "and 6 other evaluations" ], "models_studied": [ "8,500+ LLMs", "8,500+ open LLMs" ], "production_evidence": false, "summary": "RouterEval releases over 200 million performance records from more than 8,500 LLMs on 12 evaluations and reports a model-level scaling phenomenon: a capable router improves as the candidate pool grows, sometimes beating the best single model. Most existing routers leave large headroom on this benchmark.", "key_idea": "Massive routing benchmark revealing model-level scaling with candidate count.", "relevance_score": 4, "relevance_rationale": "Seed benchmark.", "found_via": "seed | seed: RouterEval", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "hu-2024-routerbench", "ong-2024-routellm" ] }, { "id": "jha-2025-itbench", "title": "ITBench: Evaluating AI Agents across Diverse Real-World IT Automation Tasks", "authors": [ "Saurabh Jha", "Rohan Arora", "Yuji Watanabe", "Takumi Yanagawa", "Yinfang Chen", "Jackson Clark", "Bhavya Bhavya", "Mudit Verma", "Harshit Kumar", "Hirokuni Kitahara", "Noah Zheutlin", "Saki Takano", "Divya Pathak", "Felix George", "Xinbo Wu", "Bekir O. Turkkan", "Gerard Vanloo", "Michael Nidd", "Ting Dai", "Oishik Chatterjee", "Pranjal Gupta", "Suranjana Samanta", "Pooja Aggarwal", "Rong Lee", "Pavankumar Murali", "Jae-wook Ahn", "Debanjana Kar", "Ameet Rahane", "Carlos Fonseca", "Amit Paradkar", "Yu Deng", "Pratibha Moogi", "Prateeti Mohapatra", "Naoki Abe", "Chandrasekhar Narayanaswami", "Tianyin Xu", "Lav R. Varshney", "Ruchi Mahindru", "Anca Sailer", "Laura Shwartz", "Daby Sow", "Nicholas C. M. Fuller", "Ruchir Puri" ], "year": 2025, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.05352", "code_url": "https://github.com/itbench-hub/ITBench", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "ITBench (94 scenarios: SRE, CISO, FinOps)" ], "models_studied": [ "SOTA LLM agents" ], "production_evidence": false, "summary": "IBM framework for benchmarking agents on 94 real-world IT automation scenarios across site reliability engineering, security compliance and FinOps, with push-button workflows and interpretable metrics. SOTA agents solve 13.8% of SRE, 25.2% of CISO and 0% of FinOps scenarios.", "key_idea": "IT-operations automation benchmark.", "relevance_score": 4, "relevance_rationale": "Close to ITSM/IT operations enterprise workloads.", "found_via": "web search: ITBench", "cites_or_cited_by": [] }, { "id": "ji-2025-manus-context-engineering", "title": "Context Engineering for AI Agents: Lessons from Building Manus", "authors": [ "Yichao 'Peak' Ji" ], "year": 2025, "venue_or_source": "Manus blog", "source_type": "blog", "url": "https://manus.im/blog/Context-Engineering-for-AI-Agents-Lessons-from-Building-Manus", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "KV-cache hit rate" ], "cost_reduction_reported": "cached vs uncached input on Claude Sonnet: $0.30 vs $3 per MTok (10x); Manus average input:output token ratio ~100:1", "benchmarks": [], "models_studied": [ "Claude Sonnet" ], "production_evidence": true, "summary": "Practitioner lessons from a production agent: KV-cache hit rate is the key cost/latency metric, so keep prompt prefixes stable and context append-only with deterministic serialization, mask tools via logits rather than removing them, use the file system as external memory, recite goals to steer attention, and keep failed actions in context.", "key_idea": "Design agent context around the KV cache: stable prefix, append-only, mask rather than remove.", "relevance_score": 4, "relevance_rationale": "Production evidence and concrete design rules for cutting agent input-token cost via caching.", "found_via": "Context Engineering for AI Agents: Lessons from Building Manus KV-cache hit rate", "cites_or_cited_by": [ "anthropic-2024-prompt-caching" ] }, { "id": "jiang-2025-cascadia", "title": "Cascadia: An Efficient Cascade Serving System for Large Language Models", "authors": [ "Youhe Jiang", "Fangcheng Fu", "Wanru Zhao", "Stephan Rabanser", "Jintao Zhang", "Nicholas D. Lane", "Binhang Yuan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.04203", "code_url": null, "topic_clusters": [ "C2", "C11" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "workload traces", "routing thresholds", "resource allocation" ], "cost_reduction_reported": "Up to 4x (2.3x avg) tighter latency SLOs and up to 5x (2.4x avg) higher throughput while maintaining target quality vs single-model and cascade-serving baselines", "benchmarks": [ "diverse workload traces" ], "models_studied": [ "DeepSeek", "Llama series" ], "production_evidence": false, "summary": "A serving system that jointly optimises GPU allocation and parallelism for each model in a cascade (MILP) and the routing thresholds (Chebyshev-guided search). Targets latency-quality trade-offs under real workload heterogeneity.", "key_idea": "Bi-level co-optimisation of cascade deployment and routing policy.", "relevance_score": 4, "relevance_rationale": "Systems view of cascades; relevant to self-hosted agent stacks.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "jiang-2025-guided-topology-diffusion", "title": "Dynamic Generation of Multi-LLM Agents Communication Topologies with Graph Diffusion Models", "authors": [ "Eric Hanchen Jiang", "Mengting Li", "Guancheng Wan", "Sophia Yin", "Yuchen Wu", "Xiao Liang", "Xinfeng Li", "Yizhou Sun", "Wei Wang", "Kai-Wei Chang", "Ying Nian Wu" ], "year": 2025, "venue_or_source": "ACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2510.07799", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "proxy reward model (accuracy, utility, cost)" ], "cost_reduction_reported": "Generates sparse, efficient topologies that significantly outperform prior methods (no single number in abstract)", "benchmarks": [ "multiple benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Guided Topology Diffusion builds multi-agent communication graphs iteratively with a discrete graph diffusion model steered by a lightweight proxy that predicts accuracy, utility and cost. The result is sparse task-adaptive topologies that save tokens.", "key_idea": "Diffusion-generated MAS topologies guided by a cost-aware proxy.", "relevance_score": 4, "relevance_rationale": "Cost-aware topology generation.", "found_via": "web search: G-Designer follow-ups", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "jitkrittum-2025-unirouter", "title": "Universal Model Routing for Efficient LLM Inference", "authors": [ "Wittawat Jitkrittum", "Harikrishna Narasimhan", "Ankit Singh Rawat", "Jeevesh Juneja", "Congchao Wang", "Zifeng Wang", "Alec Go", "Chen-Yu Lee", "Pradeep Shenoy", "Rina Panigrahy", "Aditya Krishna Menon", "Sanjiv Kumar", "Chen‐Yu Lee", "Rina Panigrahy‎" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.08773", "code_url": null, "topic_clusters": [ "C1", "C5", "C6", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "model feature vector from predictions on representative prompts", "prompt clusters", "LLM feature vector from representative prompts", "clusters" ], "cost_reduction_reported": null, "benchmarks": [ "public benchmarks (30+ unseen LLMs)", "public benchmarks with 30+ unseen LLMs" ], "models_studied": [], "production_evidence": false, "summary": "UniRoute (Google) handles dynamic routing where new LLMs appear at test time by representing each model as a vector of its errors on a small representative prompt set. Cluster-based and learned-cluster-map instantiations are shown to estimate a theoretically optimal rule, with an excess-risk bound.", "key_idea": "Represent LLMs by prediction profiles on anchor prompts to route among unseen models.", "relevance_score": 4, "relevance_rationale": "Theory plus practicality for evolving model pools.", "found_via": "snowball: cited-by ong-2024-routellm | OpenAlex title search", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "nie-2024-online-cascade-learning", "ong-2024-routellm" ] }, { "id": "joglekar-2025-intercom-small-llms", "title": "David vs Goliath: are small LLMs any good?", "authors": [ "Sagar Joglekar", "Ramil Yarullin" ], "year": 2025, "venue_or_source": "Fin.ai research blog (Intercom)", "source_type": "blog", "url": "https://fin.ai/research/david-vs-goliath-are-small-llms-any-good/", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "production conversations" ], "cost_reduction_reported": "Company-reported: fine-tuned Qwen3 14B matched GPT-4.1 answer rate (63.4%); 5% and 12.5% cost reductions on two components", "benchmarks": [ "internal issue detection and extraction tasks" ], "models_studied": [ "Qwen3 8B/14B", "Gemma 8B", "GPT-4.1", "ModernBERT" ], "production_evidence": true, "summary": "Intercom fine-tunes small open models on consented, PII-redacted production conversations to replace GPT-4.1 in issue classification and extraction, reaching parity on answer rate with modest per-transaction savings.", "key_idea": "Fine-tune small models on production data to replace a frontier model per component.", "relevance_score": 4, "relevance_rationale": "Production component-level downscaling with measured outcomes.", "found_via": "seed list: Intercom case study", "cites_or_cited_by": [ "ivanovici-2025-intercom-cost-of-serving", "yarullin-2025-intercom-escalation" ] }, { "id": "kang-2025-acon", "title": "ACON: Optimizing Context Compression for Long-horizon LLM Agents", "authors": [ "Minki Kang", "Wei-Ning Chen", "Dongge Han", "Huseyin A. Inan", "Lukas Wutschitz", "Yanzhi Chen", "Robert Sim", "Saravan Rajmohan" ], "year": 2025, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2510.00615", "code_url": "https://github.com/microsoft/acon", "topic_clusters": [ "C8", "C10" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "failure analysis of compressed vs full contexts", "natural-language compression guidelines" ], "cost_reduction_reported": "peak token usage reduced by 26-54% while improving task success over compression baselines; distilled compressors let smaller LMs gain up to 46% as long-horizon agents", "benchmarks": [ "AppWorld", "OfficeBench", "Multi-objective QA" ], "models_studied": [], "production_evidence": false, "summary": "ACON compresses both interaction history and environment observations for long-horizon agents. It optimizes the compression guideline in natural language by contrasting failures under compressed vs full context, then distils the optimized compressor into smaller models to cut overhead.", "key_idea": "Optimize the compression prompt from failure cases, then distil the compressor.", "relevance_score": 4, "relevance_rationale": "Directly reduces per-step token cost in long-horizon agents and enables smaller models.", "found_via": "task seed", "cites_or_cited_by": [ "chen-2026-coact", "dixit-2026-focus", "lindenbauer-2025-complexity-trap", "rajasekaran-2025-effective-context-engineering", "satish-2026-beyond-token-savings", "sun-2025-context-folding", "zhou-2025-mem1" ] }, { "id": "kang-2025-agent-distillation-retrieval-code", "title": "Distilling LLM Agent into Small Models with Retrieval and Code Tools", "authors": [ "Minki Kang", "Jongwon Jeong", "Seanie Lee", "Jaewoong Cho", "Sung Ju Hwang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.17612", "code_url": "https://github.com/Nardien/agent-distillation", "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "teacher trajectories", "first-thought prefix", "self-consistent actions" ], "cost_reduction_reported": "0.5B-3B student agents competitive with next-tier larger models fine-tuned with CoT distillation", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Distills full tool-using agent behavior (retrieval and code) from LLM teachers into small models, improving teacher trajectories with a first-thought prefix and student robustness with self-consistent action generation.", "key_idea": "Distill agent behavior, not just CoT, into small models.", "relevance_score": 4, "relevance_rationale": "Direct evidence small distilled agents can replace larger ones.", "found_via": "seed", "cites_or_cited_by": [ "chen-2023-fireact" ] }, { "id": "ke-2025-mas-zero", "title": "MAS-ZERO: Designing Multi-Agent Systems with Zero Supervision", "authors": [ "Zixuan Ke", "Austin Xu", "Yifei Ming", "Xuan-Phi Nguyen", "Ryan Chin", "Caiming Xiong", "Shafiq Joty" ], "year": 2025, "venue_or_source": "NeurIPS 2025 Workshop (SEA)", "source_type": "paper", "url": "https://arxiv.org/abs/2505.14996", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "meta-feedback on solvability/completeness" ], "cost_reduction_reported": "Accuracy gains up to 16.69% (reasoning), 16.66% (coding), 5.45% (agentic) while maintaining cost efficiency", "benchmarks": [ "math and graduate-level QA", "coding", "search-based agentic tasks" ], "models_studied": [], "production_evidence": false, "summary": "MAS-Zero is an inference-time meta-agent that designs, critiques and refines a MAS per problem without a validation set, and can collapse to a simpler system when that suffices. It beats manual and automatic MAS baselines while staying cost-efficient.", "key_idea": "Per-instance MAS design that can reduce to simpler systems when adequate.", "relevance_score": 4, "relevance_rationale": "Adaptive complexity per query, relevant to cost.", "found_via": "web search: MAS-Zero", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer" ] }, { "id": "kiyohara-2025-opl-sentence-personalization", "title": "An Off-Policy Learning Approach for Steering Sentence Generation towards Personalization", "authors": [ "Haruka Kiyohara", "Daniel Yiming Cao", "Yuta Saito", "Thorsten Joachims" ], "year": 2025, "venue_or_source": "ACM (DOI 10.1145/3705328.3748088)", "source_type": "paper", "url": "https://doi.org/10.1145/3705328.3748088", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "logged user feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RecSys 2025 paper by Kiyohara, Cao, Saito and Joachims proposing an off-policy learning approach to steer LLM sentence generation toward personalization using logged feedback (abstract not retrieved; details from title and author record only).", "key_idea": "Off-policy learning for steering personalized LLM generation.", "relevance_score": 4, "relevance_rationale": "OPE/OPL for LLM generation from logged data; companion to the prompt-optimization paper.", "found_via": "OpenAlex title: off-policy learning personalized generation", "cites_or_cited_by": [ "kiyohara-2025-prompt-opt-logged-bandit" ] }, { "id": "lee-2025-mapcoder-lite", "title": "MapCoder-Lite: Distilling Multi-Agent Coding into a Single Small LLM", "authors": [ "Woongkyu Lee", "Junhee Cho", "Jungwook Choi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.17489", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "trajectory distillation", "agent-wise LoRA" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Distills a four-agent coding system into a single 7B model with role-specific LoRA adapters, pass-based trajectory distillation and supervisor-guided correction.", "key_idea": "Per-role LoRA adapters make a small multi-agent system work.", "relevance_score": 4, "relevance_rationale": "Subagent-level specialization with small models.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "lee-2025-online-conformal-abstention", "title": "Online Conformal Abstention for Factuality Control Under Adversarial Bandit Feedback", "authors": [ "Minjae Lee", "Yoonjae Jung", "Sangdon Park" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.14067", "code_url": null, "topic_clusters": [ "C2", "C5", "C7" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "thumbs up/down partial feedback", "confidence score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes ExAUL, an online conformal abstention method that controls false discovery rate when the deployed system only receives partial (thumbs-style) feedback on answered queries, possibly from a non-stationary or adversarial stream. A conversion lemma turns bandit regret into an FDR bound, and a feedback-unlocking trick extracts extra signal from the abstention structure.", "key_idea": "Online risk-controlled answer/abstain gating learned from partial user feedback with FDR guarantees.", "relevance_score": 4, "relevance_rationale": "Fills the online conformal / risk-controlled deferral gap; the abstain decision maps directly onto an escalate decision.", "found_via": "query: abs:\"thumbs\" AND abs:feedback AND abs:LLM", "cites_or_cited_by": [] }, { "id": "li-2025-adaptive-graph-pruning", "title": "Adaptive Graph Pruning for Multi-Agent Communication", "authors": [ "Boyi Li", "Zhonghan Zhao", "Der-Horng Lee", "Gaoang Wang" ], "year": 2025, "venue_or_source": "ECAI 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.02951", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "task" ], "cost_reduction_reported": "Over 90% decrease in token consumption with 2.58%-9.84% performance increase", "benchmarks": [ "six benchmarks: general reasoning, math reasoning, code generation" ], "models_studied": [], "production_evidence": false, "summary": "Adaptive Graph Pruning jointly learns how many agents to use (hard pruning) and which edges to keep (soft pruning) per task through a two-stage training strategy. It improves accuracy and cuts tokens by over 90% with only a few training steps.", "key_idea": "Jointly prune agent count and communication edges per task.", "relevance_score": 4, "relevance_rationale": "MAS pruning for cost.", "found_via": "web search: agent topology pruning", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "li-2025-arg-designer", "title": "Assemble Your Crew: Automatic Multi-agent Communication Topology Design via Autoregressive Graph Generation", "authors": [ "Shiyuan Li", "Yixin Liu", "Qingsong Wen", "Chengqi Zhang", "Shirui Pan" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2507.18224", "code_url": "https://github.com/Shiy-Li/ARG-Designer", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "task query" ], "cost_reduction_reported": "Significantly greater token efficiency than prior topology designers", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "ARG-Designer generates a MAS collaboration graph from scratch autoregressively, deciding the number of agents, their roles and links conditioned on the task query. It beats template-modification methods and uses fewer tokens.", "key_idea": "Autoregressive generation of agent set and topology per query.", "relevance_score": 4, "relevance_rationale": "Query-adaptive MAS composition for efficiency.", "found_via": "web search: agent topology", "cites_or_cited_by": [] }, { "id": "li-2025-chain-of-agents-afm", "title": "Chain-of-Agents: End-to-End Agent Foundation Models via Multi-Agent Distillation and Agentic RL", "authors": [ "Weizhen Li", "Jianbo Lin", "Zhuosong Jiang", "Jingyi Cao", "Xinpeng Liu", "Jiayu Zhang", "Zhenqiang Huang", "Qianben Chen", "Weichen Sun", "Qiexiang Wang", "Hongxuan Lu", "Tianrui Qin", "Chenghao Zhu", "Yi Yao", "Shuying Fan", "Xiaowan Li", "Tiannan Wang", "Pai Liu", "King Zhu", "He Zhu", "Dingfeng Shi", "Piaohong Wang", "Yeyi Guan", "Xiangru Tang", "Minghao Liu", "Yuchen Eleanor Jiang", "Jian Yang", "Jiaheng Liu", "Ge Zhang", "Wangchunshu Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.13167", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "multi-agent distillation", "agentic RL" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Distills multi-agent system trajectories into a single end-to-end agent foundation model that simulates multi-agent collaboration internally, then applies agentic RL.", "key_idea": "Distill a multi-agent system into one model.", "relevance_score": 4, "relevance_rationale": "Replaces an expensive MAS with one model.", "found_via": "hf-papers search: distill multi-agent system into single model", "cites_or_cited_by": [] }, { "id": "li-2025-continuum", "title": "Continuum: Efficient and Robust Multi-Turn LLM Agent Scheduling with KV Cache Time-to-Live", "authors": [ "Hanchen Li", "Runyuan He", "Qiuyang Mang", "Qizheng Zhang", "Huanzhi Mao", "Xiaokun Chen", "Hangrui Zhou", "Huanchen Zhang", "Alvin Cheung", "Joseph Gonzalez", "Ion Stoica" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.02230", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "KV reload cost", "queueing delay", "tool-call duration" ], "cost_reduction_reported": "average job completion time improved by over 8x with higher throughput on SWE-Bench, BFCL and OpenHands agents", "benchmarks": [ "SWE-Bench", "BFCL", "OpenHands" ], "models_studied": [ "Llama-3.1-8B", "Llama-3.1-70B", "Gemma-3-12B", "GLM-4.5-355B" ], "production_evidence": false, "summary": "For multi-turn agents, Continuum pins the KV cache of requests that issue tool calls with a time-to-live chosen from reload cost and eviction-induced queueing, and combines this with program-level FCFS to keep multi-turn continuity.", "key_idea": "KV cache TTL across tool calls plus program-level FCFS.", "relevance_score": 4, "relevance_rationale": "Agent-specific KV retention with large JCT gains on real agent workloads.", "found_via": "task seed", "cites_or_cited_by": [ "abhyankar-2024-infercept", "luo-2025-autellix", "tiwari-2026-cachewise", "xia-2026-mori" ] }, { "id": "li-2025-knn-routing", "title": "Rethinking Predictive Modeling for LLM Routing: When Simple kNN Beats Complex Learned Routers", "authors": [ "Yang Li", "Li Yang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.12601", "code_url": null, "topic_clusters": [ "C1", "C6", "C13" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query embedding kNN", "query embedding neighbors" ], "cost_reduction_reported": null, "benchmarks": [ "standardized routing benchmarks incl. multimodal" ], "models_studied": [], "production_evidence": false, "summary": "Shows that a well-tuned k-nearest-neighbors router matches or beats complex learned routers across instruction-following, QA and reasoning benchmarks, explained by locality of model performance in embedding space. Introduces standardized routing benchmarks including a multimodal one.", "key_idea": "Simple kNN is a strong router baseline.", "relevance_score": 4, "relevance_rationale": "Practical guidance: start with kNN.", "found_via": "snowball: cited-by ong-2024-routellm | snowball: cited-by nguyen-2024-metallm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "li-2025-stageroute", "title": "Near-Optimal Online Deployment and Routing for Streaming LLMs", "authors": [ "Shaoang Li", "Jian Li" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2506.17254", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "reward UCB", "cost LCB", "concurrency cap" ], "cost_reduction_reported": null, "benchmarks": [ "diverse workloads" ], "models_studied": [], "production_evidence": false, "summary": "StageRoute jointly decides which LLMs to deploy at maintenance windows and how to route each query among live models under budget and throughput constraints, with a near-optimal T^(2/3) regret.", "key_idea": "Hierarchical bandit for model deployment plus per-query routing.", "relevance_score": 4, "relevance_rationale": "Captures streaming model inventory churn in production.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "dai-2024-c2mab-v" ] }, { "id": "lin-2025-plan-and-budget", "title": "Plan and Budget: Effective and Efficient Test-Time Scaling on Reasoning Large Language Models", "authors": [ "Junhong Lin", "Xinyue Zeng", "Jie Zhu", "Song Wang", "Julian Shun", "Jun Wu", "Dawei Zhou" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2505.16122", "code_url": "https://github.com/junhongmit/P-and-B", "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "sub-question complexity estimates" ], "cost_reduction_reported": "Up to 70% accuracy gains, 39% token reduction and 193.8% improvement in E3; DS-Qwen-32B matches DS-LLaMA-70B efficiency", "benchmarks": [], "models_studied": [ "DS-Qwen-32B", "DS-LLaMA-70B" ], "production_evidence": false, "summary": "Models reasoning as a sequence of sub-questions with different uncertainty and allocates token budgets to each sub-question by estimated complexity. Model-agnostic and test-time only.", "key_idea": "Decompose, then budget tokens per sub-question.", "relevance_score": 4, "relevance_rationale": "Sub-task level budget allocation transferable to agent steps.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "lin-2025-sleep-time-compute", "title": "Sleep-time Compute: Beyond Inference Scaling at Test-time", "authors": [ "Kevin Lin", "Charlie Snell", "Yu Wang", "Charles Packer", "Sarah Wooders", "Ion Stoica", "Joseph E. Gonzalez" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.13171", "code_url": null, "topic_clusters": [ "C9", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "offline pre-computation over context" ], "cost_reduction_reported": "~5x less test-time compute for the same accuracy on Stateful GSM-Symbolic and Stateful AIME; 2.5x lower average cost per query when amortised over related queries", "benchmarks": [ "Stateful GSM-Symbolic", "Stateful AIME", "Multi-Query GSM-Symbolic", "agentic SWE case study" ], "models_studied": [], "production_evidence": false, "summary": "Lets models think offline about a context before queries arrive, pre-computing useful inferences so test-time reasoning is cheaper; gains depend on how predictable queries are. Includes an agentic SWE case study.", "key_idea": "Move reasoning compute to idle time over shared context.", "relevance_score": 4, "relevance_rationale": "Amortising reasoning across agent queries on shared context.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "lin-2025-supervisoragent", "title": "Stop Wasting Your Tokens: Towards Efficient Runtime Multi-Agent Systems", "authors": [ "Fulin Lin", "Shaowen Chen", "Ruishan Fang", "Hongwei Wang", "Tao Lin" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2510.26585", "code_url": null, "topic_clusters": [ "C7", "C10", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "LLM-free adaptive filter" ], "cost_reduction_reported": "Reduces Smolagents token consumption by 29.68% on GAIA without lowering success rate", "benchmarks": [ "GAIA", "math reasoning", "code generation", "QA" ], "models_studied": [ "Smolagents framework" ], "production_evidence": false, "summary": "SupervisorAgent is a lightweight runtime supervisor triggered by an LLM-free filter that intervenes at critical points to correct errors, curb inefficient behavior and clean observations, without changing the base agent. It cuts token usage by about 30% on GAIA at equal success.", "key_idea": "Runtime supervision triggered cheaply to stop wasteful agent behavior.", "relevance_score": 4, "relevance_rationale": "Architecture-level MAS token reduction.", "found_via": "web search: scaling agents cost", "cites_or_cited_by": [ "wang-2025-efficient-agents", "zhang-2025-maas-agentic-supernet" ] }, { "id": "lindenbauer-2025-complexity-trap", "title": "The Complexity Trap: Simple Observation Masking Is as Efficient as LLM Summarization for Agent Context Management", "authors": [ "Tobias Lindenbauer", "Igor Slinko", "Ludwig Felder", "Egor Bogomolov", "Yaroslav Zharov" ], "year": 2025, "venue_or_source": "DL4C Workshop @ NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2508.21433", "code_url": "https://github.com/JetBrains-Research/the-complexity-trap", "topic_clusters": [ "C10", "C13" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "observation masking window" ], "cost_reduction_reported": "simple observation masking roughly halves cost relative to the raw agent while matching (sometimes slightly exceeding) the solve rate of LLM summarization on SWE-bench Verified", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "Qwen3-Coder 480B", "five model configurations" ], "production_evidence": false, "summary": "Systematic comparison in SWE-agent (with evidence for OpenHands) of LLM-based trajectory summarization versus simply masking older environment observations. Masking is as effective and cheaper; summarization can even lengthen trajectories.", "key_idea": "Just mask old observations; LLM summarization rarely pays for itself.", "relevance_score": 4, "relevance_rationale": "Strong, cheap baseline for cutting agent context cost with evidence across models.", "found_via": "The Complexity Trap observation masking LLM summarization agent context", "cites_or_cited_by": [ "kang-2025-acon", "lodha-2026-less-context-better-agents", "rajasekaran-2025-effective-context-engineering", "xiao-2025-agentdiet-trajectory-reduction" ] }, { "id": "liu-2025-structured-agent-distillation", "title": "Structured Agent Distillation for Large Language Model", "authors": [ "Jun Liu", "Zhenglun Kong", "Peiyan Dong", "Changdi Yang", "Tianqi Li", "Hao Tang", "Geng Yuan", "Wei Niu", "Wenbin Zhang", "Pu Zhao", "Xue Lin", "Dong Huang", "Yanzhi Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.13820", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "span-level reasoning/action alignment" ], "cost_reduction_reported": "Significant compression with minimal performance drop", "benchmarks": [ "ALFWorld", "HotPotQA-ReAct", "WebShop" ], "models_studied": [], "production_evidence": false, "summary": "Segments ReAct trajectories into reasoning and action spans and applies span-specific distillation losses to align student and teacher, outperforming token-level and imitation baselines.", "key_idea": "Span-aware distillation of ReAct agents.", "relevance_score": 4, "relevance_rationale": "Distillation method tailored to agent traces.", "found_via": "seed", "cites_or_cited_by": [ "chen-2023-fireact" ] }, { "id": "lou-2025-adacot", "title": "AdaCoT: Pareto-Optimal Adaptive Chain-of-Thought Triggering via Reinforcement Learning", "authors": [ "Chenwei Lou", "Zewei Sun", "Xinnian Liang", "Meng Qu", "Wei Shen", "Wenqi Wang", "Yuntao Li", "Qingping Yang", "Shuangzhi Wu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.11896", "code_url": null, "topic_clusters": [ "C9", "C14" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "implicit query complexity", "penalty coefficients" ], "cost_reduction_reported": "On a production traffic test set, reduced CoT triggering rate to as low as 3.18% and average response tokens by 69.06% while maintaining performance on complex tasks", "benchmarks": [ "production traffic test set" ], "models_studied": [], "production_evidence": true, "summary": "Frames when to invoke CoT as Pareto optimisation and trains the trigger with PPO, adjusting penalties to move along the frontier; selective loss masking stops decision-boundary collapse.", "key_idea": "RL-tuned CoT triggering evaluated on production traffic.", "relevance_score": 4, "relevance_rationale": "Production-traffic evidence for adaptive reasoning.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "lu-2025-car-certainty-routing", "title": "Prolonged Reasoning Is Not All You Need: Certainty-Based Adaptive Routing for Efficient LLM/MLLM Reasoning", "authors": [ "Jinghui Lu", "Haiyang Yu", "Siliang Xu", "Shiwei Ran", "Guozhi Tang", "Siqi Wang", "Bin Shan", "Teng Fu", "Hao Feng", "Jingqun Tang", "Han Wang", "Can Huang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.15154", "code_url": null, "topic_clusters": [ "C2", "C9" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "short-answer perplexity" ], "cost_reduction_reported": null, "benchmarks": [ "multimodal VQA/KIE benchmarks", "text reasoning datasets" ], "models_studied": [], "production_evidence": false, "summary": "Generates a short answer first and triggers long reasoning only if its perplexity signals low confidence, beating both always-short and always-long reasoning.", "key_idea": "Cascade from short answer to long reasoning on perplexity.", "relevance_score": 4, "relevance_rationale": "Cascade logic applied to reasoning length.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "lu-2025-routerarena", "title": "RouterArena: An Open Platform for Comprehensive Comparison of LLM Routers", "authors": [ "Yifan Lu", "Rixin Liu", "Jiayi Yuan", "Xingqi Cui", "Shenrun Zhang", "Hongyi Liu", "Jiarong Xing" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "benchmark", "url": "https://arxiv.org/abs/2510.00202", "code_url": "https://github.com/RouteWorks/RouterArena", "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "router outputs", "cost", "difficulty levels" ], "cost_reduction_reported": null, "benchmarks": [ "RouterArena", "RouterArena dataset (multi-domain, multi-difficulty)" ], "models_studied": [ "commercial and open-source routers" ], "production_evidence": false, "summary": "RouterArena is an open platform and leaderboard for comparing LLM routers, with a dataset covering broad knowledge domains at graded difficulty, multiple metrics and automated leaderboard updates.", "key_idea": "Open router leaderboard.", "relevance_score": 4, "relevance_rationale": "Router comparison including commercial routers.", "found_via": "snowball: cited-by ong-2024-routellm | web search: RouterEval benchmark routing LLMs", "cites_or_cited_by": [ "hu-2024-routerbench", "ong-2024-routellm" ] }, { "id": "luo-2025-autellix", "title": "Autellix: An Efficient Serving Engine for LLM Agents as General Programs", "authors": [ "Michael Luo", "Xiaoxiang Shi", "Colin Cai", "Tianjun Zhang", "Justin Wong", "Yichuan Wang", "Chi Wang", "Yanping Huang", "Zhifeng Chen", "Joseph E. Gonzalez", "Ion Stoica" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.13965", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "program-level history of completed calls" ], "cost_reduction_reported": "program throughput improved 4-15x at the same latency vs vLLM across diverse LLMs and agentic workloads", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Treats agent programs, not individual calls, as the scheduling unit: it intercepts LLM calls, attaches program-level context, and uses preemptive scheduling based on each program's completed service to reduce head-of-line blocking for single-threaded and distributed programs.", "key_idea": "Program-level (least-attained-service style) scheduling of LLM calls.", "relevance_score": 4, "relevance_rationale": "Core agent-aware scheduling work.", "found_via": "task seed", "cites_or_cited_by": [ "dai-2025-aragog", "kang-2026-thunderagent", "li-2025-continuum", "lin-2024-parrot", "ni-2025-astraea" ] }, { "id": "mehta-2025-clear", "title": "Beyond Accuracy: A Multi-Dimensional Framework for Evaluating Enterprise Agentic AI Systems", "authors": [ "Sushant Mehta" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.14136", "code_url": null, "topic_clusters": [ "C4", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "cost", "latency", "efficacy", "assurance", "reliability" ], "cost_reduction_reported": "Accuracy-only optimization yields agents 4.4-10.8x more expensive than cost-aware alternatives with comparable performance (300 enterprise tasks); up to 50x cost variation for similar precision", "benchmarks": [ "300 enterprise tasks", "analysis of 12 benchmarks" ], "models_studied": [ "six leading agents" ], "production_evidence": false, "summary": "Proposes CLEAR, an enterprise agent evaluation framework across cost, latency, efficacy, assurance and reliability, arguing accuracy-only benchmarks mislead. Expert evaluation suggests CLEAR predicts production success better than accuracy alone.", "key_idea": "Multi-dimensional, cost-controlled evaluation for enterprise agents.", "relevance_score": 4, "relevance_rationale": "Supports cost-aware objectives for agent configuration selection.", "found_via": "search: multi-objective agentic workflow optimization cost accuracy Pareto arXiv | web search: enterprise agent benchmark cost efficiency CLEAR", "cites_or_cited_by": [ "yao-2024-tau-bench" ] }, { "id": "mei-2025-omnirouter", "title": "OmniRouter: Budget and Performance Controllable Multi-LLM Routing", "authors": [ "Kai Mei", "Wujiang Xu", "Minghao Guo", "Shuhang Lin", "Yongfeng Zhang" ], "year": 2025, "venue_or_source": "SIGKDD Explorations 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.20576", "code_url": "https://github.com/dongyuanjushi/OmniRouter", "topic_clusters": [ "C1", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "retrieval-augmented performance predictor", "cost predictor", "Lagrangian dual multipliers" ], "cost_reduction_reported": "Up to 6.30% accuracy improvement while reducing computational costs by at least 10.15% vs router baselines (abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "OmniRouter treats routing as a global constrained optimization that minimizes total cost subject to a performance level, rather than per-query greedy choices. A retrieval-augmented predictor estimates capabilities and costs, and Lagrangian dual decomposition finds allocations under capacity constraints.", "key_idea": "Batch-level constrained optimization for budget-controllable routing.", "relevance_score": 4, "relevance_rationale": "Explicit global budget control.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "microsoft-2025-foundry-model-router", "title": "Model router for Microsoft Foundry (concepts)", "authors": [ "Microsoft" ], "year": 2025, "venue_or_source": "Microsoft Learn", "source_type": "docs", "url": "https://learn.microsoft.com/en-us/azure/ai-foundry/openai/concepts/model-router", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "prompt complexity", "reasoning need", "task type" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-5.x family", "GPT-4.1 family", "o4-mini", "gpt-oss-120b", "Claude Opus/Sonnet/Haiku 4.x", "Grok 4", "DeepSeek-V3.2", "Llama 4 Maverick" ], "production_evidence": false, "summary": "A trained language model deployed as a single Foundry endpoint that routes each prompt across OpenAI, Anthropic, xAI, DeepSeek and Meta models. Balanced mode picks the cheapest model within about 1-2% of the best predicted quality, Cost mode within about 5-6%, and Quality mode ignores cost; supports model subsets, failover, agent tool use and session affinity for cache reuse. No quantitative savings figures on this page.", "key_idea": "Managed multi-vendor router with quality-band routing modes.", "relevance_score": 4, "relevance_rationale": "Seed; enterprise router product with explicit cost-quality band semantics.", "found_via": "seed: Azure AI Foundry model router", "cites_or_cited_by": [] }, { "id": "ni-2025-uncertainty-heads", "title": "ReProbe: Efficient Test-Time Scaling of Multi-Step Reasoning by Probing Internal States of Large Language Models", "authors": [ "Jingwei Ni", "Ekaterina Fadeeva", "Tianyi Wu", "Mubashara Akhtar", "Jiaheng Zhang", "Elliott Ash", "Markus Leippold", "Timothy Baldwin", "See-Kiong Ng", "Artem Shelmanov", "Mrinmaya Sachan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.06209", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "uncertainty heads (<10M params)" ], "cost_reduction_reported": "Match or surpass PRMs up to 810x larger", "benchmarks": [ "math", "planning", "QA" ], "models_studied": [], "production_evidence": false, "summary": "Trains tiny uncertainty heads on frozen LLM states to verify reasoning steps, with labels from a larger LLM or self-supervision; they rival PRMs up to 810x larger.", "key_idea": "Tiny heads replace large PRMs for step verification.", "relevance_score": 4, "relevance_rationale": "Near-free step-level signal.", "found_via": "hf-papers search: uncertainty quantification LLM agents", "cites_or_cited_by": [] }, { "id": "nie-2025-w4s", "title": "Weak-for-Strong: Training Weak Meta-Agent to Harness Strong Executors", "authors": [ "Fan Nie", "Lan Feng", "Haotian Ye", "Weixin Liang", "Pan Lu", "Huaxiu Yao", "Alexandre Alahi", "James Zou" ], "year": 2025, "venue_or_source": "COLM 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2504.04785", "code_url": "https://github.com/fannie1208/W4S", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "execution feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-3.5-Turbo", "GPT-4o" ], "production_evidence": false, "summary": "W4S trains a small (7B) meta-agent with RL for agentic workflow optimization to design workflows that call a stronger executor LLM, outperforming the strongest baseline by 2.9%-24.6% across eleven benchmarks after one GPU hour of training.", "key_idea": "Weak meta-agent trained with RL designs workflows for strong executors.", "relevance_score": 4, "relevance_rationale": "Cheap optimizer model for workflow design, lowering optimization cost.", "found_via": "search: Weak-for-Strong meta-agent workflow optimization W4S arXiv", "cites_or_cited_by": [ "hu-2024-adas", "zhang-2024-aflow" ] }, { "id": "nielsen-2025-conductor", "title": "Learning to Orchestrate Agents in Natural Language with the Conductor", "authors": [ "Stefan Nielsen", "Edoardo Cetin", "Peter Schwendeman", "Qi Sun", "Jinglue Xu", "Yujin Tang" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2512.04388", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "LiveCodeBench", "GPQA" ], "models_studied": [ "7B Conductor", "open and closed worker LLMs" ], "production_evidence": false, "summary": "A 7B Conductor trained with RL learns coordination strategies over pools of powerful worker LLMs, designing communication topologies and writing focused instructions, and beats any individual worker on hard reasoning benchmarks. Trained on randomized pools, it adapts to arbitrary agent sets.", "key_idea": "Small RL-trained conductor orchestrates large worker LLMs.", "relevance_score": 4, "relevance_rationale": "Small-orchestrator/large-worker pattern.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "yue-2025-masrouter" ] }, { "id": "openai-2025-practical-guide-building-agents", "title": "A practical guide to building agents", "authors": [ "OpenAI" ], "year": 2025, "venue_or_source": "OpenAI business guide (PDF)", "source_type": "docs", "url": "https://cdn.openai.com/business-guides-and-resources/a-practical-guide-to-building-agents.pdf", "code_url": null, "topic_clusters": [ "C3", "C12", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "OpenAI's guide, distilled from customer deployments, recommends maximizing a single agent before splitting into manager or decentralized multi-agent patterns. On model selection it advises prototyping with the most capable model for every task to set a baseline, then swapping in smaller models where they remain acceptable, since simple retrieval or intent classification steps may not need the smartest model.", "key_idea": "Baseline with the strongest model everywhere, then downscale per task; prefer a single agent until complexity demands more.", "relevance_score": 4, "relevance_rationale": "Seed; canonical practitioner recipe for per-step model downscaling and architecture choice.", "found_via": "web search: seed", "cites_or_cited_by": [] }, { "id": "openpipe-2025-art", "title": "ART (Agent Reinforcement Trainer) and ART-E email agent", "authors": [ "OpenPipe" ], "year": 2025, "venue_or_source": "GitHub / OpenPipe blog", "source_type": "repo", "url": "https://github.com/OpenPipe/ART", "code_url": "https://github.com/OpenPipe/ART", "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "RULER LLM-judge reward" ], "cost_reduction_reported": "README states a Qwen 2.5 14B email research agent (ART-E) trained with ART outperforms OpenAI o3 at email retrieval; serverless training claims 40% lower cost", "benchmarks": [ "ART-E email search" ], "models_studied": [ "Qwen 2.5 14B", "Qwen 2.5 7B", "OpenAI o3" ], "production_evidence": false, "summary": "Open-source framework for RL fine-tuning of multi-step agents on custom tasks, using RULER (LLM-judge relative scoring) so no hand-labeled rewards are needed. The README links the ART-E case study (blog: openpipe.ai/blog/art-e-mail-agent) where a 14B Qwen email agent beats o3.", "key_idea": "RL-train small task agents with judge-based rewards to replace frontier models.", "relevance_score": 4, "relevance_rationale": "Practitioner tooling and case study for replacing frontier agents with small RL-tuned ones.", "found_via": "GitHub README (OpenPipe/ART) fetched; blog page not renderable", "cites_or_cited_by": [] }, { "id": "pacchiardi-2025-predictaboard", "title": "PredictaBoard: Benchmarking LLM Score Predictability", "authors": [ "Lorenzo Pacchiardi", "Konstantinos Voudouris", "Ben Slater", "Fernando Martínez‐Plumed", "José Hernández‐Orallo", "Lexin Zhou", "Wout Schellaert" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.14445", "code_url": "https://github.com/Kinds-of-Intelligence-CFI/PredictaBoard", "topic_clusters": [ "C6", "C7", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "assessor predictions" ], "cost_reduction_reported": null, "benchmarks": [ "PredictaBoard" ], "models_studied": [ "state-of-the-art LLMs" ], "production_evidence": false, "summary": "PredictaBoard benchmarks assessors that predict whether an LLM will fail on a given prompt, scoring LLM-assessor pairs by rejection rate at different error tolerances.", "key_idea": "Benchmark for instance-level LLM success prediction.", "relevance_score": 4, "relevance_rationale": "Instance-level performance predictors are the core of predictive routers.", "found_via": "OpenAlex title: PredictaBoard", "cites_or_cited_by": [ "cencerrado-2025-no-answer-needed", "xiao-2025-generalized-correctness-models", "zhou-2023-predictable-ai" ] }, { "id": "paglieri-2025-learning-when-to-plan", "title": "Learning When to Plan: Efficiently Allocating Test-Time Compute for LLM Agents", "authors": [ "Davide Paglieri", "Bartłomiej Cupiał", "Jonathan Cook", "Ulyana Piterbarg", "Jens Tuyls", "Edward Grefenstette", "Jakob Nicolaus Foerster", "Jack Parker-Holder", "Tim Rocktäschel" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.03581", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "learned planning decision" ], "cost_reduction_reported": null, "benchmarks": [ "Crafter" ], "models_studied": [], "production_evidence": false, "summary": "Shows always planning before each action (ReAct-style) is costly and can hurt long-horizon performance; trains agents via SFT then RL to decide when to plan.", "key_idea": "Agents learn when to spend compute on planning.", "relevance_score": 4, "relevance_rationale": "Step-level test-time compute allocation in agents.", "found_via": "arXiv title search: test-time compute allocation", "cites_or_cited_by": [] }, { "id": "pan-2025-kvflow", "title": "KVFlow: Efficient Prefix Caching for Accelerating LLM-Based Multi-Agent Workflows", "authors": [ "Zaifeng Pan", "Ajjkumar Patel", "Zhengding Hu", "Yipeng Shen", "Yue Guan", "Wan-Lu Li", "Lianhui Qin", "Yida Wang", "Yufei Ding" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.07400", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "agent step graph", "steps-to-execution" ], "cost_reduction_reported": "up to 1.83x speedup for single workflows with large prompts and up to 2.19x with many concurrent workflows vs SGLang with hierarchical radix cache", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Workflow-aware KV management for multi-agent systems: it models the execution schedule as an Agent Step Graph, evicts prefix-cache nodes of agents far from their next activation, and prefetches KV from CPU for agents scheduled next.", "key_idea": "Evict and prefetch KV by each agent's distance to its next turn.", "relevance_score": 4, "relevance_rationale": "Multi-agent prefix caching is a direct serving cost lever.", "found_via": "task seed", "cites_or_cited_by": [ "bian-2025-tokencake", "ni-2026-topas", "pan-2026-smoothagent", "ye-2025-kvcomm", "yu-2026-pythia", "zhang-2026-cachescout", "zheng-2023-sglang", "zheng-2026-pbkv" ] }, { "id": "pan-2025-route-to-reason", "title": "Route to Reason: Adaptive Routing for LLM and Reasoning Strategy Selection", "authors": [ "Zhihong Pan", "Kai Zhang", "Yuze Zhao", "Yupeng Han" ], "year": 2025, "venue_or_source": "The Web Conference (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2505.19435", "code_url": null, "topic_clusters": [ "C1", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "model and strategy embeddings", "query complexity", "budget", "learned model and strategy embeddings", "predicted performance", "predicted token count" ], "cost_reduction_reported": "Higher accuracy than the best single model while reducing token usage by over 60% (seven open models, four reasoning strategies)", "benchmarks": [], "models_studied": [ "seven open-source models" ], "production_evidence": false, "summary": "Route-To-Reason jointly selects the LLM and the reasoning strategy (e.g. CoT variants) for each query under a user budget, using learned embeddings of model-strategy pairs and a routing table of predicted cost and performance.", "key_idea": "Joint model plus reasoning-strategy routing.", "relevance_score": 4, "relevance_rationale": "Extends routing to configuration choices.", "found_via": "snowball: cited-by ong-2024-routellm | web search: route between reasoning and non-reasoning models", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "poon-2025-online-multi-llm-context-evolution", "title": "Online Multi-LLM Selection via Contextual Bandits under Unstructured Context Evolution", "authors": [ "Manhin Poon", "XiangXiang Dai", "Xutong Liu", "Fang Kong", "John C. S. Lui", "Jinhang Zuo" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2506.17670", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "evolving prompt context", "budget", "position" ], "cost_reduction_reported": null, "benchmarks": [ "diverse benchmarks" ], "models_studied": [ "multiple LLMs" ], "production_evidence": false, "summary": "Studies sequential LLM selection across multi-step query refinement, where the prompt changes in response to earlier outputs. A LinUCB-based algorithm with myopic regret guarantees, plus budget-aware and position-aware variants, needs no offline data.", "key_idea": "LinUCB for multi-turn LLM selection under evolving context.", "relevance_score": 4, "relevance_rationale": "Online routing over multi-turn interactions; relevant to multi-step settings.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "dai-2024-c2mab-v", "li-2010-linucb" ] }, { "id": "prabhakar-2025-apigen-mt", "title": "APIGen-MT: Agentic Pipeline for Multi-Turn Data Generation via Simulated Agent-Human Interplay", "authors": [ "Akshara Prabhakar", "Zuxin Liu", "Weiran Yao", "Jianguo Zhang", "Ming Zhu", "Shiyu Wang", "Zhiwei Liu", "Tulika Awalgaonkar", "Haolin Chen", "Thai Hoang", "Juan Carlos Niebles", "Shelby Heinecke", "Huan Wang", "Silvio Savarese", "Caiming Xiong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.03601", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "simulated agent-human interplay" ], "cost_reduction_reported": "xLAM-2-fc-r models outperform GPT-4o and Claude 3.5 on tau-bench and BFCL; smaller models surpass larger ones in multi-turn", "benchmarks": [ "tau-bench", "BFCL" ], "models_studied": [ "xLAM-2 1B-70B" ], "production_evidence": false, "summary": "Generates verified multi-turn agent data via blueprint validation and simulated human-agent interplay, training xLAM-2 models that beat frontier models on tau-bench.", "key_idea": "Verified multi-turn synthetic data for customer-service style agents.", "relevance_score": 4, "relevance_rationale": "Close to enterprise tool-agent settings.", "found_via": "hf-papers search: synthetic agent data generation", "cites_or_cited_by": [ "liu-2024-apigen" ] }, { "id": "qiu-2025-agentdistill-mcp", "title": "AgentDistill: Training-Free Agent Distillation with Generalizable MCP Boxes", "authors": [ "Jiahao Qiu", "Xinzhe Juan", "Yimin Wang", "Ling Yang", "Xuan Qi", "Tongcheng Zhang", "Jiacheng Guo", "Yifu Lu", "Zixin Yao", "Hongru Wang", "Shilong Liu", "Xun Jiang", "Liu Leqi", "Mengdi Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.14728", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "teacher-generated MCP boxes" ], "cost_reduction_reported": "Small-LM student agents reach performance comparable to OctoTools (GPT-4o) on biomedical and math benchmarks", "benchmarks": [ "biomedical and math benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Training-free agent distillation by reusing Model-Context-Protocol modules generated by teacher agents, letting small-model students approach teacher-level performance.", "key_idea": "Distill via reusable tool modules, no training.", "relevance_score": 4, "relevance_rationale": "Cheap route to downscale agents.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "ren-2025-agserve", "title": "Transcending Cost-Quality Tradeoff in Agent Serving via Session-Awareness", "authors": [ "Yanyu Ren", "Li Chen", "Dan Li", "Xizheng Wang", "Zhiyuan Wu", "Yukai Miao", "Yu Bai" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://doi.org/10.52202/085713-0097", "code_url": null, "topic_clusters": [ "C2", "C3", "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "real-time quality assessment", "session state" ], "cost_reduction_reported": "Comparable response quality to GPT-4o at 16.5% of the cost; 1.8x quality improvement relative to the tradeoff curve", "benchmarks": [], "models_studied": [ "GPT-4o" ], "production_evidence": false, "summary": "AgServe is an agent-serving system with a session-aware server that improves KV-cache reuse, a client that cascades models within a session from real-time quality assessment, and a dynamic GPU scheduler. Agents can pick and upgrade models during a session, reaching GPT-4o-level quality at a fraction of the cost on real testbeds.", "key_idea": "Session-aware per-round model cascading plus KV-cache management for agent serving.", "relevance_score": 4, "relevance_rationale": "Per-round model selection inside agent sessions with large reported savings; systems-level evidence.", "found_via": "snowball: cited-by nie-2024-online-cascade-learning", "cites_or_cited_by": [ "nie-2024-online-cascade-learning" ] }, { "id": "ro-2025-sherlock", "title": "Sherlock: Reliable and Efficient Agentic Workflow Execution", "authors": [ "Yeonju Ro", "Haoran Qiu", "Íñigo Goiri", "Rodrigo Fonseca", "Ricardo Bianchini", "Aditya Akella", "Zhangyang Wang", "Mattan Erez", "Esha Choukse" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.00330", "code_url": null, "topic_clusters": [ "C7", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "fault-aware verification", "speculative execution with rollback" ], "cost_reduction_reported": "18.3% average accuracy gain vs non-verifying baseline; workflow execution time reduced up to 48.7% vs non-speculative execution; verification cost 26.0% lower than Monte Carlo search-based verification", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Selectively verifies error-prone steps of agentic workflows, continues speculatively while verification runs, and rolls back to the last verified output on failure.", "key_idea": "Speculate past verification and roll back on failure.", "relevance_score": 4, "relevance_rationale": "Balances verification cost and latency in agent workflows.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [] }, { "id": "schroeder-2025-vcache", "title": "vCache: Verified Semantic Prompt Caching", "authors": [ "Luis Gaspar Schroeder", "Aditya Desai", "Alejandro Cuadron", "Kyle Chu", "Shu Liu", "Mark Zhao", "Stephan Krusche", "Alfons Kemper", "Matei Zaharia", "Joseph E. Gonzalez" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2502.03771", "code_url": "https://github.com/vcache-project/vCache", "topic_clusters": [ "C7", "C10" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "per-entry similarity thresholds", "online correctness feedback", "user error bound delta" ], "cost_reduction_reported": "up to 12.5x higher cache hit rate and 26x lower error rate than static-threshold and fine-tuned-embedding baselines while meeting the user-specified error bound", "benchmarks": [ "four released semantic-cache benchmarks" ], "models_studied": [ "various embedding models", "LLM responders" ], "production_evidence": false, "summary": "Replaces the single global similarity threshold of semantic caches with per-cached-prompt thresholds learned online, and makes a probabilistic hit/explore decision that provably keeps the error rate under a user-chosen bound. This turns semantic caching into a controllable cost-quality tradeoff.", "key_idea": "Learn per-entry thresholds online to give a guaranteed maximum error rate for semantic cache hits.", "relevance_score": 4, "relevance_rationale": "Principled correctness control for caching; online-learning design mirrors routing under an error budget.", "found_via": "vCache verified semantic prompt caching error rate guarantee", "cites_or_cited_by": [ "bang-2023-gptcache", "baral-2026-semantic-cache-operational-gap", "kulkarni-2026-cache-eviction-study", "singh-2026-krites" ] }, { "id": "sharma-2025-slm-agentic-survey", "title": "Small Language Models for Agentic Systems: A Survey of Architectures, Capabilities, and Deployment Trade offs", "authors": [ "Raghav Sharma", "Manan Mehta" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.03847", "code_url": null, "topic_clusters": [ "C3", "C8", "C12", "C15" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "uncertainty-aware routing", "verifier cascades", "uncertainty", "verifier checks" ], "cost_reduction_reported": "Claims SLMs with guided decoding can match or surpass LLMs on tool use, function calling and RAG at 10x-100x lower token cost (abstract, synthesized from literature)", "benchmarks": [ "BFCL v3/v4", "StableToolBench" ], "models_studied": [ "Phi-4-Mini", "Qwen-2.5-7B", "Gemma-2-9B", "Llama-3.2", "Ministral", "Llama-3.2-1B/3B", "DeepSeek-R1-Distill" ], "production_evidence": false, "summary": "Survey arguing small language models suffice for many schema-constrained agentic workloads and formalizing SLM-default, LLM-fallback designs with uncertainty-aware routing and verifier cascades. Proposes metrics such as cost per successful task.", "key_idea": "SLM-default, LLM-fallback agent design.", "relevance_score": 4, "relevance_rationale": "Directly about downscaling agent calls.", "found_via": "search: survey LLM routing | OpenAlex search: LLM agent cascade small large | hf-papers search: small language model function calling", "cites_or_cited_by": [] }, { "id": "she-2025-hawkeye", "title": "Hawkeye:Efficient Reasoning with Model Collaboration", "authors": [ "Jianshu She", "Zhuohao Li", "Zhemin Huang", "Qi Li", "Peiran Xu", "Haonan Li", "Qirong Ho" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.00424", "code_url": null, "topic_clusters": [ "C2", "C3", "C9" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "concise CoT instructions from large model" ], "cost_reduction_reported": "Comparable quality using only 35% of full CoTs; up to 3.4x faster end-to-end reasoning and up to 60% lower inference cost on complex math", "benchmarks": [ "complex math tasks" ], "models_studied": [], "production_evidence": false, "summary": "A large model writes a short, information-dense CoT (trained by RL to remove redundancy) and a small model expands it into the final response.", "key_idea": "Large model thinks briefly, small model writes.", "relevance_score": 4, "relevance_rationale": "Large/small split of reasoning vs generation.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "shen-2025-sater", "title": "SATER: A Self-Aware and Token-Efficient Approach to Routing and Cascading", "authors": [ "Yuanzhe Shen", "Yide Liu", "Zisu Huang", "Ruicheng Yin", "Xiaoqing Zheng", "Xuanjing Huang" ], "year": 2025, "venue_or_source": "EMNLP 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2510.05164", "code_url": null, "topic_clusters": [ "C1", "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "self-aware rejection", "shortest-response preference optimization" ], "cost_reduction_reported": "Comparable performance while reducing computational costs by over 50% and cascade latency by over 80% (abstract)", "benchmarks": [ "six datasets" ], "models_studied": [ "three SLMs" ], "production_evidence": false, "summary": "SATER fine-tunes small models with shortest-response preference optimization and a confidence-aware rejection mechanism so they work in both pre-generation routing and cascade modes. It cuts redundant output and latency.", "key_idea": "Train the SLM to know when to refuse and escalate.", "relevance_score": 4, "relevance_rationale": "Dual routing/cascade efficiency.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "shirkavand-2025-cscr", "title": "Cost-Aware Contrastive Routing for LLMs", "authors": [ "Reza Shirkavand", "Shangqian Gao", "Peiran Yu", "Heng Huang" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2508.12491", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "logit footprints", "perplexity fingerprints", "contrastive cost bands", "FAISS kNN" ], "cost_reduction_reported": "Improves accuracy-cost tradeoff by up to 25% (abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CSCR maps prompts and models into a shared embedding space, using logit footprints for open models and perplexity fingerprints for API models, and trains a contrastive encoder to favor the cheapest accurate model within cost bands. Routing reduces to one kNN lookup with microsecond latency and no retraining when the pool changes.", "key_idea": "Cost-spectrum contrastive embeddings with kNN routing.", "relevance_score": 4, "relevance_rationale": "Fast, pool-agnostic router.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "singhvi-2025-databricks-gepa-90x", "title": "Building State-of-the-Art Enterprise Agents 90x Cheaper with Automated Prompt Optimization", "authors": [ "Arnav Singhvi", "Ivan Zhou", "Erich Elsen", "Krista Opsahl-Ong", "Michael Bendersky", "Matei Zaharia", "Xing Chen", "Omar Khattab", "Xiangrui Meng", "Simon Favreau-Lessard" ], "year": 2025, "venue_or_source": "Databricks blog", "source_type": "blog", "url": "https://www.databricks.com/blog/building-state-art-enterprise-agents-90x-cheaper-automated-prompt-optimization", "code_url": null, "topic_clusters": [ "C4", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task metric on IE Bench" ], "cost_reduction_reported": "GEPA-optimized gpt-oss-120b surpasses Claude Sonnet 4 and Claude Opus 4.1 baselines by ~3% while being ~22x and ~90x cheaper to serve on IE Bench; GEPA ~20% cheaper to serve than SFT alternatives", "benchmarks": [ "IE Bench (Databricks internal information extraction)" ], "models_studied": [ "gpt-oss-120b", "Claude Sonnet 4", "Claude Opus 4.1", "gpt-4.1" ], "production_evidence": false, "summary": "Databricks reports that automated prompt optimization (GEPA and others in Agent Bricks) lets an open model beat frontier proprietary models on an enterprise information-extraction benchmark at a fraction of the serving cost, and compares favorably with supervised fine-tuning on lifetime cost.", "key_idea": "Optimize prompts for a cheap open model instead of paying for a frontier model.", "relevance_score": 4, "relevance_rationale": "Practitioner evidence that config/prompt optimization plus model downscaling yields large cost savings on enterprise-style tasks; not real traffic.", "found_via": "search: Databricks GEPA prompt optimization enterprise agents cheaper models blog", "cites_or_cited_by": [ "agrawal-2025-gepa" ] }, { "id": "soiffer-2025-semantic-agreement-cascades", "title": "Semantic Agreement Enables Efficient Open-Ended LLM Cascades", "authors": [ "Duncan Soiffer", "Steven Kolawole", "Virginia Smith" ], "year": 2025, "venue_or_source": "EMNLP 2025 Industry Track", "source_type": "paper", "url": "https://arxiv.org/abs/2509.21837", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "semantic agreement among ensemble outputs" ], "cost_reduction_reported": "Semantic cascades match or surpass target-model quality at 40% of the cost and reduce latency by up to 60%", "benchmarks": [ "open-ended generation benchmarks" ], "models_studied": [ "500M to 70B-parameter models" ], "production_evidence": false, "summary": "For open-ended generation, uses meaning-level agreement among several cheap models' outputs as the deferral signal instead of token confidence. Works with black-box APIs and is robust to model updates.", "key_idea": "Semantic consensus of cheap ensemble outputs as a black-box deferral signal.", "relevance_score": 4, "relevance_rationale": "Training-free, API-compatible deferral for open-ended outputs.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "song-2025-irt-router", "title": "IRT-Router: Effective and Interpretable Multi-LLM Routing via Item Response Theory", "authors": [ "Wei Song", "Zhenya Huang", "Cheng Cheng", "Weibo Gao", "Bihan Xu", "GuanHao Zhao", "Fei Wang", "Runze Wu" ], "year": 2025, "venue_or_source": "ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.01048", "code_url": "https://github.com/Mercidaiha/IRT-Router", "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "item response theory ability/difficulty", "query embedding", "semantic warm-up", "IRT ability and difficulty" ], "cost_reduction_reported": null, "benchmarks": [ "12 datasets" ], "models_studied": [ "20 LLMs" ], "production_evidence": false, "summary": "IRT-Router borrows item response theory to jointly model LLM abilities and query difficulty/attributes, giving both accurate performance prediction and interpretable parameters. An online warm-up based on semantic similarity helps cold-start queries; experiments span 20 LLMs and 12 datasets.", "key_idea": "Psychometric IRT model of model ability vs query difficulty for routing.", "relevance_score": 4, "relevance_rationale": "Seed; interpretable difficulty estimation useful for enterprise auditing.", "found_via": "seed | OpenAlex title search", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm", "polo-2024-tinybenchmarks" ] }, { "id": "sun-2025-context-folding", "title": "Scaling Long-Horizon LLM Agent via Context-Folding", "authors": [ "Weiwei Sun", "Miao Lu", "Zhan Ling", "Kang Liu", "Xuesong Yao", "Yiming Yang", "Jiecao Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.11967", "code_url": null, "topic_clusters": [ "C9", "C10" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "branch/return actions", "process rewards (FoldGRPO)" ], "cost_reduction_reported": "matches or outperforms ReAct baselines on Deep Research and SWE tasks with a 10x smaller active context, and beats summarization-based context management", "benchmarks": [ "Deep Research tasks", "SWE tasks" ], "models_studied": [], "production_evidence": false, "summary": "Agents learn to branch into a sub-trajectory for a subtask and fold it on return, keeping only a short summary of the outcome in the main context. An RL method (FoldGRPO) with process rewards trains effective decomposition and context management.", "key_idea": "Learned branch-and-fold keeps agent working context small.", "relevance_score": 4, "relevance_rationale": "Learned context management for long-horizon agents; subagent-like decomposition inside one model.", "found_via": "Scaling Long-Horizon LLM Agent via Context-Folding", "cites_or_cited_by": [ "kang-2025-acon", "liu-2025-context-as-a-tool", "wu-2025-resum", "ye-2025-agentfold" ] }, { "id": "tran-2025-arch-router", "title": "Arch-Router: Aligning LLM Routing with Human Preferences", "authors": [ "Co Tran", "Salman Paracha", "Adil Hafeez", "Shuguang Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.16655", "code_url": "https://huggingface.co/katanemo/Arch-Router-1.5B", "topic_clusters": [ "C1", "C6", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "user-defined domain/action policies", "1.5B router LM", "domain-action preference policies", "domain-action preference labels" ], "cost_reduction_reported": null, "benchmarks": [ "conversational datasets" ], "models_studied": [ "Arch-Router-1.5B", "Arch-Router 1.5B", "proprietary LLMs" ], "production_evidence": false, "summary": "Arch-Router is a 1.5B model that maps queries to user-defined domain or action preferences, which are then bound to specific models. New models can be added by editing the policy mapping without retraining; the authors report state-of-the-art preference matching against proprietary models.", "key_idea": "Route by matching queries to human-written policy descriptions rather than benchmark scores.", "relevance_score": 4, "relevance_rationale": "Seed; practical preference-aligned router from the Arch gateway team.", "found_via": "seed | OpenAlex title search | seed list: LLM gateway (katanemo archgw)", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "katanemo-plano-archgw", "ong-2024-routellm" ] }, { "id": "valkanas-2025-c3po", "title": "C3PO: Optimized Large Language Model Cascades with Probabilistic Cost Constraints for Reasoning", "authors": [ "Antonios Valkanas", "Soumyasundar Pal", "Pavel Rumiantsev", "Yingxue Zhang", "Mark Coates" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.07396", "code_url": null, "topic_clusters": [ "C2", "C9" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "unlabelled model outputs", "regret vs most powerful model", "conformal bounds" ], "cost_reduction_reported": "State-of-the-art accuracy and cost-efficiency vs cascading baselines (no single number in abstract)", "benchmarks": [ "GSM8K", "MATH-500", "BigBench-Hard", "AIME" ], "models_studied": [], "production_evidence": false, "summary": "Self-supervised cascade optimisation that minimises regret against the most powerful model using only unlabelled outputs. Conformal prediction bounds the probability that inference cost exceeds a user budget, with generalisation guarantees.", "key_idea": "Label-free cascade tuning with conformal cost-overrun control.", "relevance_score": 4, "relevance_rationale": "Label-free, budget-controlled cascades for reasoning.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "varangot-reille-2025-doing-more-with-less", "title": "Doing More with Less: A Survey on Routing Strategies for Resource Optimisation in Large Language Model-Based Systems", "authors": [ "Clovis Varangot-Reille", "Christophe Bouvard", "Antoine Gourru", "Mathieu Ciancone", "Marion Schaeffer", "François Jacquenet" ], "year": 2025, "venue_or_source": "Journal of Artificial Intelligence Research", "source_type": "paper", "url": "https://arxiv.org/abs/2502.00409", "code_url": null, "topic_clusters": [ "C1", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of routing strategies in LLM-based systems covering when, why and how to route queries to smaller or specialized components to save cost and energy.", "key_idea": "Survey of routing for resource optimization.", "relevance_score": 4, "relevance_rationale": "Core routing survey.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "wang-2025-icl-router", "title": "ICL-Router: In-Context Learned Model Representations for LLM Routing", "authors": [ "Chenxu Wang", "Hao Li", "Yiqun Zhang", "Linyao Chen", "Jianhao Chen", "Ping Jian", "Peng Ye", "Qiaosheng Zhang", "Shuyue Hu" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2510.09719", "code_url": "https://github.com/lalalamdbf/ICL-Router", "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "in-context vectors of model performance", "query embedding projector" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ICL-Router represents each candidate model through in-context vectors built from its performance on a profiling query set, and trains an LLM-based router to predict whether a model will answer a new query correctly. New models can be integrated without retraining.", "key_idea": "In-context model capability vectors for retrain-free router extension.", "relevance_score": 4, "relevance_rationale": "Addresses model-pool churn.", "found_via": "search: ICL-Router", "cites_or_cited_by": [] }, { "id": "wang-2025-scoreflow", "title": "ScoreFlow: Mastering LLM Agent Workflows via Score-based Preference Optimization", "authors": [ "Yinjie Wang", "Ling Yang", "Guohao Li", "Mengdi Wang", "Bryon Aragam" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.04306", "code_url": "https://github.com/Gen-Verse/ScoreFlow", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "quantitative evaluation scores (Score-DPO)" ], "cost_reduction_reported": "8.2% improvement over baselines on six benchmarks; enables smaller models to outperform larger ones with lower inference costs", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ScoreFlow trains a workflow generator with Score-DPO, a DPO variant that uses quantitative feedback, performing gradient-based optimization in a continuous space rather than discrete search. It improves over prior workflow optimizers across QA, coding and math.", "key_idea": "Preference-optimize a workflow generator using score-aware DPO.", "relevance_score": 4, "relevance_rationale": "Automated workflow optimization with a smaller-model-beats-larger cost angle.", "found_via": "seed list: ScoreFlow", "cites_or_cited_by": [ "zhang-2024-aflow" ] }, { "id": "wang-2025-strmac", "title": "Optimal-Agent-Selection: State-Aware Routing Framework for Efficient Multi-Agent Collaboration", "authors": [ "Jingbo Wang", "Sendong Zhao", "Haochun Wang", "Yuzheng Fan", "Ting Liu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.02200", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "interaction history encoding", "agent knowledge encoding" ], "cost_reduction_reported": "Up to 23.8% improvement over baselines; data collection overhead reduced up to 90.1% vs exhaustive search", "benchmarks": [ "collaborative reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "STRMAC routes each step of multi-agent collaboration to the single most suitable agent using separate encodings of interaction history and agent knowledge, with self-evolving data generation to cheaply collect training paths.", "key_idea": "Per-step single-agent selection in MAS.", "relevance_score": 4, "relevance_rationale": "Step-level routing among agents.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration" ] }, { "id": "wang-2025-when-to-reason", "title": "When to Reason: Semantic Router for vLLM", "authors": [ "Chen Wang", "Xunzhuo Liu", "Yuhan Liu", "Yue Zhu", "Xiangxi Mo", "Junchen Jiang", "Huamin Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.08731", "code_url": null, "topic_clusters": [ "C1", "C9", "C11" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "reasoning-need classifier", "query reasoning-requirement classifier" ], "cost_reduction_reported": "+10.2 pp accuracy on MMLU-Pro while reducing latency 47.1% and tokens 48.5% vs direct inference with vLLM (abstract)", "benchmarks": [ "MMLU-Pro" ], "models_studied": [], "production_evidence": false, "summary": "A semantic router classifies whether a query needs reasoning mode and enables chain-of-thought only when beneficial, integrated with vLLM.", "key_idea": "Route reasoning on/off per query.", "relevance_score": 4, "relevance_rationale": "Cheap reasoning-mode control.", "found_via": "snowball: cited-by ong-2024-routellm | arXiv title search: reasoning router", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "woisetschlager-2025-mess-plus", "title": "MESS+: Dynamically Learned Inference-Time LLM Routing in Model Zoos with Service Level Guarantees", "authors": [ "Herbert Woisetschläger", "Ryan Zhang", "Shiqiang Wang", "Hans-Arno Jacobsen" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2505.19947", "code_url": "https://github.com/laminair/mess-plus", "topic_clusters": [ "C1", "C5", "C15" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "learned request satisfaction probability", "virtual queues", "SLA constraint" ], "cost_reduction_reported": "Average 2x cost savings vs existing LLM routing techniques (abstract)", "benchmarks": [ "LLM benchmarks" ], "models_studied": [ "open-weight model zoo" ], "production_evidence": false, "summary": "MESS+ is a stochastic-optimization router for model zoos that learns satisfaction probabilities online and solves per-request optimization with virtual queues to meet SLA quality guarantees while minimizing cost. Includes theory on cost optimality and constraint satisfaction.", "key_idea": "Lyapunov-style virtual queues for SLA-guaranteed cost-optimal routing.", "relevance_score": 4, "relevance_rationale": "SLA framing matches enterprise needs.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "wu-2025-inter-cascade", "title": "From Deferral to Learning: Online In-Context Knowledge Distillation for LLM Cascades", "authors": [ "Yu Wu", "Shuo Wu", "Ye Tao", "Yansong Li", "Anand D. Sarwate", "Wu Yu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.22984", "code_url": null, "topic_clusters": [ "C2", "C5", "C8", "C10" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "weak-model confidence", "similarity retrieval of stored strategies", "deferral", "strategy retrieval", "retrieved strategies" ], "cost_reduction_reported": "Up to 33.06% weak-model and 6.35% system accuracy gain; up to 48.05% fewer strong-model calls and up to 49.63% fee savings vs standard cascades", "benchmarks": [ "multiple reasoning benchmarks" ], "models_studied": [ "open-source and API LLMs" ], "production_evidence": false, "summary": "When the strong model resolves a deferred query, it also writes a reusable strategy that is stored and retrieved for similar future queries to help the weak model. The cascade thus learns online without fine-tuning.", "key_idea": "Strong model acts as an in-context teacher; stored strategies reduce future deferrals.", "relevance_score": 4, "relevance_rationale": "Online learning from live traffic inside a cascade.", "found_via": "arXiv search: \"LLM cascade\" | snowball: cited-by belcak-2025-slms-future-agentic-ai | snowball: cited-by nie-2024-online-cascade", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai", "nie-2024-online-cascade-learning", "ramirez-2023-cache-and-distil" ] }, { "id": "wu-2025-training-free-online-routing", "title": "Efficient Training-Free Online Routing for High-Volume Multi-LLM Serving", "authors": [ "Fangzhou Wu", "Sandeep Silwal" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2509.02718", "code_url": null, "topic_clusters": [ "C1", "C5", "C15" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "approximate nearest neighbor features", "one-time LP on initial queries", "token budget" ], "cost_reduction_reported": "Average 3.55x improvement in overall performance, 1.85x in cost efficiency and ~4.25x throughput vs 8 baselines (abstract)", "benchmarks": [ "3 benchmark datasets" ], "models_studied": [], "production_evidence": false, "summary": "Presents the first training-free algorithm for online routing under high query volume and token budgets, estimating query features via ANN search and solving a one-time optimization on initial queries. Provides a 1-o(1) competitive ratio guarantee.", "key_idea": "Online budgeted routing with competitive-ratio guarantees.", "relevance_score": 4, "relevance_rationale": "Theory for online budget-constrained routing.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "xi-2025-agentprm-xi", "title": "AgentPRM: Process Reward Models for LLM Agents via Step-Wise Promise and Progress", "authors": [ "Zhiheng Xi", "Chenyang Liao", "Guanyu Li", "Yajie Yang", "Wenxiang Chen", "Zhihao Zhang", "Binghai Wang", "Senjie Jin", "Yuhao Zhou", "Jian Guan", "Wei Wu", "Tao Ji", "Tao Gui", "Qi Zhang", "Xuanjing Huang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.08325", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward", "promise", "progress", "TD/GAE labels" ], "cost_reduction_reported": "Reported as over 8x more compute-efficient than baselines when scaling test-time compute", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Redefines PRMs for agent tasks: instead of step correctness, it scores each step's promise (proximity to goal) and progress, capturing interdependence between sequential decisions. Labels are obtained scalably with TD-based estimation combined with GAE; experiments cover agentic tasks such as web shopping and browser navigation.", "key_idea": "Agent PRM scoring promise and progress per step, trained with TD/GAE labels.", "relevance_score": 4, "relevance_rationale": "Cheap step-level scorer for agent trajectories, supports test-time compute allocation.", "found_via": "WebSearch: process reward model for LLM agents AgentPRM", "cites_or_cited_by": [ "choudhury-2025-agentprm-choudhury" ] }, { "id": "xiao-2025-agentdiet-trajectory-reduction", "title": "Reducing Cost of LLM Agents with Trajectory Reduction", "authors": [ "Yuan-An Xiao", "Pengfei Gao", "Chao Peng", "Yingfei Xiong" ], "year": 2025, "venue_or_source": "FSE 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2509.23586", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "redundancy/expiry of trajectory content", "useless/redundant/expired trajectory content detection" ], "cost_reduction_reported": "input tokens reduced 39.9%-59.7% and total computational cost reduced 21.1%-35.9% while maintaining agent performance", "benchmarks": [ "SWE-bench Verified", "Multi-SWE-bench (two benchmarks)", "two coding-agent benchmarks" ], "models_studied": [ "two LLMs" ], "production_evidence": false, "summary": "AgentDiet removes useless, redundant and expired information from coding-agent trajectories at inference time, using a cheaper reflection model over a sliding window. It cuts input tokens and total cost substantially with no loss in resolve rate.", "key_idea": "Prune waste from growing agent trajectories during execution.", "relevance_score": 4, "relevance_rationale": "Context-level cost lever for long-horizon agents; uses a cheaper model for the reduction step.", "found_via": "web search: model switching mid-trajectory | SWE-agent context management trajectory reduction cost coding agents arXiv", "cites_or_cited_by": [ "hu-2026-cost-inefficient-coding-agents", "lindenbauer-2025-complexity-trap", "wang-2026-swe-pruner", "weinberger-2026-token-reduction-not-cost" ] }, { "id": "xiao-2025-generalized-correctness-models", "title": "Generalized Correctness Models: Learning Calibrated and Model-Agnostic Correctness Predictors from Historical Patterns", "authors": [ "Hanqi Xiao", "Vaidehi Patil", "Hyunji Lee", "Elias Stengel-Eskin", "Mohit Bansal" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.24988", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "historical correctness patterns", "answer phrasing" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU", "TriviaQA" ], "models_studied": [ "Qwen3-8B", "5 model families" ], "production_evidence": false, "summary": "Trains correctness predictors on historical correctness data across many models and finds they generalize across models, suggesting confidence estimation is learnable from history rather than self-introspection.", "key_idea": "Model-agnostic correctness predictors from logged outcomes.", "relevance_score": 4, "relevance_rationale": "Predict success of any model from historical logs, key for routers.", "found_via": "snowball: cited-by pacchiardi-2025-predictaboard", "cites_or_cited_by": [ "pacchiardi-2025-predictaboard" ] }, { "id": "xu-2025-boad", "title": "BOAD: Discovering Hierarchical Software Engineering Agents via Bandit Optimization", "authors": [ "Iris Xu", "Guangtao Zeng", "Zexue He", "Charles Jin", "Aldo Pareja", "Dan Gutfreund", "Chuang Gan", "Zhang-Wei Hong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.23631", "code_url": "https://github.com/iamxjy/BOAD-SWE-Agent", "topic_clusters": [ "C3", "C4", "C5" ], "routing_granularity": "subagent", "learning_regime": "bandit-online", "signals_used": [ "sub-agent helpfulness reward" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-bench Verified", "SWE-bench Live" ], "models_studied": [ "36B system" ], "production_evidence": false, "summary": "Discovers a hierarchical SWE agent by treating candidate sub-agents as bandit arms whose reward is their helpfulness when collaborating, allowing efficient exploration under limited evaluation budgets. The discovered 36B system ranked second on SWE-bench Live at evaluation time.", "key_idea": "Multi-armed bandit search over sub-agent designs for hierarchical agents.", "relevance_score": 4, "relevance_rationale": "Bandit-driven agent configuration search with credit to sub-agents.", "found_via": "query: \"bandit\" \"cascade\" LLM", "cites_or_cited_by": [] }, { "id": "xu-2025-trinity-coordinator", "title": "TRINITY: An Evolved LLM Coordinator", "authors": [ "Jinglue Xu", "Qi Sun", "Peter S Schwendeman", "Stefan Nielsen", "Edoardo Cetin", "Yu-Jin Tang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.04695", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "coordinator hidden states" ], "cost_reduction_reported": null, "benchmarks": [ "LiveCodeBench", "math", "reasoning", "domain knowledge" ], "models_studied": [], "production_evidence": false, "summary": "Trinity uses a ~0.6B coordinator plus a ~10K-parameter head, optimized with separable CMA-ES, that at each turn assigns a Thinker, Worker or Verifier role to a chosen LLM. It beats individual models and prior methods, reaching 86.2% on LiveCodeBench.", "key_idea": "Tiny evolved coordinator assigns roles and models per turn.", "relevance_score": 4, "relevance_rationale": "Per-turn role+model assignment by a small coordinator; quality-focused rather than cost-focused.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "xu-2025-verimap", "title": "Verification-Aware Planning for Multi-Agent Systems", "authors": [ "Tianyang Xu", "Dan Zhang", "Kushan Mitra", "Estevam Hruschka" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.17109", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "planner-defined verification functions" ], "cost_reduction_reported": null, "benchmarks": [ "diverse datasets" ], "models_studied": [], "production_evidence": false, "summary": "VeriMAP's planner decomposes tasks, models dependencies and writes verification functions for each subtask, so executors can be checked and retried. Because executors get well-scoped subtasks, cheaper models can execute them, partly offsetting planner cost.", "key_idea": "Planner writes per-subtask verifiers so executors can be smaller.", "relevance_score": 4, "relevance_rationale": "Planner-executor split enabling cheaper executors with verification.", "found_via": "web search: planner executor", "cites_or_cited_by": [] }, { "id": "yang-2025-shapleyflow", "title": "Understanding and Optimizing Agentic Workflows via Shapley value", "authors": [ "Yingxuan Yang", "Bo Huang", "Siyuan Qi", "Chao Feng", "Haoyi Hu", "Yuxuan Zhu", "Jinbo Hu", "Haoran Zhao", "Ziyi He", "Xiao Liu", "Muning Wen", "Zongyu Wang", "Lin Qiu", "Xuezhi Cao", "Xunliang Cai", "et al." ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.00510", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "Shapley value attribution" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ShapleyFlow applies cooperative game theory to attribute contributions of workflow components and identify task-specific component configurations, including different LLMs per component, that beat single-LLM workflows across 7 scenarios.", "key_idea": "Shapley attribution to find optimal per-component configurations.", "relevance_score": 4, "relevance_rationale": "Per-component model configuration with attribution.", "found_via": "search: FlowBank query-adaptive agentic workflows precompute reuse", "cites_or_cited_by": [] }, { "id": "yazdani-2025-masprm", "title": "MASPRM: Multi-Agent System Process Reward Model", "authors": [ "Milad Yazdani", "Mahdi Mostajabdaveh", "Zirui Zhou", "Ying Xiong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.24803", "code_url": "https://github.com/milad1378yz/MASPRM", "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward over agent messages", "multi-agent process reward", "value per agent turn" ], "cost_reduction_reported": "+30.7 EM on GSM8K and +22.9 on MATH over a single MAS pass when guiding search", "benchmarks": [ "GSM8K", "MATH", "MMLU", "LogiQA" ], "models_studied": [ "1.5B and 7B scorers" ], "production_evidence": false, "summary": "MASPRM is a process reward model for multi-agent transcripts, trained from MCTS rollouts with only terminal rewards, that guides step-level beam search and MCTS over agent messages. It beats size-matched outcome reward models under comparable search budgets.", "key_idea": "Process rewards for which agent message advanced progress.", "relevance_score": 4, "relevance_rationale": "Step-level quality signal usable for MAS routing or pruning.", "found_via": "arXiv API search | hf-papers search: process reward model agents tool use", "cites_or_cited_by": [] }, { "id": "ye-2025-kvcomm", "title": "KVCOMM: Online Cross-context KV-cache Communication for Efficient LLM-based Multi-agent Systems", "authors": [ "Hancheng Ye", "Zhengqi Gao", "Mingyuan Ma", "Qinsi Wang", "Yuzhe Fu", "Ming-Yu Chung", "Yueqian Lin", "Zhijian Liu", "Jianyi Zhang", "Danyang Zhuo", "Yiran Chen" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2510.12872", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "anchor pool of KV deviations" ], "cost_reduction_reported": "over 70% KV reuse rate across multi-agent workloads without quality degradation; up to 7.8x speedup (TTFT ~430 ms to ~55 ms) in a five-agent setting", "benchmarks": [ "RAG", "math reasoning", "collaborative coding" ], "models_studied": [], "production_evidence": false, "summary": "Lets agents in a multi-agent system reuse each other's KV caches for shared content despite different prefixes by estimating the prefix-induced deviation from a pool of cached anchor examples maintained online.", "key_idea": "Cross-agent KV reuse with online-estimated offset correction.", "relevance_score": 4, "relevance_rationale": "Removes redundant prefill across agents that see the same content.", "found_via": "multi-agent LLM KV cache sharing reuse across agents", "cites_or_cited_by": [ "jeon-2026-lragent", "pan-2025-kvflow" ] }, { "id": "ye-2025-mas-gpt", "title": "MAS-GPT: Training LLMs to Build LLM-based Multi-Agent Systems", "authors": [ "Rui Ye", "Shuo Tang", "Rui Ge", "Yaxin Du", "Zhenfei Yin", "Siheng Chen", "Jing Shao" ], "year": 2025, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2503.03686", "code_url": "https://github.com/rui-ye/MAS-GPT", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query" ], "cost_reduction_reported": null, "benchmarks": [ "9 benchmarks" ], "models_studied": [ "5 LLMs" ], "production_evidence": false, "summary": "MAS-GPT reframes MAS construction as generation: a fine-tuned mid-size LLM emits an executable query-specific MAS in one inference, trained on consistency-filtered query-MAS pairs. It outperforms 10+ MAS baselines while avoiding the repeated LLM calls of search-based MAS design.", "key_idea": "Generate an executable query-specific MAS in a single forward pass.", "relevance_score": 4, "relevance_rationale": "Query-adaptive MAS construction at low design-time cost.", "found_via": "web search: MAS-GPT", "cites_or_cited_by": [] }, { "id": "ye-2025-speculative-actions", "title": "Speculative Actions: A Lossless Framework for Faster Agentic Systems", "authors": [ "Naimeng Ye", "Arnav Ahuja", "Georgios Liargkovas", "Yunan Lu", "Kostis Kaffes", "Tianyi Peng" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.04371", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "next-action agreement", "fast-model next-action prediction" ], "cost_reduction_reported": "up to 55% next-action prediction accuracy translating into up to 20% latency reduction", "benchmarks": [ "gaming", "e-commerce", "web search", "OS environments", "web search environments" ], "models_studied": [], "production_evidence": false, "summary": "Speculative Actions uses fast models to guess an agent's next actions and executes them in parallel, committing only when the main model agrees, analogous to speculative decoding. It gives lossless latency savings and analyzes the cost-latency trade-off of speculative breadth.", "key_idea": "Speculate agent actions with a small model; verify with the large one.", "relevance_score": 4, "relevance_rationale": "Big-little pairing within agent loops, latency-focused.", "found_via": "web search: speculative actions | speculative actions agent tool calls parallel execution latency LLM agents", "cites_or_cited_by": [ "kim-2023-llmcompiler", "sui-2026-paste" ] }, { "id": "zeighami-2025-bargain", "title": "Cut Costs, Not Accuracy: LLM-Powered Data Processing with Guarantees", "authors": [ "Sepanta Zeighami", "Shreya Shankar", "Aditya Parameswaran" ], "year": 2025, "venue_or_source": "SIGMOD 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2509.02896", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "log-probability confidence", "adaptive sampling", "statistical estimation" ], "cost_reduction_reported": "Reduces cost on average by up to 86% more than state of the art across 8 real-world datasets, with stronger accuracy guarantees", "benchmarks": [ "8 real-world datasets" ], "models_studied": [ "GPT-4o", "GPT-4o-mini", "Claude Sonnet", "Claude Haiku" ], "production_evidence": false, "summary": "For LLM-powered data processing, uses adaptive sampling of expensive-model labels and tight statistical estimation to decide which records the cheap model can handle, with guarantees on accuracy, precision or recall.", "key_idea": "Cascades with statistical accuracy guarantees via adaptive sampling.", "relevance_score": 4, "relevance_rationale": "Guaranteed cascade thresholds with minimal labelling.", "found_via": "arXiv search: model cascade language model", "cites_or_cited_by": [] }, { "id": "zellinger-2025-early-abstention-cascades", "title": "Cost-Saving LLM Cascades with Early Abstention", "authors": [ "Michael J. Zellinger", "Rex Liu", "Matt Thomson" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.09054", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "confidence", "abstention" ], "cost_reduction_reported": "Reduces overall test loss by 2.2% on average across six benchmarks; 13.0% cost reduction and 5.0% error reduction for a 4.1% higher abstention rate", "benchmarks": [ "GSM8K", "MedMCQA", "MMLU", "TriviaQA", "TruthfulQA", "XSum" ], "models_studied": [], "production_evidence": false, "summary": "Adds abstention to LLM cascades and asks whether early, cheap models should be allowed to abstain. Because error patterns correlate across models, early abstention saves the cost of calling large models that would abstain anyway.", "key_idea": "Let the cheap model abstain early, exploiting correlated errors.", "relevance_score": 4, "relevance_rationale": "Abstention plus cascade design for risk-sensitive deployments.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "zellinger-2025-economic-evaluation", "title": "Economic Evaluation of LLMs", "authors": [ "Michael J. Zellinger", "Matt Thomson" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.03834", "code_url": null, "topic_clusters": [ "C2", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "cost of error", "latency cost", "abstention cost" ], "cost_reduction_reported": "Reasoning models give better accuracy-cost trade-offs once a mistake costs more than a small dollar threshold (MATH)", "benchmarks": [ "MATH" ], "models_studied": [ "reasoning and non-reasoning LLMs" ], "production_evidence": false, "summary": "Proposes scoring LLMs by a single dollar-denominated number based on the use case's cost of mistakes, latency and abstention, allowing comparison beyond Pareto plots.", "key_idea": "Price errors, latency and abstention to compare models.", "relevance_score": 4, "relevance_rationale": "Useful objective for routing and cascade decisions.", "found_via": "web search: Cost-of-Pass economic framework", "cites_or_cited_by": [] }, { "id": "zellinger-2025-rational-tuning-cascades", "title": "Rational Tuning of LLM Cascades via Probabilistic Modeling", "authors": [ "Michael J. Zellinger", "Matt Thomson" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.09345", "code_url": null, "topic_clusters": [ "C2", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "confidence thresholds", "joint error model (Markov copula)" ], "cost_reduction_reported": "Improves area under error-cost curve by 4.3% on average vs Bayesian-optimisation thresholds for cascades with k>=3; 10.2% with n<=30 training examples", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Models the joint distribution of confidence and correctness across a chain of LLMs with a parametric Markov-copula, then tunes the confidence thresholds by continuous optimisation. The probabilistic model is especially sample-efficient.", "key_idea": "Probabilistic joint model of cascade errors for threshold tuning.", "relevance_score": 4, "relevance_rationale": "Sample-efficient threshold tuning from small offline data.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "zhang-2025-agentracer", "title": "AgenTracer: Who Is Inducing Failure in the LLM Agentic Systems?", "authors": [ "Guibin Zhang", "Junhao Wang", "Junjie Chen", "Wangchunshu Zhou", "Kun Wang", "Shuicheng Yan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.03312", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "failure tracer model" ], "cost_reduction_reported": "AgenTracer-8B outperforms Gemini-2.5-Pro and Claude-4-Sonnet by up to 18.18% on Who&When", "benchmarks": [ "Who&When" ], "models_studied": [ "AgenTracer-8B", "Gemini-2.5-Pro", "Claude-4-Sonnet" ], "production_evidence": false, "summary": "Builds annotated failure trajectories via counterfactual replay and fault injection, then trains an 8B failure tracer with multi-granular RL that outperforms large proprietary LLMs at failure attribution and improves agent systems via feedback.", "key_idea": "Small RL-trained tracer beats frontier LLMs at failure attribution.", "relevance_score": 4, "relevance_rationale": "Cheap attribution model usable in production monitoring.", "found_via": "snowball: cited-by which-agent-when", "cites_or_cited_by": [ "zhang-2025-which-agent-when" ] }, { "id": "zhang-2025-confidence-driven-router", "title": "Leveraging Uncertainty Estimation for Efficient LLM Routing", "authors": [ "Tuo Zhang", "Asal Mehradfar", "Dimitrios Dimitriadis", "Salman Avestimehr" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.11021", "code_url": null, "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "uncertainty estimate", "LLM-as-a-judge ratings" ], "cost_reduction_reported": null, "benchmarks": [ "MT-Bench", "GSM8K", "MMLU" ], "models_studied": [], "production_evidence": false, "summary": "Proposes a confidence-driven router for edge-cloud deployments that uses uncertainty estimation to decide routing, and evaluates response quality with LLM-as-a-judge as a proxy for human preference. Reports better quality than prior routers at similar cost.", "key_idea": "Uncertainty-based edge-cloud routing evaluated by judges.", "relevance_score": 4, "relevance_rationale": "Signal choice for routing.", "found_via": "snowball: cited-by ong-2024-routellm | hf-papers search: confidence estimation routing escalation", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "zhang-2025-meta-router", "title": "Meta-Router: Bridging Gold-standard and Preference-based Evaluations in Large Language Model Routing", "authors": [ "Yichi Zhang", "Fangzheng Xie", "Shu Yang", "Chong Wu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.25535", "code_url": null, "topic_clusters": [ "C1", "C6", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "gold-standard labels", "preference/judge labels", "causal CATE correction" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Meta-Router casts training with combined gold-standard and preference-based labels as causal inference, correcting bias in cheap judge/crowd labels via a conditional average treatment effect estimate.", "key_idea": "Causal debiasing of judge-labeled routing supervision.", "relevance_score": 4, "relevance_rationale": "Directly relevant to training routers on judge-labeled logs.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "zhang-2025-model-sat", "title": "Capability Instruction Tuning: A New Paradigm for Dynamic LLM Routing", "authors": [ "Yi-Kai Zhang", "De-Chuan Zhan", "Han-Jia Ye" ], "year": 2025, "venue_or_source": "AAAI 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.17282", "code_url": "https://github.com/Now-Join-Us/CIT-LLM-Routing", "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "model capability representation (aptitude test)", "instruction" ], "cost_reduction_reported": null, "benchmarks": [ "50 tasks" ], "models_studied": [], "production_evidence": false, "summary": "Model-SAT builds capability instructions from a model's aptitude-test results and trains a lightweight LLM to predict whether each candidate can handle an instruction. New models are onboarded by running a quick 50-task aptitude test, without candidate inference at routing time.", "key_idea": "Capability instruction tuning: route by aptitude-test profiles.", "relevance_score": 4, "relevance_rationale": "Fast onboarding of new models.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "zhang-2025-reasoning-models-know-right", "title": "Reasoning Models Know When They're Right: Probing Hidden States for Self-Verification", "authors": [ "Anqi Zhang", "Yulin Chen", "Jane Pan", "Chen Zhao", "Aurojit Panda", "Jinyang Li", "He He" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.05419", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state probe", "intermediate answer correctness" ], "cost_reduction_reported": "Probe-based early exit reduces inference tokens by 24% without compromising performance", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Probes reasoning-model hidden states to verify intermediate answers with high accuracy and calibration, even predicting correctness before the answer is fully formed. Using the probe for early exit cuts reasoning tokens by 24% at equal accuracy.", "key_idea": "Hidden states encode correctness; use a probe to stop reasoning early.", "relevance_score": 4, "relevance_rationale": "Cheap in-trajectory quality signal enabling budget control.", "found_via": "WebSearch: probes hidden states LLM correctness prediction routing escalation", "cites_or_cited_by": [] }, { "id": "zhang-2025-safesieve", "title": "SafeSieve: From Heuristics to Experience in Progressive Pruning for LLM-based Multi-Agent Communication", "authors": [ "Ruijia Zhang", "Xinyan Zhao", "Ruixiang Wang", "Sigen Chen", "Guibin Zhang", "An Zhang", "Kun Wang", "Qingsong Wen" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2508.11733", "code_url": "https://github.com/csgen/SafeSieve", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "LLM semantic evaluation", "accumulated performance feedback" ], "cost_reduction_reported": "94.01% average accuracy with 12.4%-27.8% token reduction; 13.3% deployment cost reduction in heterogeneous settings", "benchmarks": [ "SVAMP", "HumanEval" ], "models_studied": [], "production_evidence": false, "summary": "SafeSieve progressively prunes inter-agent communication, starting from LLM-based semantic scoring and moving to experience-driven refinement, while clustering to keep coherent agent groups. It reduces tokens and cost, including in heterogeneous-model deployments, and is robust to prompt injection.", "key_idea": "Progressive heuristic-to-experience pruning of MAS communication.", "relevance_score": 4, "relevance_rationale": "MAS pruning with heterogeneous-deployment cost results.", "found_via": "arXiv search: multi-agent token pruning", "cites_or_cited_by": [] }, { "id": "zhang-2025-stop-overvaluing-mad", "title": "Stop Overvaluing Multi-Agent Debate -- We Must Rethink Evaluation and Embrace Model Heterogeneity", "authors": [ "Hangfan Zhang", "Zhiyao Cui", "Jianhao Chen", "Xinrun Wang", "Qiaosheng Zhang", "Zhen Wang", "Dinghao Wu", "Shuyue Hu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.08788", "code_url": null, "topic_clusters": [ "C3", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "9 benchmarks" ], "models_studied": [ "4 foundation models" ], "production_evidence": false, "summary": "A systematic evaluation of five multi-agent debate methods on nine benchmarks finds they often fail to beat CoT and self-consistency despite using more compute. Mixing heterogeneous models consistently improves MAD.", "key_idea": "MAD rarely pays off; model heterogeneity is the reliable improvement.", "relevance_score": 4, "relevance_rationale": "Key evidence on cost-effectiveness of debate and heterogeneous agents (sub-theme g).", "found_via": "arXiv query ti:debate AND ti:heterogeneous", "cites_or_cited_by": [] }, { "id": "zhang-2025-synapseroute", "title": "SynapseRoute: An Auto-Route Switching Framework on Dual-State Large Language Model", "authors": [ "Wencheng Zhang", "Shiqin Qiao", "Lingjie Luo", "Yinfeng Li", "Chuanyang Zheng", "Qian Xu", "Meng Li", "Yong Gui", "Yijun He", "Jianing Qiu", "Jindong Hong", "Jiankai Sun" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.02822", "code_url": null, "topic_clusters": [ "C1", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query complexity features" ], "cost_reduction_reported": "Accuracy 0.8390 vs 0.8272 for thinking-only, with 36.8% lower inference time and 39.66% fewer tokens on medical datasets", "benchmarks": [ "medical QA datasets" ], "models_studied": [ "dual-state (thinking/non-thinking) LLM" ], "production_evidence": false, "summary": "Finds about 58% of medical questions are answered correctly without thinking mode, and trains a router that sends each query to thinking or non-thinking mode. Proposes an accuracy-inference-token index for evaluation.", "key_idea": "Route queries between thinking and non-thinking modes of one model.", "relevance_score": 4, "relevance_rationale": "Direct thinking-mode routing with cost numbers.", "found_via": "web search: routing between thinking and non-thinking modes", "cites_or_cited_by": [] }, { "id": "zhang-2025-which-agent-when", "title": "Which Agent Causes Task Failures and When? On Automated Failure Attribution of LLM Multi-Agent Systems", "authors": [ "Shaokun Zhang", "Ming Yin", "Jieyu Zhang", "Jiale Liu", "Zhiguang Han", "Jingyang Zhang", "Beibin Li", "Chi Wang", "Huazheng Wang", "Yiran Chen", "Qingyun Wu" ], "year": 2025, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2505.00212", "code_url": "https://github.com/ag2ai/Agents_Failure_Attribution", "topic_clusters": [ "C3", "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "LLM judge", "failure attribution" ], "cost_reduction_reported": null, "benchmarks": [ "Who&When" ], "models_studied": [ "o1", "DeepSeek R1" ], "production_evidence": false, "summary": "Defines automated failure attribution for LLM multi-agent systems: identify the responsible agent and the decisive error step in a failed trace. Releases the Who&When dataset (failure logs from 127 MAS); the best method reaches 53.5% agent accuracy but only 14.2% step accuracy.", "key_idea": "Benchmark and baselines for who-and-when failure attribution in MAS.", "relevance_score": 4, "relevance_rationale": "Attribution tells which subagent/step to upgrade or downscale; shows current judges are weak.", "found_via": "seed", "cites_or_cited_by": [ "ge-2025-famas-spectrum-attribution", "li-2026-afanet-gnn-attribution", "qiao-2026-verifymas", "zhang-2025-agentracer", "zhang-2026-agentforesight", "zhang-2026-duotrace-cascade-attribution" ] }, { "id": "zhou-2025-mass", "title": "Multi-Agent Design: Optimizing Agents with Better Prompts and Topologies", "authors": [ "Han Zhou", "Xingchen Wan", "Ruoxi Sun", "Hamid Palangi", "Shariq Iqbal", "Ivan Vulić", "Anna Korhonen", "Sercan Ö. Arık" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2502.02533", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "validation score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Gemini 1.5 Pro", "Gemini 1.5 Flash", "Claude 3.5 Sonnet", "Mistral Nemo" ], "production_evidence": false, "summary": "Analyzes the multi-agent design space and finds prompts and topologies both matter; proposes MASS, which interleaves block-level prompt optimization, topology optimization, and workflow-level prompt optimization.", "key_idea": "Staged local-to-global optimization of prompts and topology for MAS.", "relevance_score": 4, "relevance_rationale": "Strong MAS configuration optimizer; model choice not optimized.", "found_via": "search: MASS Multi-Agent Design optimizing agents prompts topologies arXiv", "cites_or_cited_by": [ "hu-2024-adas", "zhang-2024-aflow" ] }, { "id": "zhu-2025-test-time-compute-agents", "title": "Scaling Test-time Compute for LLM Agents", "authors": [ "King Zhu", "Hanhao Li", "Siwei Wu", "Tianshun Xing", "Dehua Ma", "Xiangru Tang", "Minghao Liu", "Jian Yang", "Jiaheng Liu", "Yuchen Eleanor Jiang", "Changwang Zhang", "Chenghua Lin", "Jun Wang", "Ge Zhang", "Wangchunshu Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.12928", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "verifiers", "list-wise merging", "reflection timing" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Systematic study of test-time scaling for language agents: parallel sampling, sequential revision, verification and merging, and rollout diversity. Finds knowing when to reflect matters and list-wise merging works best.", "key_idea": "Test-time scaling recipes for agents.", "relevance_score": 4, "relevance_rationale": "Agent-level compute allocation guidance.", "found_via": "web search: Scaling LLM Test-Time Compute (related results)", "cites_or_cited_by": [] }, { "id": "abhyankar-2024-infercept", "title": "InferCept: Efficient Intercept Support for Augmented Large Language Model Inference", "authors": [ "Reyna Abhyankar", "Zijian He", "Vikranth Srivatsa", "Hao Zhang", "Yiying Zhang" ], "year": 2024, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2402.01869", "code_url": "https://github.com/WukLab/InferCept", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "interception duration", "context length" ], "cost_reduction_reported": "serving throughput improved 1.6x-2x and 2x more completed requests per second vs state-of-the-art systems; recomputation after interceptions was 37-40% of forwarding time", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "When augmented LLMs pause for tools, standard engines discard state and recompute on resume. InferCept chooses per interception whether to discard, keep in GPU memory, or swap the KV cache to host memory to minimize wasted GPU work.", "key_idea": "Manage KV cache across tool-call interceptions.", "relevance_score": 4, "relevance_rationale": "Directly addresses tool-call-induced waste in agent serving.", "found_via": "InferCept intercept augmented LLM inference tool calls", "cites_or_cited_by": [ "li-2025-continuum", "shahout-2024-lamps", "wang-2025-augserve", "xu-2024-conveyor" ] }, { "id": "almahairi-2024-anyscale-router", "title": "Building an LLM Router for High-Quality and Cost-Effective Responses", "authors": [ "Amjad Almahairi" ], "year": 2024, "venue_or_source": "Anyscale blog", "source_type": "blog", "url": "https://www.anyscale.com/blog/building-an-llm-router-for-high-quality-and-cost-effective-responses", "code_url": "https://github.com/lm-sys/RouteLLM", "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "LLM-judge score labels", "query text" ], "cost_reduction_reported": "(vendor-reported) Up to 70% cost reduction on MT-Bench, 30% on MMLU, 40% on GSM8K vs random routing / commercial routers", "benchmarks": [ "MT-Bench", "MMLU", "GSM8K" ], "models_studied": [ "GPT-4", "Mixtral-8x7B", "Llama3-8B (router)" ], "production_evidence": false, "summary": "Tutorial companion to RouteLLM: fine-tunes a Llama3-8B causal classifier on ~109K Nectar queries labeled 1-5 by a GPT-4 judge on Mixtral responses, routing to Mixtral when predicted score is high and to GPT-4 otherwise.", "key_idea": "Judge-labeled data to fine-tune an LLM classifier router.", "relevance_score": 4, "relevance_rationale": "Practical recipe for building a strong/weak router from judge labels.", "found_via": "seed: Anyscale router blog", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "anthropic-2024-building-effective-agents", "title": "Building effective agents", "authors": [ "Erik S.", "Barry Zhang", "Anthropic" ], "year": 2024, "venue_or_source": "Anthropic engineering blog", "source_type": "blog", "url": "https://www.anthropic.com/engineering/building-effective-agents", "code_url": null, "topic_clusters": [ "C1", "C12", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "input classification" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude Haiku", "Claude Sonnet", "Claude models" ], "production_evidence": true, "summary": "Anthropic distills lessons from dozens of customer agent deployments: prefer simple composable workflow patterns (prompt chaining, routing, parallelization, orchestrator-workers, evaluator-optimizer) and only move to autonomous agents when needed, since agentic systems trade latency and cost for performance. The routing pattern explicitly sends easy questions to smaller models such as Haiku and hard ones to Sonnet.", "key_idea": "Start simple; add agentic complexity only when it pays for its cost; route easy inputs to smaller models.", "relevance_score": 4, "relevance_rationale": "Seed; widely used practitioner guidance on architecture-level cost decisions, based on customer deployments.", "found_via": "web search: seed | seed list: Anthropic engineering blog", "cites_or_cited_by": [] }, { "id": "bottaro-2024-linkedin-genai-product", "title": "Musings on Building a Generative AI Product", "authors": [ "Juan Pablo Bottaro", "Karthik R." ], "year": 2024, "venue_or_source": "LinkedIn engineering blog", "source_type": "blog", "url": "https://www.linkedin.com/blog/engineering/generative-ai/musings-on-building-a-generative-ai-product", "code_url": null, "topic_clusters": [ "C3", "C14" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "smaller and larger LLMs (unnamed)" ], "production_evidence": true, "summary": "Describes LinkedIn's premium AI assistant: a router picks a specialized agent, then retrieval and generation, with smaller models for routing/retrieval and larger ones for generation. Discusses GPU capacity limits, streaming and async pipelines, and daily evaluation of up to 500 conversations; quality reached 80% quickly then took months to reach 95%.", "key_idea": "Small models for routing/retrieval, large for generation.", "relevance_score": 4, "relevance_rationale": "Production role-level model assignment.", "found_via": "seed list: LinkedIn case study", "cites_or_cited_by": [] }, { "id": "chan-2024-agentmonitor", "title": "AgentMonitor: A Plug-and-Play Framework for Predictive and Secure Multi-Agent Systems", "authors": [ "Chi-Min Chan", "Jianxuan Yu", "Weize Chen", "Chunyang Jiang", "Xinyu Liu", "Weijie Shi", "Zhiyuan Liu", "Xue, Wei", "Yike Guo" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.14972", "code_url": null, "topic_clusters": [ "C7", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "agent input/output features", "XGBoost predictor" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AgentMonitor captures per-agent inputs/outputs in a multi-agent system and trains a regression model (XGBoost) to predict task performance (Spearman 0.89 in-domain), also filtering harmful content.", "key_idea": "Predict MAS performance from agent-level features before/while running.", "relevance_score": 4, "relevance_rationale": "Performance prediction for MAS configurations.", "found_via": "keyword: failure detection in agents", "cites_or_cited_by": [] }, { "id": "chen-2024-model-cascading-code", "title": "Model Cascading for Code: A Cascaded Black-Box Multi-Model Framework for Cost-Efficient Code Completion with Self-Testing", "authors": [ "Boyuan Chen", "Mingzhi Zhu", "Brendan Dolan-Gavitt", "Muhammad Shafique", "Siddharth Garg" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.15842", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "self-generated tests", "pass rate threshold" ], "cost_reduction_reported": "Reduces costs by an average of 26%, up to 70% in the best case, while maintaining or improving accuracy vs single-model self-testing schemes", "benchmarks": [ "code generation benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Cascades code-generation models and uses self-generated tests both to improve answers and to decide when to escalate to a larger model. A heuristic sets the number of solutions and tests per model under a budget.", "key_idea": "Self-testing as the cascade escalation signal for code.", "relevance_score": 4, "relevance_rationale": "Execution-based verifier signals for coding agents.", "found_via": "arXiv search: model cascade language model", "cites_or_cited_by": [] }, { "id": "chen-2024-routerdc", "title": "RouterDC: Query-Based Router by Dual Contrastive Learning for Assembling Large Language Models", "authors": [ "Shuhao Chen", "Weisen Jiang", "Baijiong Lin", "James T. Kwok", "Yu Zhang" ], "year": 2024, "venue_or_source": "NeurIPS 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2409.19886", "code_url": "https://github.com/shuhao02/RouterDC", "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding", "learned LLM embeddings", "contrastive losses" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RouterDC trains a query encoder and per-LLM embeddings with two contrastive losses (sample-LLM and sample-sample) so that queries where several LLMs do well are handled gracefully. It outperforms individual top models and prior routers in and out of distribution (+2.76% and +1.90%).", "key_idea": "Dual contrastive learning of query and LLM embeddings for routing.", "relevance_score": 4, "relevance_rationale": "Widely used learned-router baseline.", "found_via": "seed", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "chiang-2024-chatbot-arena", "title": "Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference", "authors": [ "Wei-Lin Chiang", "Lianmin Zheng", "Ying Sheng", "Anastasios Nikolas Angelopoulos", "Tianle Li", "Dacheng Li", "Hao Zhang", "Banghua Zhu", "Michael Jordan", "Joseph E. Gonzalez", "Ion Stoica" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2403.04132", "code_url": null, "topic_clusters": [ "C5", "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "pairwise human votes" ], "cost_reduction_reported": null, "benchmarks": [ "Chatbot Arena" ], "models_studied": [ "many LLMs" ], "production_evidence": true, "summary": "Describes the Chatbot Arena platform, which collects crowdsourced pairwise votes between LLMs on real user prompts (over 240K votes at time of writing) and ranks models with Bradley-Terry style statistics.", "key_idea": "Live crowdsourced pairwise preference collection for LLM ranking.", "relevance_score": 4, "relevance_rationale": "Canonical source of online preference data used to train routers (e.g. RouteLLM).", "found_via": "OpenAlex title: Chatbot Arena", "cites_or_cited_by": [ "boubdir-2023-elo-uncovered", "chatzi-2024-prediction-powered-ranking", "li-2024-arena-hard", "ong-2024-routellm", "spangher-2025-chatbot-arena-estimate", "zhao-2024-eagle", "zheng-2023-lmsys-chat-1m", "zheng-2023-mt-bench" ] }, { "id": "chuang-2024-self-ref", "title": "Learning to Route LLMs with Confidence Tokens", "authors": [ "Yu-Neng Chuang", "Prathusha Kameswara Sarma", "Parikshit Gopalan", "John Boccio", "Sara Bolouki", "Xia Hu", "Helen Zhou" ], "year": 2024, "venue_or_source": "ICML (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2410.13284", "code_url": null, "topic_clusters": [ "C1", "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "learned confidence tokens", "confidence tokens" ], "cost_reduction_reported": "Significant improvements in downstream routing and rejection tasks vs verbalised confidence and token probabilities (no single number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Self-REF fine-tunes LLMs to emit confidence tokens indicating whether an answer is correct, from which a confidence score is extracted. These scores outperform verbalized confidence and token probabilities for routing to another model or rejecting.", "key_idea": "Train the model to emit calibrated confidence tokens for routing/deferral.", "relevance_score": 4, "relevance_rationale": "Quality signal for escalation decisions.", "found_via": "snowball: cited-by ong-2024-routellm | GitHub list: Awesome-Efficient-Reasoning-LLMs | keyword: confidence routing", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm", "sui-2025-stop-overthinking" ] }, { "id": "damani-2024-learning-how-hard-to-think", "title": "Learning How Hard to Think: Input-Adaptive Allocation of LM Computation", "authors": [ "Mehul Damani", "Idan Shenfeld", "Andi Peng", "Andreea Bobu", "Jacob Andreas" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.04707", "code_url": null, "topic_clusters": [ "C1", "C2", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted reward distribution per input and budget" ], "cost_reduction_reported": "Reduces computation by up to 50% at no cost to response quality, or improves quality by up to 10% at fixed budget (programming, math, dialog)", "benchmarks": [ "programming", "mathematics", "dialog tasks" ], "models_studied": [], "production_evidence": false, "summary": "Predicts, per input, how much reward extra decoding compute would add and allocates best-of-k samples or routes between cheap and expensive decoding procedures accordingly.", "key_idea": "Learn per-input marginal value of compute, then allocate.", "relevance_score": 4, "relevance_rationale": "Adaptive compute allocation and routing.", "found_via": "snowball: related work of snell-2024-test-time-compute-optimal", "cites_or_cited_by": [ "snell-2024-test-time-compute-optimal" ] }, { "id": "ding-2024-hybrid-llm", "title": "Hybrid LLM: Cost-Efficient and Quality-Aware Query Routing", "authors": [ "Dujian Ding", "Ankur Mallick", "Chi Wang", "Robert Sim", "Subhabrata Mukherjee", "Victor Ruhle", "Laks V. S. Lakshmanan", "Ahmed Hassan Awadallah", "Robert B. Sim", "Victor Rühle" ], "year": 2024, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2404.14618", "code_url": null, "topic_clusters": [ "C1", "C2", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "difficulty estimate", "predicted quality gap", "desired quality threshold", "predicted query difficulty", "quality gap estimate", "difficulty" ], "cost_reduction_reported": "Up to 40% fewer calls to the large model with no drop in response quality (abstract)", "benchmarks": [ "MixInstruct" ], "models_studied": [ "small vs large LLM pairs" ], "production_evidence": false, "summary": "Proposes a hybrid inference setup where a lightweight router predicts, per query, whether a small edge-deployable model will match the large model's quality. The quality threshold is tunable at test time to trade quality for cost.", "key_idea": "Route to small vs large model by predicting the quality gap, with a test-time tunable threshold.", "relevance_score": 4, "relevance_rationale": "Foundational difficulty-aware two-model router; directly transferable to per-call downscaling.", "found_via": "seed | arXiv title search: hybrid LLM cost-efficient quality-aware | OpenAlex title search", "cites_or_cited_by": [ "aggarwal-2023-automix", "ashiga-2025-ensemble-survey", "askin-2026-federated-router", "barrak-2025-cargo", "behera-2025-multi-llm-inference-survey", "bhatti-2026-proteus", "chen-2024-routerdc", "chen-2024-small-models-survey", "chen-2025-concur", "chen-2025-llm-ensemble-survey", "chen-2025-llm-slm-collab-survey", "chen-2025-tagrouter", "chen-2026-co-failure-ceiling", "chiang-2025-dueling-routing", "chuang-2024-self-ref", "dai-2024-c2mab-v", "dekoninck-2024-cascade-routing", "ding-2025-best-route", "feng-2024-graphrouter", "feng-2025-fusionfactory", "feng-2025-ipr", "feng-2026-llmrouter", "fernandez-2025-radar", "frick-2025-p2l", "garg-2026-unsolvability-ceiling", "hao-2026-racer", "huang-2025-routereval", "huang-2025-thriftllm", "jin-2025-radialrouter", "jitkrittum-2025-unirouter", "kassem-2025-router-robustness", "lai-2026-dars", "lai-2026-routefm", "lai-2026-routing-collapse", "lai-2026-saverouter", "li-2025-edge-cloud-survey", "li-2025-knn-routing", "li-2026-llmrouterbench", "lin-2025-router-vulnerabilities", "liu-2026-harness-native-routing", "liu-2026-trouter", "lu-2024-merge-ensemble-cooperate", "mahmood-2026-routing-user-choice", "manoharan-2026-t2mo", "markovic-voronov-2026-batch-routing", "maurya-2024-selectllm", "mei-2025-omnirouter", "mohammadshahi-2024-routoo", "moslem-2026-routing-cascading-survey", "nguyen-2024-metallm", "ong-2024-routellm", "pan-2025-route-to-reason", "panda-2025-pilot", "patel-2026-locus", "qian-2025-xrouter", "ramirez-2024-two-tier", "shen-2025-sater", "shi-2025-inferencedynamics", "shirkavand-2025-cscr", "song-2025-irt-router", "srivatsa-2024-lessons-routing", "stripelis-2024-to-router", "taberner-miller-2026-paretobandit", "tran-2025-arch-router", "varangot-reille-2025-doing-more-with-less", "wang-2025-mixllm", "wang-2025-reasoning-economy", "woisetschlager-2025-mess-plus", "wu-2025-training-free-online-routing", "xie-2025-gmtrouter", "xu-2026-seqroute", "yan-2026-zerorouter", "yang-2026-twinrouterbench", "yue-2025-dont-overthink-survey", "zhang-2025-meta-router", "zhang-2025-model-sat", "zhang-2025-router-r1", "zhang-2026-budget-aware-agentic-routing", "zheng-2025-disrouter", "zhou-2026-agent-as-a-router", "zhu-2025-concise-adaptive-thinking-survey" ] }, { "id": "feng-2024-graphrouter", "title": "GraphRouter: A Graph-based Router for LLM Selections", "authors": [ "Tao Feng", "Yanzhen Shen", "Jiaxuan You" ], "year": 2024, "venue_or_source": "ICLR (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2410.03834", "code_url": "https://github.com/ulab-uiuc/GraphRouter", "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "task-query-LLM heterogeneous graph", "edge prediction of effect and cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "GraphRouter builds a heterogeneous graph of task, query and LLM nodes and predicts edge attributes (response quality and cost) inductively. This lets it adapt to newly introduced LLMs without retraining; it reports at least 12.3% improvement over prior routers across three effect-cost weightings.", "key_idea": "Inductive graph edge prediction for LLM selection that generalizes to new models.", "relevance_score": 4, "relevance_rationale": "Key seed; handles changing model pools.", "found_via": "seed", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "fu-2024-dynasor-certaindex", "title": "Efficiently Scaling LLM Reasoning with Certaindex", "authors": [ "Yichao Fu", "Junda Chen", "Siqi Zhu", "Zheyu Fu", "Zhongdongming Dai", "Yonghao Zhuang", "Yian Ma", "Aurick Qiao", "Tajana Rosing", "Ion Stoica", "Hao Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.20993", "code_url": "https://github.com/hao-ai-lab/Dynasor.git", "topic_clusters": [ "C9", "C11" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "Certaindex answer stability", "certaindex (answer stability)" ], "cost_reduction_reported": "Up to 50% compute savings and 3.3x higher throughput in real workloads with no accuracy drop (Dynasor serving system)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Certaindex measures how stable intermediate answers are across reasoning algorithms and triggers early exit or reallocation. Built into Dynasor, a reasoning-aware serving system with gang scheduling.", "key_idea": "Answer-stability metric driving a reasoning-aware scheduler.", "relevance_score": 4, "relevance_rationale": "Serving-level reasoning budget control.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking) | Dynasor efficiently serving LLM reasoning programs Certaindex", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "gupta-2024-lm-cascades-token-uncertainty", "title": "Language Model Cascades: Token-level uncertainty and beyond", "authors": [ "Neha Gupta", "Harikrishna Narasimhan", "Wittawat Jitkrittum", "Ankit Singh Rawat", "Aditya Krishna Menon", "Sanjiv Kumar" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.10136", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "token-level uncertainty", "small-model embeddings", "large-model intermediate layers", "embeddings" ], "cost_reduction_reported": null, "benchmarks": [ "range of NLP benchmarks", "FLAN-T5 NLP benchmarks" ], "models_studied": [ "FLAN-T5 family", "FLAN-T5" ], "production_evidence": false, "summary": "Systematically studies deferral rules for generative LM cascades and shows that sequence-level uncertainty suffers from length bias. Learned post-hoc deferral rules over token-level uncertainty vectors, plus embeddings from both models, give better cost-quality trade-offs.", "key_idea": "Learned deferral over token-level uncertainty beats naive sequence-probability thresholds.", "relevance_score": 4, "relevance_rationale": "Seed; principled deferral-rule design for generative cascades.", "found_via": "seed: Language Model Cascades: Token-level uncertainty and beyond | keyword: confidence routing escalation", "cites_or_cited_by": [ "dong-2026-llm-shepherding", "farinhas-2025-qe-deferral-translation", "jitkrittum-2023-confidence-cascade-deferral", "lee-2025-steer", "nie-2025-foundation-model-programs", "shankar-2026-task-cascades", "warren-2025-bidirectional-cascading", "yoon-2025-reasoning-models-confidence" ] }, { "id": "han-2024-uala-uncertainty-aware-agent", "title": "Towards Uncertainty-Aware Language Agent", "authors": [ "Jiuzhou Han", "Wray Buntine", "Ehsan Shareghi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2401.14016", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "answer uncertainty", "tool-call gating" ], "cost_reduction_reported": "Improves performance while substantially reducing tool calls and tokens versus ReAct", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "UALA uses uncertainty estimates to decide when an agent should act on its own answer versus call external tools, improving accuracy with fewer tool calls and tokens than ReAct; finds verbalized confidence unreliable.", "key_idea": "Gate tool use by uncertainty to cut agent cost.", "relevance_score": 4, "relevance_rationale": "Step-level uncertainty gating directly reduces agent cost.", "found_via": "keyword: uncertainty LLM agents", "cites_or_cited_by": [] }, { "id": "hu-2024-adas", "title": "Automated Design of Agentic Systems", "authors": [ "Shengran Hu", "Cong Lu", "Jeff Clune" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.08435", "code_url": "https://github.com/ShengranHu/ADAS", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "validation accuracy", "archive of prior agents" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defines Automated Design of Agentic Systems (ADAS) and proposes Meta Agent Search, where a meta agent writes new agents in code using an archive of prior designs. Discovered agents outperform hand-designed agents and transfer across domains and models.", "key_idea": "A meta agent programs new agent designs in code, iterating over an archive of discoveries.", "relevance_score": 4, "relevance_rationale": "Seminal automated agent design work; the search space covers workflow and prompts but not cost-aware model choice.", "found_via": "seed: ADAS", "cites_or_cited_by": [ "el-2025-inefficiencies-meta-agents", "nie-2025-w4s", "zhang-2025-dgm", "zhou-2025-mass" ] }, { "id": "kolawole-2024-agreement-based-cascading", "title": "Agreement-Based Cascading for Efficient Inference", "authors": [ "Steven Kolawole", "Don Dennis", "Ameet Talwalkar", "Virginia Smith" ], "year": 2024, "venue_or_source": "TMLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2407.02348", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "ensemble agreement" ], "cost_reduction_reported": "Edge-to-cloud: up to 14x lower communication cost; cloud serving: 3x lower rental cost; API services: 2-25x lower average price per token/request vs state-of-the-art LLM cascades", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Uses agreement among an ensemble at each cascade level as the routing signal. The extra ensemble cost is offset by size gaps and parallel execution, and it can replace the best single model with better accuracy and efficiency.", "key_idea": "Ensemble agreement as a training-free deferral signal.", "relevance_score": 4, "relevance_rationale": "Simple black-box cascade rule with cost numbers across settings.", "found_via": "snowball: related work of cascade papers", "cites_or_cited_by": [] }, { "id": "kossen-2024-semantic-entropy-probes", "title": "Semantic Entropy Probes: Robust and Cheap Hallucination Detection in LLMs", "authors": [ "Jannik Kossen", "Jiatong Han", "Muhammed Razzak", "Lisa Schut", "Shreshth Malik", "Yarin Gal" ], "year": 2024, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2406.15927", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden states", "linear probe", "semantic entropy" ], "cost_reduction_reported": "Reduces semantic-uncertainty overhead to almost zero versus 5-10x sampling cost of semantic entropy", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Trains linear probes on hidden states of a single generation to predict semantic entropy, removing the need for multiple samples. SEPs keep strong hallucination detection and generalize out of distribution better than probes trained directly on accuracy.", "key_idea": "Approximate semantic entropy from one forward pass with a probe.", "relevance_score": 4, "relevance_rationale": "Exactly the kind of near-free signal that can gate routing/escalation.", "found_via": "seed (semantic entropy); WebSearch: semantic entropy probes", "cites_or_cited_by": [ "farquhar-2024-semantic-entropy-nature", "kuhn-2023-semantic-uncertainty" ] }, { "id": "li-2024-sparse-topology-debate", "title": "Improving Multi-Agent Debate with Sparse Communication Topology", "authors": [ "Yunxuan Li", "Yibing Du", "Jiageng Zhang", "Le Hou", "Peter Grabowski", "Yeqing Li", "Eugene Ie" ], "year": 2024, "venue_or_source": "EMNLP 2024 Findings", "source_type": "paper", "url": "https://arxiv.org/abs/2406.11776", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "Comparable or superior performance at significantly reduced computational cost; search summary cites over 40% input-token cost reduction in some configurations", "benchmarks": [ "reasoning and factuality tasks", "multimodal reasoning", "alignment labeling" ], "models_studied": [ "GPT", "Mistral" ], "production_evidence": false, "summary": "Studies communication connectivity in multi-agent debate and shows that sparse topologies match or beat fully connected debate at much lower compute. It also finds that giving stronger LLMs to more central agents helps, a form of role-aware model placement.", "key_idea": "Sparse debate graphs cut tokens; place stronger models at high-centrality nodes.", "relevance_score": 4, "relevance_rationale": "Both communication pruning and heterogeneous model placement by role.", "found_via": "web search: sparse communication topology debate", "cites_or_cited_by": [] }, { "id": "lin-2024-parrot", "title": "Parrot: Efficient Serving of LLM-based Applications with Semantic Variable", "authors": [ "Chaofan Lin", "Zhenhua Han", "Chengruidong Zhang", "Yuqing Yang", "Fan Yang", "Chen Chen", "Lili Qiu" ], "year": 2024, "venue_or_source": "OSDI 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2405.19888", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "semantic variables (inter-request dataflow)" ], "cost_reduction_reported": "up to an order-of-magnitude end-to-end improvement for popular LLM application use cases", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Parrot exposes application structure to the LLM service through Semantic Variables that annotate prompt inputs/outputs and link requests into a dataflow graph, letting the service schedule, batch and share prefixes across the requests of one application.", "key_idea": "Expose inter-request dataflow to the serving layer.", "relevance_score": 4, "relevance_rationale": "Application-aware serving for multi-call LLM apps, a direct fit to multi-agent pipelines.", "found_via": "task seed", "cites_or_cited_by": [ "luo-2025-autellix", "tan-2024-teola" ] }, { "id": "mohammadshahi-2024-routoo", "title": "Routoo: Learning to Route to Large Language Models Effectively", "authors": [ "Alireza Mohammadshahi", "Arshad Rafiq Shaikh", "Majid Yazdani" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2401.13979", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "LLM-based performance predictor", "cost/latency constraints" ], "cost_reduction_reported": "Matches Mixtral 8x7b on MMLU at one-third lower inference cost; with GPT-4 in pool nearly matches GPT-4 at half the cost (abstract)", "benchmarks": [ "MMLU" ], "models_studied": [ "Mixtral-8x7B", "GPT-4" ], "production_evidence": false, "summary": "Routoo uses a lightweight LLM performance predictor and a cost-aware selector to choose among LLMs per prompt under cost and latency constraints. On MMLU it matches Mixtral at a third lower cost and nearly matches GPT-4 at half the cost.", "key_idea": "Performance predictor plus cost-aware selector.", "relevance_score": 4, "relevance_rationale": "Clear cost numbers vs frontier baseline.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "narasimhan-2024-speculative-cascades", "title": "Faster Cascades via Speculative Decoding", "authors": [ "Harikrishna Narasimhan", "Wittawat Jitkrittum", "Ankit Singh Rawat", "Seungyeon Kim", "Neha Gupta", "Aditya Krishna Menon", "Sanjiv Kumar" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.19261", "code_url": null, "topic_clusters": [ "C2", "C11" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "token-level deferral rule", "draft/target distributions" ], "cost_reduction_reported": "Better cost-quality trade-offs than cascading and speculative decoding baselines (no single number in abstract)", "benchmarks": [ "range of language benchmarks" ], "models_studied": [ "Gemma", "T5" ], "production_evidence": false, "summary": "Combines cascades (defer hard inputs) with speculative decoding (parallel verification) by implementing a token-level deferral rule through speculative execution. Derives the optimal deferral rule and uses a plug-in approximation. Google Research also described it in a blog post on speculative cascades.", "key_idea": "Speculative execution of a cascade deferral rule, per token.", "relevance_score": 4, "relevance_rationale": "Seed; bridges cascades and serving-level speculative decoding.", "found_via": "seed: speculative cascades", "cites_or_cited_by": [] }, { "id": "notdiamond-2024-reduce-own-inference-costs", "title": "Using Not Diamond to reduce Not Diamond's inference costs by 51% (with one line of code)", "authors": [ "Not Diamond" ], "year": 2024, "venue_or_source": "Not Diamond blog", "source_type": "blog", "url": "https://www.notdiamond.ai/blog/using-not-diamond-to-reduce", "code_url": null, "topic_clusters": [ "C1", "C5", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "prompt", "user preference feedback", "cost tradeoff parameter" ], "cost_reduction_reported": "Enabling the cost-tradeoff parameter on Not Diamond's consumer chatbot cut inference cost 51% on average (~$750,000 annualised); power-user median messages fell from 5 to 4/day while week-over-week message growth was 43% vs 16% the prior week (vendor-reported, before/after on live traffic)", "benchmarks": [ "live chatbot traffic before/after" ], "models_studied": [], "production_evidence": true, "summary": "Not Diamond reports switching its own consumer chatbot's router from quality-maximising to a cost-tradeoff setting and compares cost and engagement metrics before and after the change on live traffic. Costs roughly halved with a small drop in power-user messaging.", "key_idea": "A single cost-quality tradeoff knob on a learned router halved production inference cost with small engagement impact.", "relevance_score": 4, "relevance_rationale": "Production evidence for query-level routing, though an uncontrolled before/after comparison.", "found_via": "direct: notdiamond.ai/blog index", "cites_or_cited_by": [] }, { "id": "ong-2024-routellm", "title": "RouteLLM: Learning to Route LLMs with Preference Data", "authors": [ "Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang", "Tianhao Wu", "Joseph E. Gonzalez", "M Waleed Kadous", "Ion Stoica", "Vin‐Cent Wu", "Ion Gabriel Stoica" ], "year": 2024, "venue_or_source": "arXiv (later ICLR 2025 per authors; venue not verified here)", "source_type": "paper", "url": "https://arxiv.org/abs/2406.18665", "code_url": "https://github.com/lm-sys/RouteLLM", "topic_clusters": [ "C1", "C6", "C13", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding", "human preference labels (Chatbot Arena)", "matrix factorization", "BERT classifier", "similarity-weighted ranking", "human preference data (Chatbot Arena)", "data augmentation", "human preference data", "LLM-judge labels" ], "cost_reduction_reported": "Blog: >85% cost reduction on MT-Bench, 45% on MMLU, 35% on GSM8K vs GPT-4 at 95% of GPT-4 quality; paper: >2x cost reduction in some cases without quality loss", "benchmarks": [ "MT-Bench", "MMLU", "GSM8K" ], "models_studied": [ "GPT-4", "Mixtral-8x7B", "Llama 2 70B" ], "production_evidence": false, "summary": "Trains binary routers that pick between a strong and a weak LLM per query using Chatbot Arena human preference data plus augmentation. Compares matrix factorization, BERT classifier, causal-LLM classifier and similarity-weighted ranking routers, and shows they transfer when the model pair is swapped at test time.", "key_idea": "Learn strong-vs-weak routing from pairwise human preference data with a tunable threshold.", "relevance_score": 4, "relevance_rationale": "Canonical open-source query-level router with threshold control; a baseline every routing study compares against.", "found_via": "seed | OpenAlex title: RouteLLM | seed: RouteLLM/LMSYS blog", "cites_or_cited_by": [ "almahairi-2024-anyscale-router", "askin-2026-federated-router", "behera-2025-multi-llm-inference-survey", "bell-2026-continual-model-routing", "bhatti-2026-proteus", "bouchard-2026-escalation-decision-theory", "chen-2024-small-models-survey", "chen-2025-concur", "chen-2025-llm-ensemble-survey", "chen-2025-tagrouter", "chen-2026-co-failure-ceiling", "chiang-2024-chatbot-arena", "chiang-2025-dueling-routing", "chuang-2024-self-ref", "dekoninck-2024-cascade-routing", "ding-2024-hybrid-llm", "ding-2025-best-route", "feng-2025-efficient-reasoning-models-survey", "feng-2025-fusionfactory", "feng-2026-llmrouter", "fernandez-2025-radar", "fisch-2026-pandoras-router", "frick-2025-p2l", "garg-2026-unsolvability-ceiling", "hao-2026-racer", "huang-2025-routereval", "jitkrittum-2025-unirouter", "kassem-2025-router-robustness", "kumar-2026-task-session-routing-eval", "kwartler-2026-harness-tokenomics", "lai-2026-dars", "lai-2026-routefm", "lai-2026-routing-collapse", "lai-2026-saverouter", "li-2024-serving-survey", "li-2025-edge-cloud-survey", "li-2025-knn-routing", "li-2026-llmrouterbench", "lin-2025-router-vulnerabilities", "liu-2025-efficient-inference-lrm-survey", "liu-2026-harness-native-routing", "liu-2026-trouter", "liu-2026-vllm-semantic-router", "lu-2025-routerarena", "mahmood-2026-routing-user-choice", "manoharan-2026-t2mo", "mei-2025-omnirouter", "moslem-2026-routing-cascading-survey", "nguyen-2024-metallm", "notdiamond-2024-rorf", "pan-2025-route-to-reason", "panda-2025-pilot", "patel-2025-proxrouter", "patel-2026-locus", "paul-2026-agentrouter-step-tiers", "pulishetty-2025-cross-attention-router", "qian-2025-xrouter", "qu-2025-efficient-reasoning-lrm-survey", "shirkavand-2025-cscr", "shnitzer-2023-routing-benchmark-datasets", "sikeridis-2024-pickllm", "son-2026-swe-router", "song-2025-irt-router", "sridhar-2026-correlated-bandits-routing", "stripelis-2024-to-router", "sui-2025-stop-overthinking", "taberner-miller-2026-paretobandit", "tran-2025-arch-router", "tsiourvas-2025-causal-routing", "unify-product", "varangot-reille-2025-doing-more-with-less", "wang-2025-mixllm", "wang-2025-reasoning-economy", "wang-2025-slm-llm-collab-survey", "wang-2025-when-to-reason", "woisetschlager-2025-mess-plus", "wu-2025-training-free-online-routing", "wu-2026-routerxbench", "xu-2026-seqroute", "yadav-2024-moerging-survey", "yang-2026-twinrouterbench", "zeng-2025-pac-efficient-routing", "zhang-2025-avengers", "zhang-2025-confidence-driven-router", "zhang-2025-meta-router", "zhang-2025-router-r1", "zhang-2026-budget-aware-agentic-routing", "zhao-2024-eagle", "zheng-2025-disrouter", "zhou-2026-agent-as-a-router", "zhu-2025-concise-adaptive-thinking-survey", "zhuang-2024-embedllm" ] }, { "id": "opsahlong-2024-mipro", "title": "Optimizing Instructions and Demonstrations for Multi-Stage Language Model Programs", "authors": [ "Krista Opsahl-Ong", "Michael J Ryan", "Josh Purtell", "David Broman", "Christopher Potts", "Matei Zaharia", "Omar Khattab" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.11695", "code_url": "https://github.com/stanfordnlp/dspy", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "stochastic mini-batch metric", "Bayesian surrogate model" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Llama-3-8B" ], "production_evidence": false, "summary": "Presents MIPRO, an optimizer for multi-stage LM programs that jointly searches instructions and few-shot demonstrations for every module without module-level labels, using a Bayesian surrogate over mini-batch evaluations and meta-optimized proposal generation. Beats baseline optimizers on five of seven programs, by up to 13% accuracy with Llama-3-8B.", "key_idea": "Bayesian-surrogate search over per-module instructions and demos with program-aware proposal generation.", "relevance_score": 4, "relevance_rationale": "Core optimizer for compound LM programs (DSPy MIPROv2); widely used baseline for config optimization.", "found_via": "seed: MIPRO", "cites_or_cited_by": [ "agrawal-2025-gepa", "khattab-2023-dspy", "tan-2025-langprobe", "wang-2025-evoagentx", "wang-2026-dropbox-dash-dspy", "wu-2025-optimas" ] }, { "id": "patel-2024-lotus", "title": "Semantic Operators: A Declarative Model for Rich, AI-based Data Processing", "authors": [ "Liana Patel", "Siddharth Jha", "Melissa Pan", "Harshit Gupta", "Parth Asawa", "Carlos Guestrin", "Matei Zaharia" ], "year": 2024, "venue_or_source": "arXiv (PVLDB version: Semantic Operators and Their Optimization in LOTUS)", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.11418", "code_url": "https://github.com/lotus-data/lotus", "topic_clusters": [ "C2", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "proxy scores", "gold-algorithm correlation" ], "cost_reduction_reported": "Optimizations accelerate semantic filter/join/group-by/top-k by up to 1,000x; LOTUS programs match or exceed SOTA pipeline accuracy while running up to 3.6x faster", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defines semantic operators (LLM-powered filter, join, map, top-k) with a declarative model, and optimizes them by replacing an expensive gold algorithm with cheap proxies and cascades while giving accuracy guarantees.", "key_idea": "Declarative semantic operators optimized via cascades with accuracy guarantees.", "relevance_score": 4, "relevance_rationale": "Cost-reduction via per-operator cascades with statistical guarantees.", "found_via": "search: LOTUS semantic operators model cascades optimization arXiv", "cites_or_cited_by": [] }, { "id": "ramirez-2024-two-tier", "title": "Optimising Calls to Large Language Models with Uncertainty-Based Two-Tier Selection", "authors": [ "Guillem Ramírez", "Alexandra Birch", "Ivan Titov", "Ivan S. Titov" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.02134", "code_url": null, "topic_clusters": [ "C1", "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "small-model generation uncertainty" ], "cost_reduction_reported": "Outperforms existing cascading and routing methods on 25 of 27 setups (three model pairs, nine tasks)", "benchmarks": [ "nine tasks" ], "models_studied": [ "three small/large LLM pairs" ], "production_evidence": false, "summary": "Uses only the uncertainty of the small LLM's generation to decide whether to call the large model, and beats routing and cascading methods requiring extra neural models in 25 of 27 setups.", "key_idea": "Small-model uncertainty as a simple escalation criterion.", "relevance_score": 4, "relevance_rationale": "Strong simple baseline.", "found_via": "snowball: cited-by ding-2024-hybrid-llm | snowball: related work of cascade papers | OpenAlex title search", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ramirez-2023-cache-and-distil" ] }, { "id": "shekhar-2024-optimizing-costs-llm-usage", "title": "Towards Optimizing the Costs of LLM Usage", "authors": [ "Shivanshu Shekhar", "Tanishq Dubey", "Koyel Mukherjee", "Apoorv Saxena", "Atharv Tyagi", "Nishanth Kotla" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.01742", "code_url": null, "topic_clusters": [ "C1", "C2", "C6", "C10" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted output quality", "token-reduction heuristics", "token count" ], "cost_reduction_reported": "Reduces costs by 40%-90% while improving quality by 4%-7% (enterprise and open-source document-processing datasets)", "benchmarks": [ "enterprise document datasets", "open-source summarisation/QA datasets", "enterprise datasets", "annotated open-source datasets" ], "models_studied": [], "production_evidence": true, "summary": "Predicts LLM output quality without calling the models and solves an LP-rounding selection under budget and latency, plus token-reduction heuristics for prompts.", "key_idea": "Quality prediction plus LP model selection plus prompt shortening.", "relevance_score": 4, "relevance_rationale": "Enterprise document-processing cost optimisation.", "found_via": "snowball: related work of cascade papers (verified on arXiv) | OpenAlex title search", "cites_or_cited_by": [ "sakota-2023-forc" ] }, { "id": "shen-2024-alpha-umi-multi-llm-agent", "title": "Small LLMs Are Weak Tool Learners: A Multi-LLM Agent", "authors": [ "Weizhou Shen", "Chenliang Li", "Hongzhan Chen", "Ming Yan", "Xiaojun Quan", "Hehong Chen", "Ji Zhang", "Fei Huang" ], "year": 2024, "venue_or_source": "EMNLP 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2401.07324", "code_url": "https://github.com/X-PLUG/Multi-LLM-Agent", "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "role-specific fine-tuning" ], "cost_reduction_reported": null, "benchmarks": [ "tool-use benchmarks" ], "models_studied": [ "7B/13B LLM backbones" ], "production_evidence": false, "summary": "Alpha-UMi decomposes a tool-using agent into planner, caller and summarizer roles, each backed by its own small LLM, trained with a global-then-local progressive fine-tuning scheme. The modular multi-LLM agent outperforms a single fine-tuned LLM of the same backbone on tool-use benchmarks.", "key_idea": "Split one agent into role-specialized small LLMs (planner/caller/summarizer).", "relevance_score": 4, "relevance_rationale": "Role-level model assignment with small models; shows specialization can replace a single larger model.", "found_via": "web search: small LLMs weak tool learners | hf-papers search: RL fine-tuning small LLM tool use", "cites_or_cited_by": [] }, { "id": "snell-2024-test-time-compute-optimal", "title": "Scaling LLM Test-Time Compute Optimally can be More Effective than Scaling Model Parameters", "authors": [ "Charlie Snell", "Jaehoon Lee", "Kelvin Xu", "Aviral Kumar" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.03314", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "prompt difficulty estimate", "process reward model" ], "cost_reduction_reported": "Compute-optimal allocation improves test-time compute efficiency by more than 4x vs best-of-N; in FLOPs-matched evaluation, a smaller model with test-time compute outperforms a 14x larger model on problems where it has non-trivial success", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies scaling test-time compute via verifier search and adaptive revision and finds the best method depends on prompt difficulty. Allocating compute per prompt by difficulty beats uniform best-of-N and can substitute for a much larger model on easier problems.", "key_idea": "Difficulty-adaptive test-time compute allocation can beat parameter scaling.", "relevance_score": 4, "relevance_rationale": "Seed; foundation for small-model-plus-compute vs large-model trade-offs.", "found_via": "seed: Scaling LLM Test-Time Compute Optimally", "cites_or_cited_by": [ "balachandran-2025-inference-time-scaling-complex-tasks", "brown-2024-large-language-monkeys", "damani-2024-learning-how-hard-to-think", "wu-2024-inference-scaling-laws", "zhang-2024-osca" ] }, { "id": "stripelis-2024-to-router", "title": "TensorOpera Router: A Multi-Model Router for Efficient LLM Inference", "authors": [ "Dimitris Stripelis", "Zijian Hu", "Jipeng Zhang", "Zhaozhuo Xu", "Alay Dilipbhai Shah", "Han Jin", "Yuhang Yao", "Salman Avestimehr", "Chaoyang He" ], "year": 2024, "venue_or_source": "EMNLP 2024 (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2408.12320", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding", "expert performance labels" ], "cost_reduction_reported": "Up to 40% improvement in query efficiency and up to 30% cost reduction while maintaining or improving performance by up to 10% vs standalone experts (abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "TensorOpera Router integrates multiple LLM experts behind one interface and routes each query to the best expert for its requirements. It reports efficiency and cost gains versus standalone experts.", "key_idea": "Commercial multi-expert query router behind a single API.", "relevance_score": 4, "relevance_rationale": "Industry router with reported cost/efficiency gains.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "tang-2024-minicheck", "title": "MiniCheck: Efficient Fact-Checking of LLMs on Grounding Documents", "authors": [ "Liyan Tang", "Philippe Laban", "Greg Durrett" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.10774", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "grounding fact-check" ], "cost_reduction_reported": "GPT-4-level fact-checking accuracy at 400x lower cost (MiniCheck-FT5, 770M)", "benchmarks": [ "LLM-AggreFact" ], "models_studied": [ "MiniCheck-FT5 (770M)", "GPT-4" ], "production_evidence": false, "summary": "Trains small fact-checkers on GPT-4-generated synthetic errors, reaching GPT-4 accuracy on grounding verification at 400x lower cost.", "key_idea": "Distill a frontier verifier into a tiny model.", "relevance_score": 4, "relevance_rationale": "Cheap grounding verifier usable to gate RAG agents.", "found_via": "keyword: verifier cheap", "cites_or_cited_by": [] }, { "id": "verga-2024-poll-juries", "title": "Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models", "authors": [ "Pat Verga", "Sebastian Hofstatter", "Sophia Althammer", "Yixuan Su", "Aleksandra Piktus", "Arkady Arkhangorodsky", "Minjie Xu", "Naomi White", "Patrick Lewis" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.18796", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "panel of small judges" ], "cost_reduction_reported": "Panel of smaller judges outperforms a single GPT-4 judge while being over seven times less expensive", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes evaluating with a panel of diverse smaller LLM judges (PoLL) and finds it outperforms a single large judge, has less intra-model bias and costs over 7x less.", "key_idea": "A jury of small judges beats one big judge at a fraction of the cost.", "relevance_score": 4, "relevance_rationale": "Cheap judge design directly relevant for routing feedback loops.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "wang-2024-agent-workflow-memory", "title": "Agent Workflow Memory", "authors": [ "Zora Zhiruo Wang", "Jiayuan Mao", "Daniel Fried", "Graham Neubig" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2409.07429", "code_url": "https://github.com/zorazrw/agent-workflow-memory", "topic_clusters": [ "C8", "C10" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "induced workflows from trajectories" ], "cost_reduction_reported": "24.6% and 51.1% relative success-rate improvement on Mind2Web and WebArena, with fewer steps taken on successful WebArena tasks", "benchmarks": [ "Mind2Web", "WebArena" ], "models_studied": [], "production_evidence": false, "summary": "AWM induces reusable routines (workflows) from agent trajectories, offline from training examples or online from test queries, and adds the relevant ones to the agent's memory to guide later tasks. It improves success and reduces the number of steps needed.", "key_idea": "Induce and reuse workflows so agents solve recurring tasks in fewer steps.", "relevance_score": 4, "relevance_rationale": "Workflow reuse lowers per-task steps (and hence calls); a basis for plan caching.", "found_via": "Agent Workflow Memory arXiv Wang", "cites_or_cited_by": [ "feng-2025-agentrr", "zhang-2025-agentic-plan-caching" ] }, { "id": "wang-2024-reasoning-token-economies", "title": "Reasoning in Token Economies: Budget-Aware Evaluation of LLM Reasoning Strategies", "authors": [ "Junlin Wang", "Siddhartha Jain", "Dejiao Zhang", "Baishakhi Ray", "Varun Kumar", "Ben Athiwaratkun" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.06461", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "compute budget" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Argues reasoning strategies should be compared at matched compute; with equal budgets, CoT self-consistency often beats more complex strategies, and multi-agent debate or Reflexion can get worse with more budget.", "key_idea": "Budget-aware evaluation of reasoning strategies.", "relevance_score": 4, "relevance_rationale": "Evaluation principle for multi-agent cost claims.", "found_via": "arXiv title search: token budget reasoning", "cites_or_cited_by": [] }, { "id": "wang-2024-rethinking-bounds-multi-agent-discussion", "title": "Rethinking the Bounds of LLM Reasoning: Are Multi-Agent Discussions the Key?", "authors": [ "Qineng Wang", "Zihao Wang", "Ying Su", "Hanghang Tong", "Yangqiu Song" ], "year": 2024, "venue_or_source": "ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2402.18272", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "reasoning tasks" ], "models_studied": [], "production_evidence": false, "summary": "Re-examines whether multi-agent discussion improves LLM reasoning and finds that a single agent with strong prompts (demonstrations) matches the best discussion frameworks across tasks and backbones. Multi-agent discussion only helps when no demonstrations are given.", "key_idea": "A well-prompted single agent matches multi-agent discussion.", "relevance_score": 4, "relevance_rationale": "Direct evidence on when multi-agent architecture is not worth its extra calls.", "found_via": "web search: seed", "cites_or_cited_by": [] }, { "id": "xia-2024-agentless", "title": "Agentless: Demystifying LLM-based Software Engineering Agents", "authors": [ "Chunqiu Steven Xia", "Yinlin Deng", "Soren Dunn", "Lingming Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.01489", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "$0.70 per issue at 32.00% on SWE-bench Lite, highest among open-source agents at the time", "benchmarks": [ "SWE-bench Lite" ], "models_studied": [], "production_evidence": false, "summary": "Agentless replaces an autonomous coding agent with a fixed three-phase pipeline (localize, repair, validate) and reaches the best open-source SWE-bench Lite result at the time at about $0.70 per issue. It questions whether complex agent loops are needed for many software tasks.", "key_idea": "A fixed simple pipeline can beat complex agents on cost and accuracy.", "relevance_score": 4, "relevance_rationale": "Key architecture-level cost evidence: workflow vs agent.", "found_via": "arXiv title search: Agentless", "cites_or_cited_by": [] }, { "id": "xia-2024-which-llm-to-play", "title": "Which LLM to Play? Convergence-Aware Online Model Selection with Time-Increasing Bandits", "authors": [ "Yu Xia", "Fang Kong", "Tong Yu", "Liya Guo", "Ryan A. Rossi", "Sungchul Kim", "Shuai Li" ], "year": 2024, "venue_or_source": "WWW 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2403.07213", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "task reward", "fine-tuning progress" ], "cost_reduction_reported": null, "benchmarks": [ "classification model selection", "online LLM selection" ], "models_studied": [ "API LLMs vs fine-tuned small LLMs" ], "production_evidence": false, "summary": "Studies online model selection when candidate models improve over time (e.g. a small model being fine-tuned), using a time-increasing bandit (TI-UCB) with change detection for convergence. Proves logarithmic regret and tests on classification and LLM selection.", "key_idea": "Increasing-then-converging bandit for choosing between costly API LLM and improving local model.", "relevance_score": 4, "relevance_rationale": "Directly models the API-vs-local-small-model decision under online learning.", "found_via": "OpenAlex title: online model selection LLM", "cites_or_cited_by": [ "dai-2024-c2mab-v" ] }, { "id": "xu-2024-theagentcompany", "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "authors": [ "Frank F. Xu", "Yufan Song", "Boxuan Li", "Yuxuan Tang", "Kritanjali Jain", "Mengxue Bao", "Zora Z. Wang", "Xuhui Zhou", "Zhitong Guo", "Murong Cao", "Mingyang Yang", "Hao Yang Lu", "Amaad Martin", "Zhe Su", "Leander Maben", "Raj Mehta", "Wayne Chi", "Lawrence Jang", "Yiqing Xie", "Shuyan Zhou", "Graham Neubig" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.14161", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "checkpoint-based partial credit" ], "cost_reduction_reported": null, "benchmarks": [ "TheAgentCompany" ], "models_studied": [ "closed API and open-weight LLMs" ], "production_evidence": false, "summary": "Simulated small software company with internal websites, data and simulated coworkers, where agents browse, code, run programs and communicate. The best baseline agent completes 30% of tasks autonomously.", "key_idea": "Digital-worker benchmark with consequential multi-tool tasks.", "relevance_score": 4, "relevance_rationale": "Enterprise-like long-horizon digital-worker tasks.", "found_via": "seed list: TheAgentCompany", "cites_or_cited_by": [] }, { "id": "zhang-2024-ecoact", "title": "EcoAct: Economic Agent Determines When to Register What Action", "authors": [ "Shaokun Zhang", "Jieyu Zhang", "Dujian Ding", "Mirian Hipolito Garcia", "Ankur Mallick", "Daniel Madrigal", "Menglin Xia", "Victor Rühle", "Qingyun Wu", "Chi Wang" ], "year": 2024, "venue_or_source": "ICLR 2025 Workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2411.01643", "code_url": null, "topic_clusters": [ "C3", "C10", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "tool names" ], "cost_reduction_reported": "Over 50% reduction in computational cost on multi-step reasoning tasks while maintaining performance", "benchmarks": [ "multi-step reasoning tool-use tasks" ], "models_studied": [], "production_evidence": false, "summary": "EcoAct gives the agent a single meta-tool to register tools on demand from their names, instead of loading all tool descriptions into context every step. Integrating tool registration into reasoning cuts cost by more than half on multi-step tasks without hurting accuracy.", "key_idea": "Register tools lazily inside the reasoning loop to shrink per-step context.", "relevance_score": 4, "relevance_rationale": "Seed; step-level tool-context cost lever.", "found_via": "web search: seed", "cites_or_cited_by": [] }, { "id": "zhang-2024-osca", "title": "Scaling LLM Inference with Optimized Sample Compute Allocation", "authors": [ "Kexun Zhang", "Shang Zhou", "Danqing Wang", "William Yang Wang", "Lei Li" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.22480", "code_url": "https://github.com/LeiLiLab/OSCA", "topic_clusters": [ "C4", "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "per-configuration sample allocation", "per-configuration accuracy estimates" ], "cost_reduction_reported": "Beats best single configuration with 128x less compute on code generation and 25x less on 4 reasoning tasks; better SWE-Bench accuracy with 3x less compute than default configuration", "benchmarks": [ "code generation", "4 reasoning tasks", "SWE-Bench" ], "models_studied": [], "production_evidence": false, "summary": "Learns an optimal mix of sampling configurations (model, temperature, language) and sample counts under a compute budget, including for an agentic SWE workflow.", "key_idea": "Optimise the mixture of inference configurations under budget.", "relevance_score": 4, "relevance_rationale": "Configuration optimisation across models including agent tasks.", "found_via": "snowball: related work of snell-2024-test-time-compute-optimal | search: LLM configuration tuning inference hyperparameters optimization arXiv", "cites_or_cited_by": [ "snell-2024-test-time-compute-optimal" ] }, { "id": "zhang-2024-treacle", "title": "Efficient Contextual LLM Cascades through Budget-Constrained Policy Learning", "authors": [ "Xuechen Zhang", "Zijian Huang", "Ege Onur Taga", "Carlee Joe-Wong", "Samet Oymak", "Jiasi Chen", "Carlee Joe‐Wong" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.13082", "code_url": null, "topic_clusters": [ "C1", "C2", "C5", "C6" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "question embedding", "response history consistency", "budget and latency", "budget" ], "cost_reduction_reported": "Cost savings of up to 85% compared to baselines while maintaining high accuracy", "benchmarks": [ "GSM8K", "CSQA", "LLC" ], "models_studied": [ "various LLMs and prompts" ], "production_evidence": false, "summary": "TREACLE is an RL policy that jointly picks the model and prompt scheme for each question under long-run monetary and latency budgets. It uses question embeddings and the consistency of previous responses as state.", "key_idea": "Budget-constrained RL policy over (model, prompt) choices with re-query ability.", "relevance_score": 4, "relevance_rationale": "Cascade as sequential decision process under budget.", "found_via": "arXiv search: \"LLM cascade\" | OpenAlex title search", "cites_or_cited_by": [ "xu-2026-seqroute" ] }, { "id": "zhao-2024-eagle", "title": "Eagle: Efficient Training-Free Router for Multi-LLM Inference", "authors": [ "Zesen Zhao", "Shuowei Jin", "Z. Morley Mao" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2409.15518", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "global and local ELO ratings", "query embedding neighbors", "global ELO", "local ELO over similar queries", "pairwise feedback" ], "cost_reduction_reported": "Up to 23.52% AUC improvement; 1/20 of baseline initialization time and 100-200x faster incremental updates (abstract)", "benchmarks": [ "RouterBench", "RouterBench-style datasets" ], "models_studied": [], "production_evidence": false, "summary": "Eagle is a training-free router combining global ELO rankings with local ELO computed over similar past queries. It improves routing AUC and supports very fast incremental updates for high-volume online serving.", "key_idea": "Training-free ELO-based routing with fast online updates.", "relevance_score": 4, "relevance_rationale": "Cheap to maintain in production.", "found_via": "snowball: cited-by ong-2024-routellm | OpenAlex title search", "cites_or_cited_by": [ "chiang-2024-chatbot-arena", "ong-2024-routellm" ] }, { "id": "zhao-2024-lora-land", "title": "LoRA Land: 310 Fine-tuned LLMs that Rival GPT-4, A Technical Report", "authors": [ "Justin Zhao", "Timothy Wang", "Wael Abid", "Geoffrey Angus", "Arnav Garg", "Jeffery Kinnison", "Alex Sherstinsky", "Piero Molino", "Travis Addair", "Devvret Rishi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.00732", "code_url": null, "topic_clusters": [ "C8", "C11", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "task datasets" ], "cost_reduction_reported": "4-bit LoRA fine-tuned models beat base models by 34 points and GPT-4 by 10 points on average; 25 LoRA Mistral-7B models served on one A100", "benchmarks": [ "31 tasks" ], "models_studied": [ "Mistral-7B", "GPT-4" ], "production_evidence": true, "summary": "Predibase technical report evaluating 310 LoRA fine-tuned models across 10 base models and 31 tasks, and LoRAX serving of many adapters on a single GPU (LoRA Land app).", "key_idea": "Many small task-specific LoRAs beat GPT-4 and can be served cheaply together.", "relevance_score": 4, "relevance_rationale": "Industry evidence on specialization plus multi-adapter serving economics.", "found_via": "seed (Predibase)", "cites_or_cited_by": [] }, { "id": "zhuang-2024-embedllm", "title": "EmbedLLM: Learning Compact Representations of Large Language Models", "authors": [ "Richard Zhuang", "Tianhao Wu", "Zhaojin Wen", "Andrew Li", "Jiantao Jiao", "Kannan Ramchandran", "Andrew C. Li" ], "year": 2024, "venue_or_source": "ICLR (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2410.02223", "code_url": null, "topic_clusters": [ "C1", "C6", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "learned model embeddings", "query embedding", "correctness labels", "model embedding", "historical correctness" ], "cost_reduction_reported": null, "benchmarks": [ "multiple benchmarks", "correctness forecasting across benchmarks", "model routing" ], "models_studied": [ "many HF models", "many Hugging Face LLMs" ], "production_evidence": false, "summary": "EmbedLLM learns compact vector representations of LLMs with an encoder-decoder trained on query-level correctness data. The embeddings support routing and forecasting benchmark performance without running the model, and outperform prior methods on routing accuracy and latency.", "key_idea": "Matrix-factorization-style LLM embeddings reused across routing and performance prediction.", "relevance_score": 4, "relevance_rationale": "Scalable model representation for routing over large and changing pools.", "found_via": "seed | OpenAlex title search | seed: EmbedLLM", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "zhuge-2024-agent-as-a-judge", "title": "Agent-as-a-Judge: Evaluate Agents with Agents", "authors": [ "Mingchen Zhuge", "Changsheng Zhao", "Dylan Ashley", "Wenyi Wang", "Dmitrii Khizbullin", "Yunyang Xiong", "Zechun Liu", "Ernie Chang", "Raghuraman Krishnamoorthi", "Yuandong Tian", "Yangyang Shi", "Vikas Chandra", "Jürgen Schmidhuber" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.10934", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "agent judge", "requirement satisfaction" ], "cost_reduction_reported": null, "benchmarks": [ "DevAI" ], "models_studied": [ "MetaGPT", "GPT-Pilot", "OpenHands" ], "production_evidence": false, "summary": "Proposes using agentic systems (with tools to inspect workspaces and traces) to judge other agents, giving intermediate feedback over the whole task process. Introduces DevAI, 55 AI-development tasks with 365 hierarchical requirements, and finds Agent-as-a-Judge is far more reliable than LLM-as-a-Judge and comparable to human evaluators.", "key_idea": "Use an agent with tools as the evaluator of agent trajectories.", "relevance_score": 4, "relevance_rationale": "Provides a ground-truth-free trajectory quality signal; the paper also reports large cost/time savings versus human evaluation.", "found_via": "seed", "cites_or_cited_by": [ "zheng-2023-mt-bench" ] }, { "id": "aggarwal-2023-automix", "title": "AutoMix: Automatically Mixing Language Models", "authors": [ "Pranjal Aggarwal", "Aman Madaan", "Ankit Anand", "Srividya Pranavi Potharaju", "Swaroop Mishra", "Pei Zhou", "Aditya Gupta", "Dheeraj Rajagopal", "Karthik Kappaganthu", "Yiming Yang", "Shyam Upadhyay", "Manaal Faruqui", "Mausam" ], "year": 2023, "venue_or_source": "NeurIPS 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.12963", "code_url": null, "topic_clusters": [ "C1", "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "few-shot self-verification", "POMDP router over confidence", "answer confidence", "POMDP belief", "self-verification", "POMDP router" ], "cost_reduction_reported": "Reduces computational cost by over 50% for comparable performance (five LMs, five datasets)", "benchmarks": [ "five datasets", "five context-grounded reasoning datasets" ], "models_studied": [ "five language models", "LLaMA2-13B", "LLaMA2-70B" ], "production_evidence": false, "summary": "AutoMix lets a small model answer, estimates correctness through few-shot self-verification, and uses a POMDP-based router to decide whether to escalate to larger models given noisy verification. Across five models and datasets it cuts cost by over half at comparable quality.", "key_idea": "Self-verification plus POMDP escalation policy.", "relevance_score": 4, "relevance_rationale": "Widely cited cascade/router hybrid with a sequential decision formulation.", "found_via": "snowball: related work of ding-2024-hybrid-llm | seed: AutoMix | keyword: self-verification routing", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "huang-2026-rlcascaderouter", "pona-2026-calibrate-then-delegate", "rajput-2026-cheap-verifiers-blind-spots", "zeng-2026-learning-to-seek-help" ] }, { "id": "chen-2023-fireact", "title": "FireAct: Toward Language Agent Fine-tuning", "authors": [ "Baian Chen", "Chang Shu", "Ehsan Shareghi", "Nigel Collier", "Karthik Narasimhan", "Shunyu Yao" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.05915", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "teacher trajectories (GPT-4)" ], "cost_reduction_reported": "Fine-tuning Llama2-7B on 500 GPT-4 agent trajectories gives a 77% HotpotQA performance increase; paper discusses cost/efficiency gains versus prompting", "benchmarks": [ "HotpotQA" ], "models_studied": [ "Llama2-7B", "GPT-4" ], "production_evidence": false, "summary": "Fine-tunes smaller LMs on GPT-4-generated ReAct-style agent trajectories across multiple tasks and prompting methods, showing large gains in accuracy, robustness and efficiency versus few-shot prompting.", "key_idea": "Fine-tune small agents on teacher trajectories instead of prompting large models.", "relevance_score": 4, "relevance_rationale": "Seminal agent distillation evidence with explicit cost/efficiency discussion.", "found_via": "seed", "cites_or_cited_by": [ "kang-2025-agent-distillation-retrieval-code", "liu-2025-structured-agent-distillation" ] }, { "id": "chen-2023-frugalgpt", "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance", "authors": [ "Lingjiao Chen", "Matei Zaharia", "James Zou" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "paper", "url": "https://arxiv.org/abs/2305.05176", "code_url": null, "topic_clusters": [ "C1", "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "answer quality scorer", "cascade thresholds", "learned answer scorer", "cost budget" ], "cost_reduction_reported": "Matches best individual LLM (e.g. GPT-4) with up to 98% cost reduction, or +4% accuracy over GPT-4 at the same cost (abstract)", "benchmarks": [], "models_studied": [ "GPT-4", "ChatGPT", "J1-Jumbo" ], "production_evidence": false, "summary": "Surveys cost reduction strategies for LLM APIs (prompt adaptation, LLM approximation, cascades) and proposes FrugalGPT, a learned cascade that decides which LLM combinations to query per input. It reports large cost reductions on classification-style tasks.", "key_idea": "Learned LLM cascade with a scorer deciding when to stop.", "relevance_score": 4, "relevance_rationale": "Foundational cost-reduction paper that framed routing/cascades for API LLMs.", "found_via": "snowball: related work of ong-2024-routellm | arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "khattab-2023-dspy", "title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines", "authors": [ "Omar Khattab", "Arnav Singhvi", "Paridhi Maheshwari", "Zhiyuan Zhang", "Keshav Santhanam", "Sri Vardhamanan", "Saiful Haq", "Ashutosh Sharma", "Thomas T. Joshi", "Hanna Moazam", "Heather Miller", "Matei Zaharia", "Christopher Potts" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.03714", "code_url": "https://github.com/stanfordnlp/dspy", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task metric on training set", "bootstrapped demonstrations" ], "cost_reduction_reported": "Compiled DSPy programs on small open LMs (770M T5, llama2-13b-chat) are competitive with expert-written prompt chains for proprietary GPT-3.5", "benchmarks": [], "models_studied": [ "GPT-3.5", "llama2-13b-chat", "T5-770M" ], "production_evidence": false, "summary": "Introduces DSPy, a programming model where LM pipelines are declarative modules with learnable prompts/demonstrations, plus a compiler that optimizes a whole pipeline toward a metric. Shows bootstrapped pipelines beat standard few-shot prompting and let small open models approach proprietary-model pipelines.", "key_idea": "Treat LM pipelines as parameterized programs and compile (optimize) prompts/demos end-to-end against a metric.", "relevance_score": 4, "relevance_rationale": "Foundational framework for optimizing compound AI pipelines; the cheaper-model-with-optimized-program result is a direct cost lever.", "found_via": "seed: DSPy", "cites_or_cited_by": [ "opsahlong-2024-mipro", "soylu-2024-bettertogether", "tan-2025-langprobe", "zaharia-2024-compound-ai-systems", "zhang-2026-po-coin-flip" ] }, { "id": "kim-2023-llmcompiler", "title": "An LLM Compiler for Parallel Function Calling", "authors": [ "Sehoon Kim", "Suhong Moon", "Ryan Tabrizi", "Nicholas Lee", "Michael W. Mahoney", "Kurt Keutzer", "Amir Gholami" ], "year": 2023, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2312.04511", "code_url": "https://github.com/SqueezeAILab/LLMCompiler", "topic_clusters": [ "C4", "C11", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "planner-generated DAG", "function-call dependency DAG" ], "cost_reduction_reported": "up to 3.7x latency speedup, up to 6.7x cost savings and up to ~9% accuracy improvement vs ReAct; up to 1.35x latency gain over OpenAI parallel function calling", "benchmarks": [ "function-calling tasks with varied dependency patterns" ], "models_studied": [ "LLaMA", "GPT models" ], "production_evidence": false, "summary": "LLMCompiler plans function calls as a DAG and executes independent calls in parallel via a planner, task-fetching unit and executor, instead of ReAct-style sequential reasoning. It cuts latency and cost while improving accuracy.", "key_idea": "Compile agent tool calls into a parallel execution plan.", "relevance_score": 4, "relevance_rationale": "Architecture-level cost reduction for tool-using agents; complements config search.", "found_via": "seed: LLMCompiler | LLMCompiler parallel function calling", "cites_or_cited_by": [ "ye-2025-speculative-actions" ] }, { "id": "lin-2023-swiftsage", "title": "SwiftSage: A Generative Agent with Fast and Slow Thinking for Complex Interactive Tasks", "authors": [ "Bill Yuchen Lin", "Yicheng Fu", "Karina Yang", "Faeze Brahman", "Shiyu Huang", "Chandra Bhagavatula", "Prithviraj Ammanabrolu", "Yejin Choi", "Xiang Ren" ], "year": 2023, "venue_or_source": "NeurIPS 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.17390", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "heuristic triggers (action failure, stagnation)" ], "cost_reduction_reported": null, "benchmarks": [ "ScienceWorld" ], "models_studied": [ "small fine-tuned encoder-decoder LM (Swift)", "GPT-4 (Sage)" ], "production_evidence": false, "summary": "A dual-process agent pairs a small fine-tuned encoder-decoder that proposes actions quickly with a large LLM that is invoked for planning and grounding when heuristics signal trouble. The combination beats ReAct, Reflexion and SayCan on 30 ScienceWorld tasks while calling the expensive LLM less often.", "key_idea": "Small imitation-learned policy acts by default; a large LLM is called only when heuristics detect difficulty.", "relevance_score": 4, "relevance_rationale": "Canonical big-little switching inside an agent loop; directly analogous to per-step downscaling.", "found_via": "arXiv title search: SwiftSage", "cites_or_cited_by": [] }, { "id": "lu-2023-zooter", "title": "Routing to the Expert: Efficient Reward-guided Ensemble of Large Language Models", "authors": [ "Keming Lu", "Hongyi Yuan", "Runji Lin", "Junyang Lin", "Zheng Yuan", "Chang Zhou", "Jingren Zhou" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "paper", "url": "https://arxiv.org/abs/2311.08692", "code_url": null, "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "reward model scores (distilled)", "query tags", "query embedding", "distilled reward model scores", "tags" ], "cost_reduction_reported": null, "benchmarks": [ "benchmark collection of 26 subsets", "26 subsets across domains" ], "models_studied": [ "open LLMs" ], "production_evidence": false, "summary": "Zooter distills reward-model scores on training queries into a routing function that sends each query to the LLM with relevant expertise, avoiding the cost of generating from every model and ranking. Tag-based label enhancement reduces noise from silver reward supervision; across 26 subsets it beats the best single model on average.", "key_idea": "Distill off-the-shelf reward-model rankings into a cheap pre-generation router.", "relevance_score": 4, "relevance_rationale": "Early reward-guided router showing routing can replace expensive output reranking.", "found_via": "seed | OpenAlex title search", "cites_or_cited_by": [ "hasanaliyev-2026-expected-reward-prediction" ] }, { "id": "ramirez-2023-cache-and-distil", "title": "Cache & Distil: Optimising API Calls to Large Language Models", "authors": [ "Guillem Ramírez", "Matthias Lindemann", "Alexandra Birch", "Ivan Titov", "Ivan S. Titov" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.13561", "code_url": null, "topic_clusters": [ "C2", "C5", "C8", "C10" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "active-learning criteria (margin sampling, query by committee)", "active learning criteria (margin, QBC)" ], "cost_reduction_reported": null, "benchmarks": [ "classification tasks" ], "models_studied": [ "LLM teacher", "small student" ], "production_evidence": false, "summary": "Neural caching: a student model is continuously trained on LLM responses and a policy decides which requests the student handles alone. Margin sampling and query-by-committee policies work best.", "key_idea": "Online student distillation with active-learning deferral.", "relevance_score": 4, "relevance_rationale": "Distillation from production traffic with deferral.", "found_via": "snowball: related work of FrugalGPT (2305.05176) | OpenAlex title search", "cites_or_cited_by": [ "nie-2024-online-cascade-learning", "ramirez-2024-two-tier", "wu-2025-inter-cascade" ] }, { "id": "sakota-2023-forc", "title": "Fly-Swat or Cannon? Cost-Effective Language Model Choice via Meta-Modeling", "authors": [ "Marija Šakota", "Maxime Peyrard", "Robert West" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2308.06077", "code_url": "https://github.com/epfl-dlab/forc", "topic_clusters": [ "C1", "C2", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "meta-model predicted performance", "cost budget" ], "cost_reduction_reported": "match the performance of the largest LM while achieving a cost reduction of 63% (14 datasets, four LMs)", "benchmarks": [ "14 datasets, five task types", "14 datasets across five tasks" ], "models_studied": [ "four candidate LMs" ], "production_evidence": false, "summary": "A meta-model predicts how well each candidate LM will do on each input, then an assignment solves budget-constrained or cost-minimising allocation across a batch.", "key_idea": "Meta-model plus batch assignment for LM choice.", "relevance_score": 4, "relevance_rationale": "Early router; batch-assignment framing.", "found_via": "snowball: related work of cascade papers (verified on arXiv) | OpenAlex title search", "cites_or_cited_by": [ "shekhar-2024-optimizing-costs-llm-usage", "shnitzer-2023-routing-benchmark-datasets" ] }, { "id": "shnitzer-2023-routing-benchmark-datasets", "title": "Large Language Model Routing with Benchmark Datasets", "authors": [ "Tal Shnitzer", "Anthony Ou", "Mírian Silva", "Kate Soule", "Yuekai Sun", "Justin Solomon", "Neil Thompson", "Mikhail Yurochkin" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2309.15789", "code_url": null, "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "per-model correctness classifiers", "query embedding (kNN)", "benchmark correctness labels" ], "cost_reduction_reported": null, "benchmarks": [ "HELM-style benchmark collection" ], "models_studied": [ "open-source LLMs" ], "production_evidence": false, "summary": "Recasts benchmark datasets as training data for a router that picks the best LLM for new tasks, reducing the problem to a collection of binary correctness classifiers. It shows consistent gains over any single model and discusses limits of transferring from benchmarks.", "key_idea": "Reuse benchmark outcomes to train per-model correctness predictors for routing.", "relevance_score": 4, "relevance_rationale": "Early principled formulation of learned routing from benchmark logs.", "found_via": "snowball: related work of ong-2024-routellm | OpenAlex title search", "cites_or_cited_by": [ "ong-2024-routellm", "sakota-2023-forc" ] }, { "id": "stogiannidis-2023-cache-me-if-you-can", "title": "Cache me if you Can: an Online Cost-aware Teacher-Student framework to Reduce the Calls to Large Language Models", "authors": [ "Ilias Marios Stogiannidis", "Stavros Vassos", "Prodromos Malakasiotis", "Ion Androutsopoulos" ], "year": 2023, "venue_or_source": "Conference on Empirical Methods in Natural Language Processing", "source_type": "paper", "url": "https://arxiv.org/abs/2310.13395", "code_url": null, "topic_clusters": [ "C5", "C8", "C10" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "student confidence", "LLM teacher labels" ], "cost_reduction_reported": "Reports significant operating-expense savings with only slightly lower performance on intent recognition and sentiment analysis (GPT-3.5/GPT-4 teachers)", "benchmarks": [], "models_studied": [ "GPT-3.5", "GPT-4", "k-NN", "MLP" ], "production_evidence": false, "summary": "Proposes an online teacher-student framework in which a cheap local model is trained on LLM responses as they arrive and gradually answers more queries itself, calling the LLM only when needed. Gives a method to tune the call criteria and measure the cost-performance trade-off on business classification tasks.", "key_idea": "Distill an LLM into a cheap student online from live calls and route to the student when it is confident.", "relevance_score": 4, "relevance_rationale": "Online learning from live traffic to cut LLM calls; directly applicable to repetitive enterprise subtasks.", "found_via": "snowball: related work of nie-2024-online-cascade-learning", "cites_or_cited_by": [ "nie-2024-online-cascade-learning" ] }, { "id": "wang-2023-ecooptigen", "title": "Cost-Effective Hyperparameter Optimization for Large Language Model Generation Inference", "authors": [ "Chi Wang", "Susan Xueqing Liu", "Ahmed H. Awadallah" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2303.04673", "code_url": "https://aka.ms/autogen", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "utility", "inference cost", "cost-based pruning" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-3.5", "GPT-4" ], "production_evidence": false, "summary": "EcoOptiGen tunes LLM inference hyperparameters (number of responses, temperature, max tokens, prompts, model) under a budget using economical hyperparameter optimization with cost-based pruning; implemented in FLAML's autogen package.", "key_idea": "Budget-aware HPO of LLM inference hyperparameters.", "relevance_score": 4, "relevance_rationale": "Early cost-aware LLM inference config tuning, ancestor of AutoGen tuning.", "found_via": "search: LLM configuration tuning inference hyperparameters optimization arXiv", "cites_or_cited_by": [] }, { "id": "yue-2023-mot-cascades", "title": "Large Language Model Cascades with Mixture of Thoughts Representations for Cost-efficient Reasoning", "authors": [ "Murong Yue", "Jie Zhao", "Min Zhang", "Liang Du", "Ziyu Yao" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.03094", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "answer consistency (sampling vote)", "CoT vs PoT agreement", "answer consistency", "mixture of thought representations" ], "cost_reduction_reported": "Performance comparable to GPT-4 alone at 40% of its cost (GPT-3.5-turbo as weak model), six reasoning benchmarks", "benchmarks": [ "six reasoning benchmarks (math, causal)" ], "models_studied": [ "GPT-3.5-turbo", "GPT-4" ], "production_evidence": false, "summary": "Uses the agreement among multiple sampled answers from a cheap LLM as a difficulty signal for escalating reasoning questions. Mixing chain-of-thought and program-of-thought samples makes the consistency check more discriminative.", "key_idea": "Answer consistency across mixed thought representations as a label-free deferral signal.", "relevance_score": 4, "relevance_rationale": "Seed; cheap, training-free escalation signal transferable to agent steps.", "found_via": "seed: Mixture-of-Thought cascades | keyword: self-consistency cost", "cites_or_cited_by": [] }, { "id": "zhang-2023-ecoassistant", "title": "EcoAssistant: Using LLM Assistant More Affordably and Accurately", "authors": [ "Jieyu Zhang", "Ranjay Krishna", "Ahmed H. Awadallah", "Chi Wang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.03046", "code_url": null, "topic_clusters": [ "C2", "C3", "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "code execution feedback", "success of past queries" ], "cost_reduction_reported": "Surpasses GPT-4 by 10 points of success rate with less than 50% of GPT-4's cost", "benchmarks": [ "code-driven API QA" ], "models_studied": [ "GPT-4" ], "production_evidence": false, "summary": "For code-driven assistant queries, tries cheaper LLMs first with an execution loop and backs off to stronger ones, and retrieves solutions from past successful queries as demonstrations.", "key_idea": "Assistant hierarchy plus solution retrieval for agentic code queries.", "relevance_score": 4, "relevance_rationale": "Early cascade for tool-using assistants.", "found_via": "snowball: related work of cascade papers (verified on arXiv)", "cites_or_cited_by": [] }, { "id": "zheng-2023-sglang", "title": "SGLang: Efficient Execution of Structured Language Model Programs", "authors": [ "Lianmin Zheng", "Liangsheng Yin", "Zhiqiang Xie", "Chuyue Sun", "Jeff Huang", "Cody Hao Yu", "Shiyi Cao", "Christos Kozyrakis", "Ion Stoica", "Joseph E. Gonzalez", "Clark Barrett", "Ying Sheng" ], "year": 2023, "venue_or_source": "NeurIPS 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2312.07104", "code_url": "https://github.com/sgl-project/sglang", "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "radix-tree prefix matching" ], "cost_reduction_reported": "up to 6.4x higher throughput than state-of-the-art inference systems on agent control, reasoning, few-shot, JSON decoding, RAG and multi-turn chat", "benchmarks": [ "agent control", "logical reasoning", "few-shot benchmarks", "JSON decoding", "RAG pipelines", "multi-turn chat" ], "models_studied": [], "production_evidence": false, "summary": "SGLang pairs a frontend language for LLM programs (generation and parallelism primitives) with a runtime featuring RadixAttention, which keeps KV caches in a radix tree for automatic prefix reuse across calls, and compressed finite-state machines for fast structured decoding. The LMSYS blog (lmsys.org/blog/2024-01-17-sglang) introduced RadixAttention.", "key_idea": "Co-design a program frontend with a runtime that reuses KV prefixes automatically.", "relevance_score": 4, "relevance_rationale": "Prefix reuse across the many calls of an agent program is a major serving cost lever.", "found_via": "task seed", "cites_or_cited_by": [ "dhanani-2026-dynamo-agentic-inference", "juravsky-2024-hydragen", "kwon-2023-vllm", "pan-2025-kvflow", "srivatsa-2024-preble", "ye-2024-chunkattention" ] }, { "id": "github-copilot-auto-model-selection", "title": "About Copilot auto model selection", "authors": [ "GitHub" ], "year": null, "venue_or_source": "GitHub Docs", "source_type": "docs", "url": "https://docs.github.com/en/copilot/concepts/auto-model-selection", "code_url": null, "topic_clusters": [ "C1", "C3", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "task complexity", "real-time model health", "cache boundaries", "conversation complexity drift" ], "cost_reduction_reported": "(vendor-reported) 10% discount on model costs when using auto selection on paid plans", "benchmarks": [], "models_studied": [ "Copilot-supported models" ], "production_evidence": true, "summary": "Copilot's auto mode combines a system that tracks model health and availability with one that estimates task complexity, and switches models at natural cache boundaries or large complexity shifts in a conversation, subject to admin policies.", "key_idea": "Cache-boundary-aware model switching in a coding assistant.", "relevance_score": 4, "relevance_rationale": "Production router explicitly aligning switches with cache boundaries.", "found_via": "seed list: automatic model selection", "cites_or_cited_by": [] }, { "id": "openrouter-2025-auto-router", "title": "Auto Router (Intelligent Model Selection)", "authors": [ "OpenRouter" ], "year": null, "venue_or_source": "OpenRouter docs", "source_type": "docs", "url": "https://openrouter.ai/docs/features/model-routing", "code_url": null, "topic_clusters": [ "C1", "C5", "C14" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "task-type classifier (~30 types)", "community spend over trailing 7 days", "cost tier" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "OpenRouter model catalog" ], "production_evidence": true, "summary": "OpenRouter's free auto router classifies each prompt into about 30 task types, ranks models by what the OpenRouter community actually spends on for that task over the last seven days, applies the user's cost tier, and returns primary picks with fallbacks and session stickiness. The current docs describe this market-signal design and do not mention Not Diamond (earlier versions were reported as Not Diamond-powered, unverified here).", "key_idea": "Route by aggregate market spend per task type.", "relevance_score": 4, "relevance_rationale": "Production router using live aggregate traffic as signal.", "found_via": "seed: OpenRouter auto-router", "cites_or_cited_by": [] }, { "id": "alballa-2026-comed", "title": "COMED: The Missing Middle Between Routing and Collaboration in Multi-LLM Inference", "authors": [ "Norah Alballa", "Wenxuan Zhang", "Salma Kharrat", "Fares Fourati", "Zafar Ayyub Qazi", "Mohamed Elhoseiny", "Marco Canini" ], "year": 2026, "venue_or_source": "AACL-IJCNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2609.26913", "code_url": null, "topic_clusters": [ "C1", "C2", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "anchor self-consistency", "router margin", "peer probe" ], "cost_reduction_reported": null, "benchmarks": [ "MedQA", "HLE" ], "models_studied": [ "GPT-5.5" ], "production_evidence": false, "summary": "COMED escalates to cross-model collaboration only when a post-anchor controller predicts rescued errors outweigh harms, improving routed anchors while using fewer calls than dense collaboration.", "key_idea": "Selective escalation between routing and full collaboration.", "relevance_score": 3, "relevance_rationale": "Selective multi-model use.", "found_via": "search: multi-LLM selection", "cites_or_cited_by": [] }, { "id": "anthropic-2026-compaction-docs", "title": "Compaction overview (Claude Platform docs)", "authors": [ "Anthropic" ], "year": 2026, "venue_or_source": "Claude Platform documentation", "source_type": "docs", "url": "https://platform.claude.com/docs/en/build-with-claude/compaction", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "input-token threshold" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude" ], "production_evidence": false, "summary": "Documents server-side compaction, where the API replaces older turns with a model-written summary either on demand or when input tokens cross a configured trigger, optionally keeping recent turns verbatim and running in the background. It contrasts this with client-side summarizers and with rule-based context editing.", "key_idea": "Provider-managed summarization of history keeps long agent tasks within a small active context.", "relevance_score": 3, "relevance_rationale": "Directly usable context-cost mechanism for long-horizon agents; no numbers.", "found_via": "direct: platform.claude.com docs", "cites_or_cited_by": [ "anthropic-2025-context-management", "openai-2026-compaction-docs" ] }, { "id": "anthropic-effort-docs", "title": "Effort (Claude Platform docs)", "authors": [ "Anthropic" ], "year": 2026, "venue_or_source": "Claude Platform documentation", "source_type": "docs", "url": "https://platform.claude.com/docs/en/build-with-claude/effort", "code_url": null, "topic_clusters": [ "C9", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude Opus 5.5", "Claude models" ], "production_evidence": false, "summary": "Describes output_config.effort, which controls how many tokens Claude spends (thinking, text and tool calls) on a single model, from low up to max; most models default to high while Opus 5.5 defaults to medium. A per-message effort setting is in beta, allowing effort to vary across turns of an agent loop, and effort is the recommended control for thinking depth where adaptive thinking is available.", "key_idea": "Single-model effort levels, settable per message, as a substitute for switching models to save cost.", "relevance_score": 3, "relevance_rationale": "Per-step effort control is a direct alternative lever to step-level model routing.", "found_via": "direct: platform.claude.com docs", "cites_or_cited_by": [ "anthropic-extended-thinking-docs", "anthropic-task-budgets-docs", "openai-reasoning-effort-docs" ] }, { "id": "ashley-2026-rpra", "title": "RPRA: Predicting an LLM-Judge for Efficient but Performant Inference", "authors": [ "Dylan R. Ashley", "Gaël Le Lan", "Changsheng Zhao", "Naina Dhingra", "Zhipeng Cai", "Ernie Chang", "Mingchen Zhuge", "Yangyang Shi", "Vikas Chandra", "Jürgen Schmidhuber" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.12634", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "self-predicted judge score", "report card in context" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Models predict, before answering, how an LLM judge would score their output, and defer when the prediction is low. Report cards in context and fine-tuning improve small models' prediction accuracy by up to 55% and 52%.", "key_idea": "Predict your own judge score to decide deferral.", "relevance_score": 3, "relevance_rationale": "Self-assessment as deferral signal.", "found_via": "arXiv search: learning to defer LLM", "cites_or_cited_by": [] }, { "id": "askin-2026-federated-router", "title": "Federate the Router: Learning Language Model Routers with Sparse and Decentralized Evaluations", "authors": [ "Baris Askin", "Shivam Patel", "Anupam Nayak", "Andrea Vigano", "Jiin Woo", "Gauri Joshi", "Carlee Joe-Wong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.22318", "code_url": null, "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "local offline evaluations", "federated MLP / K-means router" ], "cost_reduction_reported": null, "benchmarks": [ "two benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Introduces federated learning of LLM routers so clients with private, sparse query-model evaluations can learn a shared routing policy.", "key_idea": "Federated router training across clients.", "relevance_score": 3, "relevance_rationale": "Relevant for multi-tenant enterprise data.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "atreja-2026-pioneer-agent", "title": "Pioneer Agent: Continual Improvement of Small Language Models in Production", "authors": [ "Dhruv Atreja", "Julia White", "Nikhil Nayak", "Kelton Zhang", "Henrijs Princis", "George Hurn-Maloney", "Ash Lewis", "Urchade Zaratiana" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.09791", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "labeled production failures" ], "cost_reduction_reported": null, "benchmarks": [ "AdaptFT-Bench", "eight cold-start benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Pioneer Agent automates the lifecycle of specializing small models for production: data acquisition, evaluation set construction, training, and in production mode diagnosing labeled failures and retraining under regression constraints. It raises intent classification from 84.9% to 99.3% in a production-style deployment.", "key_idea": "Agent that continually specializes SLMs from production failures.", "relevance_score": 3, "relevance_rationale": "Supports replacing frontier calls with continually improved SLMs.", "found_via": "arXiv title search: small language models agentic", "cites_or_cited_by": [] }, { "id": "babu-2026-agentbetta", "title": "AgentBetta: Verification-Driven Adaptive Configuration of an AI Nano-Agent through Selective Expansion and Verified Contraction", "authors": [ "M. Babu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.23512", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "verification outcomes" ], "cost_reduction_reported": "Median context allocation reduced from 64,000 to 8,000 characters and tool exposure from five to zero at 91.38% verified success on AB-ConfigBench", "benchmarks": [ "AB-ConfigBench" ], "models_studied": [], "production_evidence": false, "summary": "AgentBetta treats an agent's model capability, context, tools, permissions and memory as an executable configuration. It starts minimal, expands selectively when verification diagnoses a deficiency, and contracts counterfactually after success.", "key_idea": "Verification-driven expand/contract of agent configuration, including model capability.", "relevance_score": 3, "relevance_rationale": "Adaptive per-task resource provisioning for agents.", "found_via": "snowball: cited-by su-2025-toolorchestra", "cites_or_cited_by": [ "su-2025-toolorchestra" ] }, { "id": "bachar-2026-llm-performance-predictors", "title": "LLM Performance Predictors: Learning When to Escalate in Hybrid Human-AI Moderation Systems", "authors": [ "Or Bachar", "Or Levi", "Sardhendu Mishra", "Adi Levi", "Manpreet Singh Minhas", "Justin Miller", "Omer Ben-Porat", "Eilon Sheetrit", "Jonathan Morra" ], "year": 2026, "venue_or_source": "AAMAS 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.07006", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "log-probabilities", "entropy", "uncertainty attribution indicators" ], "cost_reduction_reported": "Significant improvements in accuracy-cost trade-offs over existing uncertainty estimators (no single number in abstract)", "benchmarks": [ "multimodal and multilingual moderation tasks" ], "models_studied": [ "Gemini", "GPT", "Llama", "Qwen" ], "production_evidence": false, "summary": "Trains a meta-model on LLM performance predictors derived from outputs to decide when to automate moderation decisions and when to escalate to humans. Also gives explanations of failure conditions.", "key_idea": "Supervised meta-model on output features for when-to-escalate decisions.", "relevance_score": 3, "relevance_rationale": "Escalation to humans in real moderation workflows.", "found_via": "arXiv title search: escalation LLM", "cites_or_cited_by": [] }, { "id": "baral-2026-semantic-cache-operational-gap", "title": "Closing the Operational Gap in Semantic Caching", "authors": [ "Aditeya Baral", "Radoslav Ralev", "Iliya Sotirov Zhechev", "Srijith Rajamohan", "Jen Agarwal" ], "year": 2026, "venue_or_source": "EMNLP 2026 Industry Track", "source_type": "paper", "url": "https://arxiv.org/abs/2606.19719", "code_url": null, "topic_clusters": [ "C10", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "embedding similarity scores", "cache-hit threshold" ], "cost_reduction_reported": null, "benchmarks": [ "semantic caching datasets" ], "models_studied": [ "embedding / reranker models" ], "production_evidence": false, "summary": "Shows that offline ranking metrics (e.g., PR-AUC) poorly predict how a similarity model performs once a threshold is fixed in a deployed semantic cache. Proposes a precision-vs-cache-hit-ratio AUC and an operational retention rate, and argues that model selection is a threshold-utility problem.", "key_idea": "Evaluate semantic-cache models at deployment thresholds, not by offline ranking.", "relevance_score": 3, "relevance_rationale": "Evaluation methodology for cache correctness/cost tradeoff.", "found_via": "GPTCache semantic cache LLM", "cites_or_cited_by": [ "schroeder-2025-vcache" ] }, { "id": "barke-2026-agentrx", "title": "AgentRx: Diagnosing AI Agent Failures from Execution Trajectories", "authors": [ "Shraddha Barke", "Arnav Goyal", "Alind Khare", "Avaljot Singh", "Suman Nath", "Chetan Bansal" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.02475", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "failure-step localization" ], "cost_reduction_reported": null, "benchmarks": [ "AgentRx benchmark (115 failed trajectories)" ], "models_studied": [], "production_evidence": false, "summary": "Releases 115 manually annotated failed trajectories spanning API workflows, incident management and open-ended tasks, and AgentRx, an automated framework that pinpoints the critical failure step.", "key_idea": "Automated critical-step diagnosis for agent failures.", "relevance_score": 3, "relevance_rationale": "Failure diagnosis incl. incident-management workloads.", "found_via": "hf-papers search: agent failure detection", "cites_or_cited_by": [] }, { "id": "bell-2026-continual-model-routing", "title": "Continual Model Routing in Evolving Model Hubs", "authors": [ "Jack Bell", "Giacomo Carfì", "Gerlando Gramaglia", "Vincenzo Lomonaco" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.28577", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "contrastive embeddings", "replay" ], "cost_reduction_reported": null, "benchmarks": [ "CMRBench (2,000+ models)" ], "models_studied": [], "production_evidence": false, "summary": "Formalizes continual model routing as model hubs grow, releases CMRBench with over 2,000 candidates, and proposes CARvE contrastive embeddings with replay.", "key_idea": "Continual routing over growing model hubs.", "relevance_score": 3, "relevance_rationale": "Pool growth.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "belloni-2026-online-pandoras-box-cascading", "title": "Online Pandora's Box for Contextual LLM Cascading", "authors": [ "Alexandre Belloni", "Yan Chen", "Yehua Wei" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.07392", "code_url": null, "topic_clusters": [ "C2", "C5", "C15" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "request context", "output-dependent cost", "downstream reward" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formulates contextual LLM cascading as an online Pandora's Box problem where each API query reveals an output at a cost and only the deployed output's reward is observed. Learns reservation indices with GMM estimation plus UCB bounds and proves sqrt(T) regret.", "key_idea": "Weitzman-index cascading learned online with bandit feedback.", "relevance_score": 3, "relevance_rationale": "Theory for online cascade learning from traffic.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "bengal-2026-marginal-contribution-pg", "title": "Marginal-Contribution Policy Gradients under Filtered Feedback for Multi-Agent LLMs", "authors": [ "Elai Ben-Gal", "Stela Tong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.22785", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-online", "signals_used": [ "marginal contribution", "counterfactual evaluations" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K (routed)" ], "models_studied": [], "production_evidence": false, "summary": "Proves that shared-reward training of multi-agent LLM systems cannot distinguish agents that matter from ones that do not, and derives gradient-correct marginal-contribution signals identifiable from filtered feedback, with an optimal budget for exact counterfactual evaluations. In GRPO it improves routed GSM8K over winner-take-all training at no extra generation cost.", "key_idea": "Marginal-contribution credit assignment for routed multi-agent LLMs.", "relevance_score": 3, "relevance_rationale": "Credit assignment theory applicable to deciding which agents matter (sub-theme f).", "found_via": "arXiv query abs:\"credit assignment\" AND abs:\"multi-agent\" AND abs:\"LLM\" AND abs:cost", "cites_or_cited_by": [] }, { "id": "bertalanic-2026-small-llm-team-scaling-decomposition", "title": "An Exact Generate - Transform Decomposition of Small-LLM Team Scaling Across Orchestration Architectures", "authors": [ "Blaz Bertalanic", "Carolina Fortuna" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.36104", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "At equal call budgets token cost still varies 2.1x across architectures", "benchmarks": [ "GSM8K", "GSMHard", "ARC", "GPQA", "MMLU", "executable code" ], "models_studied": [ "five 7-9B models" ], "production_evidence": false, "summary": "Sweeps eight orchestration architectures with 7-9B models up to 30 calls and finds team-scaling returns are sharply task-dependent: large on arithmetic, small on multiple-choice. An exact generate-transform decomposition explains which architectures convert extra candidates into accuracy.", "key_idea": "Team scaling is a task- and architecture-specific bet.", "relevance_score": 3, "relevance_rationale": "Evidence on the value of adding agents.", "found_via": "snowball: cited-by bertalanic-2026-cost-of-consensus", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems", "zhang-2025-maas-agentic-supernet" ] }, { "id": "bian-2026-tokendance", "title": "TokenDance: Scaling Multi-Agent LLM Serving via Collective KV Cache Sharing", "authors": [ "Zhuohang Bian", "Feiyang Wu", "Chengrui Zhang", "Hangcheng Dong", "Yun Liang", "Youwei Zhuo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.03143", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "collective shared KV blocks", "sibling diffs" ], "cost_reduction_reported": "up to 2.7x more concurrent agents than vLLM prefix caching under SLO, per-agent KV storage reduced up to 17.5x, up to 1.9x prefill speedup", "benchmarks": [ "GenerativeAgents", "AgentSociety" ], "models_studied": [], "production_evidence": false, "summary": "Scales many-agent simulations by sharing KV blocks collectively and storing sibling agents' caches as block-sparse diffs against a master copy.", "key_idea": "Collective KV sharing with diff-compressed sibling caches.", "relevance_score": 3, "relevance_rationale": "Serving many agents cheaply on shared context.", "found_via": "Tokencake KV cache multi-agent function call serving", "cites_or_cited_by": [ "bian-2025-tokencake" ] }, { "id": "biswas-2026-sutradhara", "title": "Sutradhara: An Intelligent Orchestrator-Engine Co-design for Tool-based Agentic Inference", "authors": [ "Anish Biswas", "Kanishk Goel", "Srivarshinee S", "Jayashree Mohan", "Alind Khare", "Anjaly Parayil", "Ramachandran Ramjee", "Chetan Bansal" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.12967", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "orchestrator semantic hints" ], "cost_reduction_reported": "sustains up to 77% higher load at the same median first-token-response latency, or up to 15% lower median latency at the same load, and up to 11% lower end-to-end latency on A100", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Co-designs the agent orchestrator and the vLLM engine: tool-aware prompt splitting, streaming tool dispatch during decode, and orchestrator-informed cache management.", "key_idea": "Orchestrator-engine co-design for tool-based agents.", "relevance_score": 3, "relevance_rationale": "Tool-aware agent serving.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "xu-2024-conveyor" ] }, { "id": "bozoglan-2026-slm-database-agent", "title": "What Stops a Small Language Model From Driving a Database Agent", "authors": [ "Cevheri Bozoglan", "Yusuf Gundogdu", "Abdullah Kaya", "Koray Sirin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.21341", "code_url": null, "topic_clusters": [ "C3", "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "ledger events", "refusal codes" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "39 open-weight models" ], "production_evidence": false, "summary": "Over eleven days the authors drove a real open-source SQL client's agent mode with 39 local open-weight models (8,199 runs). Most small-model losses came from runs that did call tools, mostly transport/argument-shape failures, and five server-side fixes moved model results substantially without changing any model.", "key_idea": "Small-model agent failures are often interface failures, not capability gaps.", "relevance_score": 3, "relevance_rationale": "Practical evidence on making SLMs viable inside agents.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "burleigh-2026-slm-know-when-wrong", "title": "Do Small Language Models Know When They're Wrong? Confidence-Based Cascade Scoring for Educational Assessment", "authors": [ "Tyler Burleigh" ], "year": 2026, "venue_or_source": "NCME 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2604.19781", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "verbalised confidence" ], "cost_reduction_reported": "Best cascade approached large-LM accuracy (kappa 0.802 vs 0.819) at 76% lower cost and 61% lower latency", "benchmarks": [ "2,100 expert-scored student-AI math conversation decisions" ], "models_studied": [ "GPT-5.4", "Claude 4.5+", "Gemini 3.1" ], "production_evidence": false, "summary": "Evaluates verbalised confidence as the escalation signal for educational scoring cascades. Confidence discrimination varies widely across small models and is the bottleneck.", "key_idea": "Discrimination of verbalised confidence determines cascade value.", "relevance_score": 3, "relevance_rationale": "Practical evaluation of confidence-based escalation.", "found_via": "arXiv title search: cascades language models", "cites_or_cited_by": [] }, { "id": "chae-2026-optimal-sequential-annotations-ope", "title": "Optimal Sequential Annotations for Off-Policy Evaluation", "authors": [ "Woojin Chae", "Ezinne Nwankwo", "Hai-Tong Qin", "Angela Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.26707", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "LLM-as-judge labels", "expert annotations" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows how to spend a limited budget for expert reward annotation in sequential OPE when cheap LLM-judge labels are biased, using doubly robust OPE with missing rewards and variance-optimal annotation probabilities. Motivated by casenotes from a homelessness services nonprofit.", "key_idea": "Variance-optimal choice of which logged steps to label expertly for doubly robust OPE.", "relevance_score": 3, "relevance_rationale": "Label-budget allocation for evaluating logged policies when rewards come from judges.", "found_via": "query: off-policy evaluation LLM routing", "cites_or_cited_by": [] }, { "id": "chen-2026-coact", "title": "CoACT: Action-Preserving Observation Compression for Coding Agents", "authors": [ "Haorui Chen", "Yuancheng Zhu", "Yitong Zhang", "Jia Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.02911", "code_url": null, "topic_clusters": [ "C8", "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "next-action preservation reward", "length reward" ], "cost_reduction_reported": "average total token consumption reduced 33.0% on SWE-bench Verified with near-uncompressed task success", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "three agentic models" ], "production_evidence": false, "summary": "A teacher generates compressed candidates of each observation; candidates that would change the agent's next action are filtered out, and the shortest remaining ones supervise a lightweight observation compressor.", "key_idea": "Train a small compressor that keeps the agent's next action unchanged.", "relevance_score": 3, "relevance_rationale": "Action-preserving observation compression; distillation of a compressor.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [ "kang-2025-acon" ] }, { "id": "chen-2026-compression-gateway-cost", "title": "An Empirical Cost Attribution of Context-Compression Gateways in Multi-Turn Coding Agents", "authors": [ "Lu Chen", "Jiayu Shi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.22114", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Tool-schema filtering removes ~21K-57K tokens per turn; content compression saves only ~2% of cache-priced prefix per turn (real Claude Code/Codex sessions)", "benchmarks": [], "models_studied": [ "Claude Sonnet", "GPT-5" ], "production_evidence": true, "summary": "Instruments a production compression gateway between coding agents (Claude Code, Codex) and frontier LLMs and decomposes the token bill into tool-schema filtering, content compression and history summarization under controlled A/B runs. Only schema filtering is reliably positive once prompt caching is priced in.", "key_idea": "With prompt caching, compression savings are much smaller than benchmarks imply.", "relevance_score": 3, "relevance_rationale": "Production cost attribution for coding agents; supporting lever.", "found_via": "Semantic Scholar query \"per-turn model routing coding agent\"", "cites_or_cited_by": [] }, { "id": "chen-2026-concur", "title": "CONCUR: High-Throughput Agentic Batch Inference of LLM via Congestion-Based Concurrency Control", "authors": [ "Qiaoling Chen", "Zhisheng Ye", "Tian Tang", "Peng Sun", "Boyu Tian", "Guoteng Wang", "Shenggui Li", "Yonggang Wen", "Zhenhua Han", "Tianwei Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.22705", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "runtime KV cache pressure" ], "cost_reduction_reported": "batch inference throughput up to 4.09x on Qwen3-32B and 1.9x on DeepSeek-V3", "benchmarks": [ "real-world agent workloads" ], "models_studied": [ "Qwen3-32B", "DeepSeek-V3" ], "production_evidence": false, "summary": "Congestion-control-style admission for agentic batch inference: dynamically adjusts how many agents are active based on cache signals to avoid KV thrashing in the middle of runs.", "key_idea": "Admission control for agents to prevent cache thrashing.", "relevance_score": 3, "relevance_rationale": "Batch/offline agent workloads efficiency.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [] }, { "id": "chen-2026-llm-mab-survey", "title": "A Component-Based Survey of Interactions between Large Language Models and Multi-Armed Bandits", "authors": [ "Siguang Chen", "Chunli Lv", "Miao Xie" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.12945", "code_url": "https://github.com/bucky1119/Awesome-LLM-Bandit-Interaction", "topic_clusters": [ "C5", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of how bandit algorithms help LLM systems (from pretraining to RAG and personalization) and how LLMs help bandits, organized by component; with a curated GitHub list.", "key_idea": "Component-level survey of LLM-bandit interactions.", "relevance_score": 3, "relevance_rationale": "Useful map of bandit methods applied to LLMs.", "found_via": "snowball: cited-by panda-2025-pilot", "cites_or_cited_by": [] }, { "id": "chen-2026-oats-tool-selection", "title": "Outcome-Aware Tool Selection for Semantic Routers: Latency-Constrained Learning Without LLM Inference", "authors": [ "Hua-Min Chen", "Xun-Zhuo Liu", "Junchen Jiang", "Bowei He", "Xue Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.13426", "code_url": null, "topic_clusters": [ "C3", "C6" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "historical success centroids", "tool embeddings" ], "cost_reduction_reported": null, "benchmarks": [ "MetaTool", "ToolBench" ], "models_studied": [], "production_evidence": false, "summary": "OATS improves tool selection in semantic routers at LLM gateways by shifting tool embeddings toward the centroid of queries where each tool historically succeeded, an offline step that adds no serving latency. NDCG@5 on MetaTool rises from 0.869 to 0.940; learned re-rankers help only when outcome data is plentiful.", "key_idea": "Learn tool routing from historical outcomes offline with zero serving cost.", "relevance_score": 3, "relevance_rationale": "Learning routing from logged outcomes in gateways.", "found_via": "snowball: cited-by su-2025-toolorchestra", "cites_or_cited_by": [ "su-2025-toolorchestra" ] }, { "id": "chen-2026-resample-or-reroute", "title": "Resample or Reroute? Recoverable Stopping Debt Without Identified Action Selection", "authors": [ "Teng-Ruei Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.08665", "code_url": null, "topic_clusters": [ "C2", "C6" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "weak verifier acceptance" ], "cost_reduction_reported": "In a 152-query MBPP+ study, fixed escalation to Qwen2.5-14B exceeds leave-one-out 7B resampling by +2.882 pp after 7B rejection", "benchmarks": [ "MBPP+" ], "models_studied": [ "Qwen2.5-7B", "Qwen2.5-14B" ], "production_evidence": false, "summary": "Treats choosing between resampling and rerouting after a weak verifier as an identification problem because correctness is hidden, and orders three gates (recoverable stopping debt, action support, held-out selector value) that must pass before claiming a learned selector helps.", "key_idea": "Gated identification test before claiming a resample-vs-reroute selector adds value.", "relevance_score": 3, "relevance_rationale": "Rigorous treatment of counterfactual claims in escalation decisions.", "found_via": "snowball: cited-by dekoninck-2024-cascade-routing", "cites_or_cited_by": [ "dekoninck-2024-cascade-routing" ] }, { "id": "chen-2026-routing-oracle-stochastic", "title": "What Does a Routing Oracle Measure Under Stochastic Decoding? Coupling, Scorer Choice, and Single-Commit Ceilings", "authors": [ "Teng-Ruei Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.03436", "code_url": null, "topic_clusters": [ "C1", "C6", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "archived correctness samples" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K", "MATH-500", "GPQA" ], "models_studied": [ "11 open models" ], "production_evidence": false, "summary": "Shows that the hindsight oracle used in routing benchmarks is not the same decision class as a router that commits to one model before seeing outputs under stochastic decoding. Distinguishes single-commit ceilings from union probabilities and audits frozen correctness tensors from 11 models.", "key_idea": "Routing oracles under stochastic decoding overstate the achievable gain of single-commit routers.", "relevance_score": 3, "relevance_rationale": "Clarifies what can be identified when learning or evaluating routers from logged outcomes.", "found_via": "snowball: cited-by dekoninck-2024-cascade-routing", "cites_or_cited_by": [ "dekoninck-2024-cascade-routing" ] }, { "id": "chen-2026-token-economics-agents-survey", "title": "Token Economics for LLM Agents: A Dual-View Study from Computing and Economics", "authors": [ "Yuxi Chen", "Junming Chen", "Chenyu He", "Yiwei Li", "Yicheng Ji", "Yifan Wu", "Dingyu Yang", "Lansong Diao", "Lidan Shou", "Hongliang Zhang", "Huan Li", "Gang Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.09104", "code_url": null, "topic_clusters": [ "C12", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A survey framing tokens as economic production factors for agents, organized into single-agent budget control, multi-agent collaboration friction, ecosystem pricing and security. Useful framing for cost-aware agent design.", "key_idea": "Economic taxonomy of token spending in agents.", "relevance_score": 3, "relevance_rationale": "Survey of token cost levers.", "found_via": "web search: token consumption coding agents", "cites_or_cited_by": [] }, { "id": "chen-2026-workload-router-pool", "title": "The Workload-Router-Pool Architecture for LLM Inference Optimization: A Vision Paper from the vLLM Semantic Router Project", "authors": [ "Huamin Chen", "Xunzhuo Liu", "Bowei He", "Fuyuan Lyu", "Yankai Chen", "Xue Liu", "Yuhan Liu", "Junchen Jiang" ], "year": 2026, "venue_or_source": "arXiv (vision paper)", "source_type": "paper", "url": "https://arxiv.org/abs/2603.21354", "code_url": null, "topic_clusters": [ "C1", "C11", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Synthesizes the vLLM Semantic Router project's work (signal-driven routing, pool routing, semantic caching, agent and multimodal routing, fleet provisioning) into a Workload-Router-Pool architecture that couples workload mix, routing policy and model pool design.", "key_idea": "Joint view of workload, router and model pool for inference optimization.", "relevance_score": 3, "relevance_rationale": "Open-source production router context, including agentic routing.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "dai-2025-aragog", "liu-2026-adaptive-vlm-routing-cua", "zhang-2026-budget-aware-agentic-routing", "zhang-2026-evoroute" ] }, { "id": "chhikara-2026-matched-trajectory-replay", "title": "Evaluating Confidence-Gated Retrieval with Matched Trajectory Replay", "authors": [ "Prateek Chhikara" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.26846", "code_url": null, "topic_clusters": [ "C6", "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "verbalized confidence", "isotonic calibration" ], "cost_reduction_reported": null, "benchmarks": [ "HotpotQA", "MuSiQue" ], "models_studied": [ "Mistral", "GPT", "Qwen" ], "production_evidence": false, "summary": "Proposes matched trajectory replay, a protocol that fixes candidate answer states, evidence, budgets and action costs so different confidence-to-action policies can be compared on the same agent trajectories. Calibration increased accuracy among committed answers by up to 41 points but changed coverage and retrieval use.", "key_idea": "Compare agent gating policies by replaying matched trajectories with fixed costs.", "relevance_score": 3, "relevance_rationale": "Replay-based evaluation method for agent decision policies.", "found_via": "query: replay evaluation agent model change", "cites_or_cited_by": [] }, { "id": "chi-2026-agentmeter", "title": "Matching Matters: A Fair Quality-Efficiency Benchmark for Command-Line Agents", "authors": [ "Han Chi", "Jiaxin Qi", "Yan Cui", "Baisheng Lai", "Jianqiang Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.21140", "code_url": null, "topic_clusters": [ "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "AgentMeter Score (quality, budget sensitivity, zero-reward execution)" ], "cost_reduction_reported": null, "benchmarks": [ "AgentMeter" ], "models_studied": [], "production_evidence": false, "summary": "AgentMeter benchmarks command-line agents as model-CLI pairs with a metric combining task quality, budget sensitivity and wasted zero-reward execution, plus description variants for fair comparison. No CLI harness is best for every model, and success, cost and the combined score select different configurations.", "key_idea": "Evaluate model-harness pairs jointly on quality and efficiency.", "relevance_score": 3, "relevance_rationale": "Cost-aware evaluation of agent configurations.", "found_via": "arXiv search: cost-efficient LLM agent model selection", "cites_or_cited_by": [] }, { "id": "cho-2026-querybandits", "title": "No One Size Fits All: QueryBandits for Hallucination Mitigation", "authors": [ "Nicole Cho", "William Watson", "Alec Koppel", "Sumitra Ganesh", "Manuela Veloso" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.20332", "code_url": null, "topic_clusters": [ "C5", "C7" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "semantic query features", "calibrated reward" ], "cost_reduction_reported": "Top QueryBandit (Thompson Sampling) achieves 87.5% win rate over No-Rewrite across 16 QA scenarios", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Learns online which query-rewrite strategy to apply per query, using a contextual bandit over semantic features and a calibrated reward, to reduce hallucination in closed-source models. Finds no single rewrite policy is best and some static ones are worse than none.", "key_idea": "Contextual bandit choosing a prompt transformation per query online.", "relevance_score": 3, "relevance_rationale": "Per-query online configuration choice for black-box models; same pattern as config routing.", "found_via": "query: contextual bandit prompt optimization online", "cites_or_cited_by": [] }, { "id": "da-2026-routebalance", "title": "RouteBalance: Fused Model Routing and Load Balancing for Heterogeneous LLM Serving", "authors": [ "Wei Da", "E. Kalyvianaki" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.17949", "code_url": null, "topic_clusters": [ "C1", "C11" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "quality/cost predictors", "instance load" ], "cost_reduction_reported": "At its cost-priority corner, per-request cost ties the cheapest baseline; highest routing-decision quality (DeepEval 0.419) on a 13-instance 28-GPU cluster", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RouteBalance fuses model routing and load balancing into one online assignment over concrete model instances, trading off quality, latency and cost with a ~32 ms decision path on a heterogeneous cluster.", "key_idea": "Fuse router and load balancer for heterogeneous LLM serving.", "relevance_score": 3, "relevance_rationale": "Serving-aware routing relevant to multi-model agent deployments.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "dai-2026-policy-regret-embedding-routing", "title": "Policy Regret for Embedding Model Routing: Contextual Bandits with Low-Rank Experts", "authors": [ "Yan Dai", "Negin Golrezaei", "Patrick Jaillet" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.14929", "code_url": null, "topic_clusters": [ "C5", "C15" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "query context", "bandit feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formalizes routing queries across multiple embedding models as an adversarial contextual linear bandit with low-rank experts and shows that standard regret notions are misspecified or intractable for this setting. Proposes a log-quadratic policy class and a policy-gradient algorithm with policy-regret guarantees under bandit feedback.", "key_idea": "Policy-regret theory for online model routing under bandit feedback with low-rank expert structure.", "relevance_score": 3, "relevance_rationale": "Theory for online routing under partial feedback; embedding-model setting rather than generative LLMs.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "dhanani-2026-dynamo-agentic-inference", "title": "Full-Stack Optimizations for Agentic Inference with NVIDIA Dynamo", "authors": [ "Ishan Dhanani", "Matej Kosec" ], "year": 2026, "venue_or_source": "NVIDIA Technical Blog", "source_type": "blog", "url": "https://developer.nvidia.com/blog/full-stack-optimizations-for-agentic-inference-with-nvidia-dynamo/", "code_url": null, "topic_clusters": [ "C10", "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "KV block overlap scores", "agent hints (priority, expected output length)" ], "cost_reduction_reported": "Claude Code agents: 85-97% cache hit rates on repeated calls to the same worker; four-agent teams reach 97.2% aggregate cache hit rate; 11.7x read/write ratio for lead agents", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Describes making NVIDIA Dynamo agent-native: a frontend accepting agent hints, a KV-aware router with a global index of cached blocks and priority scheduling, and multi-tier KV management with TTL/priority retention for ephemeral subagent and reasoning blocks. Reports very high prefix-cache hit rates in agentic traffic.", "key_idea": "Agent hints plus KV-aware routing and tiered KV retention for agentic workloads.", "relevance_score": 3, "relevance_rationale": "Industry serving stack features aimed at agentic traffic, with cache-hit measurements.", "found_via": "NVIDIA Dynamo agentic inference KV-aware routing blog", "cites_or_cited_by": [ "qin-2024-mooncake", "zheng-2023-sglang" ] }, { "id": "dixit-2026-focus", "title": "FOCUS: Training-Free Decision-Preserving Context Compression for LLM Agents", "authors": [ "Shantanu Dixit", "Anson Bastos", "Xuchao Zhang", "Chetan Bansal", "Saravan Rajmohan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.37590", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "span-level reasoning-execution-feedback units" ], "cost_reduction_reported": "peak context cut by up to 48% and dependency by 73% while improving task success by up to 8.9 points over uncompressed execution", "benchmarks": [ "API/tool-calling", "QA", "web", "multi-turn dialogue benchmarks" ], "models_studied": [ "closed-API frontier models" ], "production_evidence": false, "summary": "Training-free, decision-preserving compression that operates on complete reasoning-execution-feedback spans in agent trajectories and attaches as a modular layer to any closed-API model.", "key_idea": "Compress agent history at span granularity while preserving decisions.", "relevance_score": 3, "relevance_rationale": "Plug-in compression for frontier-API agents.", "found_via": "tool output observation compression LLM agent tokens arXiv", "cites_or_cited_by": [ "kang-2025-acon" ] }, { "id": "dubey-2026-psmas-phase-scheduled", "title": "Phase-Scheduled Multi-Agent Systems for Token-Efficient Coordination", "authors": [ "Mohit Dubey" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.17400", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "task dependency topology" ], "cost_reduction_reported": "Mean 27.3% token reduction (21.4-34.8%) within 2.1 points of fully activated baseline", "benchmarks": [ "HotPotQA-MAS", "HumanEval-MAS", "ALFWorld-Multi", "WebArena-Coord" ], "models_studied": [], "production_evidence": false, "summary": "PSMAS activates agents on a schedule derived from task dependencies so only agents within an active window run each step, while idle agents get compressed summaries. On LangGraph it cuts tokens by about 27% at near-baseline performance.", "key_idea": "Temporal scheduling of agent activation with compressed context for idle agents.", "relevance_score": 3, "relevance_rationale": "MAS token-efficiency lever.", "found_via": "web search: scaling agents cost", "cites_or_cited_by": [] }, { "id": "elhaimeur-2026-multi-agent-tutoring-latency-cost", "title": "Latency and Cost of Multi-Agent Intelligent Tutoring at Scale", "authors": [ "Iizalaarab Elhaimeur", "Nikos Chrisochoides" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.24110", "code_url": null, "topic_clusters": [ "C11", "C12", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Both pay-per-token tiers cost well below a STEM textbook per student per semester under worst-case usage", "benchmarks": [ "live graduate STEM deployment (3,000+ requests)" ], "models_studied": [ "Gemini 2.5 Flash" ], "production_evidence": true, "summary": "Instruments a four-agent tutoring system on Gemini 2.5 Flash across throughput tiers and up to 50 concurrent users from a live deployment, showing parallel-phase latency compounding in MAS and giving tier-selection guidance on cost and latency.", "key_idea": "Production latency and cost of a deployed multi-agent tutor.", "relevance_score": 3, "relevance_rationale": "Production evidence on MAS serving cost.", "found_via": "arXiv API search", "cites_or_cited_by": [] }, { "id": "elzemity-2026-orchestrated-slms-malware", "title": "Small, Free, and Effective: Orchestrating Open-Weight Small Language Models to Outperform Single LLM for Malware Analysis", "authors": [ "Adel ElZemity", "Shujun Li", "Budi Arief" ], "year": 2026, "venue_or_source": "RAID 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2607.20216", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "CyberSecEval Malware Analysis" ], "models_studied": [ "eleven open-weight SLMs", "Qwen3-4B", "Foundation-Sec-8B", "six frontier LLMs" ], "production_evidence": false, "summary": "Tests whether orchestrated ensembles of small open-weight models match single large models on malware report analysis, comparing pipeline, debate, hierarchical consultation with a specialist and a hybrid architecture. The hybrid Qwen3-4B plus Foundation-Sec-8B system is the best SLM configuration.", "key_idea": "Orchestrated SLMs as a cheap alternative to frontier LLMs in security analysis.", "relevance_score": 3, "relevance_rationale": "Domain evidence for SLM orchestration.", "found_via": "arXiv search: small large model collaboration", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "fan-2026-agent-rrm", "title": "Exploring Reasoning Reward Model for Agents", "authors": [ "Kaixuan Fan", "Kaituo Feng", "Manyuan Zhang", "Tianshuo Peng", "Zhixun Li", "Yilei Jiang", "Shuang Chen", "Peng Pei", "Xunliang Cai", "Xiangyu Yue" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.22154", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "reasoning reward model", "critique", "score" ], "cost_reduction_reported": null, "benchmarks": [ "GAIA", "WebWalkerQA" ], "models_studied": [], "production_evidence": false, "summary": "Agent-RRM produces a reasoning trace, a critique and an overall process score for agent trajectories; integrating these signals (Reagent-U) into agentic RL reaches 43.7% on GAIA and 46.2% on WebWalkerQA.", "key_idea": "Structured reward model feedback for agent RL.", "relevance_score": 3, "relevance_rationale": "Rich trajectory-level quality signal.", "found_via": "arXiv API title search: agentic reward modeling", "cites_or_cited_by": [] }, { "id": "fan-2026-agentprocessbench", "title": "AgentProcessBench: Diagnosing Step-Level Process Quality in Tool-Using Agents", "authors": [ "Shengda Fan", "Xuyan Ye", "Yupeng Huo", "Zhi-Yuan Chen", "Yiju Guo", "Shenzhi Yang", "Wenkai Yang", "Shuqi Ye", "Jingwen Chen", "Haotian Chen", "Xin Cong", "Yankai Lin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.14465", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "step-level labels" ], "cost_reduction_reported": null, "benchmarks": [ "AgentProcessBench" ], "models_studied": [], "production_evidence": false, "summary": "1,000 tool-using trajectories with 8,509 human step labels (ternary: correct/neutral/erroneous) to diagnose step-level process quality; distinguishing neutral from erroneous actions is hard.", "key_idea": "Human-labeled step-quality benchmark for tool agents.", "relevance_score": 3, "relevance_rationale": "Measures PRMs for agents.", "found_via": "hf-papers search: process reward model agents tool use", "cites_or_cited_by": [] }, { "id": "fan-2026-science-of-collective-ai", "title": "Towards a Science of Collective AI: LLM-based Multi-Agent Systems Need a Transition from Blind Trial-and-Error to Rigorous Science", "authors": [ "Jingru Fan", "Dewen Liu", "Yufan Dang", "Huatao Li", "Yuheng Wang", "Wei Liu", "Feiyu Duan", "Xuanwen Ding", "Shu Yao", "Lin Wu", "Ruijie Shi", "Wai-Shing Leung", "Yuan Cheng", "Zhongyu Wei", "Cheng Yang", "Chen Qian", "Zhiyuan Liu", "Maosong Sun" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.05289", "code_url": null, "topic_clusters": [ "C12", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "collaboration gain metric" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A position paper arguing MAS research needs a collaboration gain metric that separates true collaboration benefits from simply spending more budget, plus a factor library for attribution. Relevant to judging whether multi-agent designs beat equal-budget single agents.", "key_idea": "Measure collaboration gain net of added budget.", "relevance_score": 3, "relevance_rationale": "Methodology for fair MAS-vs-single cost comparisons.", "found_via": "web search: science of scaling agents", "cites_or_cited_by": [ "chen-2024-optima", "dang-2025-puppeteer-evolving-orchestration", "kapoor-2024-ai-agents-that-matter", "kim-2025-science-of-scaling-agent-systems", "ye-2025-x-mas", "zhang-2024-agentprune" ] }, { "id": "fang-2026-recache", "title": "ReCache: Efficient KV Cache Reuse and Compression for Tool-Augmented LLM Agents", "authors": [ "Yichu Fang", "Sitong Wei", "Haozhe Hu", "Xiaoyu Shen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.19662", "code_url": "https://github.com/EIT-NLP/ReCache", "topic_clusters": [ "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "resource-wise attention", "layer/KV-head routing" ], "cost_reduction_reported": "resource-wise attention matches dense invocation (82.3% vs 82.4% Inv-F1) with 3.655x TTFT speedup; full framework reduces KV memory 92.43% and speeds attention 1.423x", "benchmarks": [ "benchmark from seven tool- and skill-use datasets" ], "models_studied": [], "production_evidence": false, "summary": "Encodes tool/skill schemas as reusable, composition-invariant KV blocks and restricts attention to selected resources, cutting prefill and memory for agents with large tool catalogues.", "key_idea": "Precompute tool-schema KV once and attend selectively.", "relevance_score": 3, "relevance_rationale": "Reduces the cost of large tool definitions in agent prompts.", "found_via": "arXiv search: semantic cache LLM", "cites_or_cited_by": [ "gim-2023-prompt-cache" ] }, { "id": "felicia-2026-stepshield", "title": "StepShield: When, Not Whether to Intervene on Rogue Agents", "authors": [ "Gloria Felicia", "Michael Eniolade", "Jinfeng He", "Zitha Sasindran", "Hemant Kumar", "Milan Hussain Angati", "Sandeep Bandarupalli" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.22136", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "early intervention detection", "cascaded detector" ], "cost_reduction_reported": "Cascaded HybridGuard reduces monitoring costs by 75%", "benchmarks": [ "StepShield (9,213 code agent trajectories)" ], "models_studied": [], "production_evidence": false, "summary": "Benchmark measuring when (not whether) rogue code-agent behavior is detected; LLM judges achieve 59% early intervention rate vs 26% for static analysis, and a cascaded detector cuts monitoring cost.", "key_idea": "Timeliness metrics and a cascaded monitor for agents.", "relevance_score": 3, "relevance_rationale": "Cost-aware monitoring cascade.", "found_via": "hf-papers search: step-level verifier web agent", "cites_or_cited_by": [] }, { "id": "feng-2026-dr-mas", "title": "Dr. MAS: Stable Reinforcement Learning for Multi-Agent LLM Systems", "authors": [ "Lang Feng", "Longtao Zheng", "Shuo He", "Fuxiang Zhang", "Bo An" ], "year": 2026, "venue_or_source": "NeurIPS 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.08847", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [], "cost_reduction_reported": "Heterogeneous agent-model assignment reported (in search summary) to cut latency 31.6% and API cost 41.8%; abstract states it remains effective under heterogeneous assignments while improving efficiency", "benchmarks": [ "multi-agent math reasoning", "multi-turn search" ], "models_studied": [ "Qwen2.5", "Qwen3" ], "production_evidence": false, "summary": "Dr. MAS diagnoses GRPO instability in multi-agent RL as a global normalization mismatch and fixes it with per-agent advantage normalization, plus a training framework with per-agent model serving. It supports heterogeneous agent-model assignments, e.g., smaller models for low-level agents.", "key_idea": "Per-agent advantage normalization for stable MAS RL, supporting heterogeneous models.", "relevance_score": 3, "relevance_rationale": "Training infrastructure for heterogeneous role models.", "found_via": "web search: heterogeneous MAS", "cites_or_cited_by": [] }, { "id": "furman-2026-replit-closing-the-loop-evals", "title": "Closing the loop: Evaluating and improving Replit Agent at scale", "authors": [ "Daniel Furman", "Peter Zhong", "Zhen Li", "Michele Catasta" ], "year": 2026, "venue_or_source": "Replit blog", "source_type": "blog", "url": "https://replit.com/blog/evaluating-and-improving-agent-at-scale", "code_url": null, "topic_clusters": [ "C5", "C7", "C13", "C14" ], "routing_granularity": "n/a", "learning_regime": "hybrid", "signals_used": [ "offline benchmark (ViBench) spec-conformance judge", "online A/B user behaviour, sentiment and cost", "production trace clustering", "human review" ], "cost_reduction_reported": null, "benchmarks": [ "ViBench", "Replit production A/B tests" ], "models_studied": [], "production_evidence": true, "summary": "Describes Replit Agent's evaluation loop: an offline benchmark (ViBench) of product-requirement documents drawn from anonymised production traces, graded by an eval agent that checks whether the built app meets the spec; concurrent online A/B tests for most agent-affecting changes including model swaps, tracking user continuation, sentiment and cost; and clustering of production traces to explain failures, all feeding back into evals and shipping decisions.", "key_idea": "Pair offline spec-grounded agent evals with live A/B tests and trace clustering to validate changes such as model swaps.", "relevance_score": 3, "relevance_rationale": "Production methodology for validating cost-saving model and harness changes; no cost numbers.", "found_via": "direct: replit.com/blog index", "cites_or_cited_by": [ "furman-2026-replit-free-the-models" ] }, { "id": "garg-2026-unsolvability-ceiling", "title": "Unsolvability Ceiling in Multi-LLM Routing: An Empirical Study of Evaluation Artifacts", "authors": [ "Saloni Garg", "Amit Sagtani" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07395", "code_url": null, "topic_clusters": [ "C1", "C7", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "judge vs exact-match labels" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU", "MedQA", "HumanEval", "MBPP", "Alpaca", "ShareGPT" ], "models_studied": [ "Gemma 4", "Llama 3.1" ], "production_evidence": false, "summary": "Shows much of the reported 'unsolvability' in routing studies comes from evaluation artifacts such as judge verbosity bias, truncation and format mismatch, which also distort router labels and cause collapse to the majority class.", "key_idea": "Evaluation artifacts corrupt router training signals.", "relevance_score": 3, "relevance_rationale": "Label quality caution.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "gharzeddine-2026-cyclic-subtask-graphs", "title": "Complete Cyclic Subtask Graphs for Tool-Using LLM Agents: Flexibility, Cost, and Bottlenecks in Long-Horizon Workflows", "authors": [ "Luay Gharzeddine", "Samer Saab Jr" ], "year": 2026, "venue_or_source": "TMLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2604.22820", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "token-cost accounting" ], "cost_reduction_reported": null, "benchmarks": [ "TextCraft", "ALFWorld", "Finance-Agent" ], "models_studied": [], "production_evidence": false, "summary": "Studies fully connected cyclic subtask graphs with a routing agent against ReAct and dependency-directed workflows, reporting that flexible backtracking helps in partially observable recovery settings but adds overhead in prerequisite-chain tasks. Offers a workflow-signature matrix for when the cost is worth it.", "key_idea": "When flexible subtask revisitation is worth its routing and token cost.", "relevance_score": 3, "relevance_rationale": "Workflow-structure cost trade-offs.", "found_via": "OpenAlex search: trajectory routing LLM agents cost", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "ghosh-2026-eval-costs-bottleneck", "title": "AI evals are becoming the new compute bottleneck", "authors": [ "Avijit Ghosh", "Yifan Mai", "Georgia Channing", "Leshem Choshen" ], "year": 2026, "venue_or_source": "Hugging Face blog (EvalEval Coalition)", "source_type": "blog", "url": "https://huggingface.co/blog/evaleval/eval-costs-bottleneck", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "evaluation cost" ], "cost_reduction_reported": "Reports HAL ~$40K for 21,730 rollouts; one frontier GAIA run $2,829 before caching; 33x cost spread across scaffolds on identical tasks", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Argues evaluation cost is a growing bottleneck, compiling costs for agent benchmarks and noting that repeated runs for statistical reliability multiply spend (e.g., 8 runs would take HAL to ~$320K).", "key_idea": "Reliable agent evaluation is expensive; scaffold drives cost.", "relevance_score": 3, "relevance_rationale": "Informs budgeting for cost-aware evaluation of configurations.", "found_via": "web search: AI Agents That Matter cost accuracy Pareto", "cites_or_cited_by": [ "kapoor-2025-hal-holistic-agent-leaderboard" ] }, { "id": "gonzalez-pumariega-2026-cua-reliability", "title": "On the Reliability of Computer Use Agents", "authors": [ "Gonzalo Gonzalez-Pumariega", "Saaket Agashe", "Jiachen Yang", "Ang Li", "Xin Eric Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.17849", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "pass^k", "paired statistical tests" ], "cost_reduction_reported": null, "benchmarks": [ "OSWorld" ], "models_studied": [ "computer-use agents" ], "production_evidence": false, "summary": "Studies why computer-use agents that succeed once fail on repeats, attributing unreliability to execution stochasticity, task ambiguity and behavior variability, using pass^k and paired tests over repeated OSWorld runs.", "key_idea": "Repeated-execution evaluation for computer-use agents.", "relevance_score": 3, "relevance_rationale": "Reliability methodology relevant to validating model swaps.", "found_via": "web search: agent reliability metrics", "cites_or_cited_by": [] }, { "id": "google-vertex-data-driven-prompt-optimizer", "title": "Data-driven prompt optimizer (Vertex AI / Gemini Enterprise Agent Platform docs)", "authors": [ "Google Cloud" ], "year": 2026, "venue_or_source": "Google Cloud documentation", "source_type": "docs", "url": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/prompts/data-driven-optimizer", "code_url": null, "topic_clusters": [ "C4", "C8", "C14" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "evaluation metrics", "source-model reference outputs" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Gemini models" ], "production_evidence": false, "summary": "A managed optimiser that iteratively rewrites system instructions and selects few-shot examples to maximise chosen evaluation metrics for a target model. For model upgrades or migrations, setting source_model runs the sample prompts on the old model to produce reference responses, so prompts can be re-tuned for a cheaper or newer target without hand-labelled ground truth.", "key_idea": "Use the incumbent model's outputs as labels to re-optimise prompts for a replacement model.", "relevance_score": 3, "relevance_rationale": "Productised support for migrating workloads to cheaper models; no numbers.", "found_via": "direct: docs.cloud.google.com (searching for Vertex model optimizer)", "cites_or_cited_by": [ "notdiamond-2025-prompt-adaptation-weaker-models" ] }, { "id": "gou-2026-dysco-sparse-consensus", "title": "Dynamic Trust-Aware Sparse Communication Topology for LLM-Based Multi-Agent Consensus", "authors": [ "Wanshuang Gou", "Zihan Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.01828", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "agent reliability", "answer divergence", "task relevance" ], "cost_reduction_reported": null, "benchmarks": [ "math reasoning", "logical reasoning", "factual QA" ], "models_studied": [], "production_evidence": false, "summary": "DySCo picks a few high-value communication edges per debate round under a budget, weights answers by dynamic trust and stops early when consensus stabilizes, replacing all-to-all broadcasting.", "key_idea": "Budgeted dynamic edge selection and early stopping in debate.", "relevance_score": 3, "relevance_rationale": "MAS communication cost lever.", "found_via": "web search: sparse debate", "cites_or_cited_by": [] }, { "id": "gu-2026-peek", "title": "PEEK: Context Map as an Orientation Cache for Long-Context LLM Agents", "authors": [ "Zhuohan Gu", "Qizheng Zhang", "Omar Khattab", "Samuel Madden" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.19932", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "context map" ], "cost_reduction_reported": "1.7-5.8x lower cost than ACE with fewer iterations; 1.4x lower cost than ACE on context learning", "benchmarks": [], "models_studied": [ "OpenAI Codex" ], "production_evidence": false, "summary": "Builds a reusable context map that acts as an orientation cache for agents repeatedly working over the same large external context, improving accuracy and cost compared with ACE.", "key_idea": "Cache an orientation map of recurring external contexts.", "relevance_score": 3, "relevance_rationale": "Reuse across recurring contexts cuts agent cost.", "found_via": "snowball: cited-by zhang-2025-agentic-plan-caching", "cites_or_cited_by": [ "zhang-2025-ace" ] }, { "id": "guo-2026-cascade-deployment-time-learning", "title": "CASCADE: Case-Based Continual Adaptation for Large Language Models During Deployment", "authors": [ "Siyuan Guo", "Yali Du", "Hechang Chen", "Yi Chang", "Jun Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06702", "code_url": null, "topic_clusters": [ "C5", "C10" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "episodic case memory", "task feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defines deployment-time learning, in which agents improve from experience without changing weights, and implements it as CASCADE, an evolving episodic case memory whose reuse is framed as a contextual bandit with no-regret guarantees.", "key_idea": "Frame experience reuse during deployment as a contextual bandit.", "relevance_score": 3, "relevance_rationale": "Online learning from live traffic at the agent level; not cost-targeted.", "found_via": "query: abs:\"bandit\" AND abs:\"cascade\" AND abs:\"language model\"", "cites_or_cited_by": [] }, { "id": "hao-2026-racer", "title": "RACER: Risk-Aware Calibrated Efficient Routing for Large Language Models", "authors": [ "Sai Hao", "Hao Zeng", "Hongxin Wei", "Bingyi Jing" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.06616", "code_url": null, "topic_clusters": [ "C1", "C7", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "router scores", "conformal calibration" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RACER extends base routers to output model sets with distribution-free misrouting risk control, calibrating thresholds with finite-sample bounds and allowing abstention.", "key_idea": "Conformal set-valued routing with risk guarantees.", "relevance_score": 3, "relevance_rationale": "Theory for safe routing.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "huang-2026-constrained-bandits-time-varying", "title": "Online LLM Selection via Constrained Bandits with Time-Varying Demand", "authors": [ "Y Huang", "Qingsong Liu", "Jie Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.17489", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "reward", "cost", "latency", "demand prediction" ], "cost_reduction_reported": null, "benchmarks": [ "synthetic workloads" ], "models_studied": [], "production_evidence": false, "summary": "Online LLM selection in edge-cloud systems under hard packing and soft covering constraints with time-varying demand; a confidence-bound algorithm with demand prediction gets sublinear regret and violation.", "key_idea": "Constrained bandits with demand forecasts for LLM selection.", "relevance_score": 3, "relevance_rationale": "Theory-heavy; synthetic evaluation only.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "dai-2024-c2mab-v" ] }, { "id": "huang-2026-evolverouter", "title": "EvolveRouter: Co-Evolving Routing and Prompt for Multi-Agent Question Answering", "authors": [ "Jiatan Huang", "Zheyuan Zhang", "Kaiwen Shi", "Yanfang Ye", "Chuxu Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.05149", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "graph-based routing", "reliability" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "EvolveRouter co-evolves graph-based query routing among agents with targeted instruction refinement of the agents, and adapts the number of collaborating agents per query at inference.", "key_idea": "Co-evolve routing and agent prompts; adapt team size per query.", "relevance_score": 3, "relevance_rationale": "Routing among agents with adaptive team size.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "chen-2024-optima", "yue-2025-masrouter", "zhang-2024-g-designer" ] }, { "id": "huang-2026-prefixguard", "title": "PrefixGuard: Online Failure Warning and Trace-Grounded Diagnosis for LLM Agents", "authors": [ "Xinmiao Huang", "Jinwei Hu", "Qisong He", "Rajarshi Roy", "Changshun Wu", "Yi Dong", "Xiaowei Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06455", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "learned trace events", "GRU risk", "DFA state" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Predicts horizon-specific failure risk from execution prefixes of LLM agent traces with a GRU over learned events, and organizes event histories into a finite automaton for diagnosis. Connects online failure warning with compact symbolic explanations.", "key_idea": "Online failure prediction from agent trace prefixes with symbolic diagnosis.", "relevance_score": 3, "relevance_rationale": "Mid-run success prediction that could trigger escalation or early stopping.", "found_via": "query: abs:\"predict\" AND abs:\"trajectory\" AND abs:\"failure\" AND abs:\"LLM agent\"", "cites_or_cited_by": [] }, { "id": "itkin-2026-budgeted-strong-correctors", "title": "How Much Does Correctness Cost? Budgeted Placement of Strong Correctors in a Weak Multi-Agent Swarm", "authors": [ "Igor Itkin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.09765", "code_url": null, "topic_clusters": [ "C3", "C15" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "graph consensus coherence" ], "cost_reduction_reported": null, "benchmarks": [ "math verification", "code tracing" ], "models_studied": [ "Qwen3 0.6B-32B" ], "production_evidence": false, "summary": "Models a swarm of cheap unreliable agents steered by a few expensive strong correctors placed on a consensus graph, proves the objective is submodular so greedy placement is near-optimal, and derives the budget-correctness frontier. Whether to buy a few strong or many medium correctors depends on the task's cost-quality curvature.", "key_idea": "Theory for where to place expensive models in a cheap agent swarm.", "relevance_score": 3, "relevance_rationale": "Formal treatment of heterogeneous model placement under budget.", "found_via": "arXiv title search: multi-agent cost", "cites_or_cited_by": [] }, { "id": "jain-2026-calm-slm-fleet", "title": "CALM: A Self-Adaptive Orchestration Approach for QoS-Aware Routing in Small Language Model based Systems", "authors": [ "Hemang Jain", "Divyansh Pandey", "Karthik Vaidhyanathan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.03632", "code_url": null, "topic_clusters": [ "C1", "C3", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "QoS metrics", "latency", "energy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CALM is a MAPE-K self-adaptive loop that monitors queries and QoS metrics of a fleet of specialized SLMs, then routes each query to the most suitable one and adapts continuously as workloads drift.", "key_idea": "Self-adaptive orchestration of a fleet of SLMs.", "relevance_score": 3, "relevance_rationale": "SLM-fleet routing from a software-engineering view; query-level.", "found_via": "arXiv query ti:orchestrator AND abs:\"small\" AND abs:\"cost\" AND abs:LLMs", "cites_or_cited_by": [] }, { "id": "jeon-2026-lragent", "title": "LRAgent: Efficient KV Cache Sharing for Multi-LoRA LLM Agents", "authors": [ "Hyesung Jeon", "Hyeongju Ha", "Jae-Joon Kim" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2602.01053", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "shared base cache", "low-rank adapter cache" ], "cost_reduction_reported": "throughput and TTFT close to fully shared caching with accuracy near non-shared caching on agentic QA benchmarks", "benchmarks": [ "agentic question-answering benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "For multi-agent systems built from LoRA adapters on one base model, splits each agent's KV cache into a shared base component and a low-rank adapter component, shares the base across agents and uses a custom kernel to reconstruct adapter contributions.", "key_idea": "Share base-model KV across LoRA agents; store only low-rank deltas.", "relevance_score": 3, "relevance_rationale": "Enables cheap multi-specialist agent systems on one base model.", "found_via": "multi-agent LLM KV cache sharing reuse across agents", "cites_or_cited_by": [ "wang-2026-forkkv", "ye-2025-kvcomm" ] }, { "id": "jiang-2026-cera-moa", "title": "CERA-MoA: Co-Evolving Routing Mechanisms with Continually Learning LLM Agents", "authors": [ "Jiaxuan Jiang", "Liyuan He", "Zhixuan Fang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.18779", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "mid-layer hidden-state familiarity estimate" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CERA-MoA co-evolves a router and continually fine-tuned agents in a Mixture-of-Agents, using hidden-state familiarity scores to activate a minimal agent subset per query and allocating training data by competence.", "key_idea": "Co-train router and agents; activate minimal subsets.", "relevance_score": 3, "relevance_rationale": "Sparse MoA activation with co-evolving specialists.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "ke-2026-mas-orchestra", "wang-2026-oi-mas", "zhang-2026-evoroute" ] }, { "id": "jiang-2026-moae", "title": "MOAE: Multi-Objective Agent Evolution with Pareto-Preserving Search", "authors": [ "Hengle Jiang", "Qijun Cai", "Ziying Luo", "Ke Tang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.05992", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task performance", "trajectory quality", "safety" ], "cost_reduction_reported": null, "benchmarks": [ "TravelPlanner", "AgentDojo" ], "models_studied": [], "production_evidence": false, "summary": "MOAE performs Pareto-preserving in-context evolutionary search over agent rollouts, keeping an archive of non-dominated candidates and applying constraint-aware selection only at deployment, so no scalarization is fixed during search.", "key_idea": "Pareto-archive evolutionary agent refinement without fixed scalarization.", "relevance_score": 3, "relevance_rationale": "Multi-objective agent optimization method; cost not a primary objective in abstract.", "found_via": "search: multi-objective agentic workflow optimization cost accuracy Pareto arXiv", "cites_or_cited_by": [] }, { "id": "jiang-2026-reliability-at-k", "title": "Beyond Pass@k: Measuring Reliability and Security of Agentic Code Generation", "authors": [ "Jiajun Jiang", "Sharon Zheng", "Natan Vidra", "Spurthi Setty" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.14711", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "independent rollouts" ], "cost_reduction_reported": null, "benchmarks": [ "synthetic multi-rollout benchmark", "coding-agent benchmarks" ], "models_studied": [ "coding agents" ], "production_evidence": false, "summary": "Shows coding-agent leaderboards misapply the pass@k estimator by counting unit tests rather than independent rollouts, and proposes reliability@k; the misapplied metric inflates scores substantially in a synthetic benchmark.", "key_idea": "Correct pass@k over independent rollouts.", "relevance_score": 3, "relevance_rationale": "Metric correctness for reliability claims.", "found_via": "web search: agent reliability metrics", "cites_or_cited_by": [] }, { "id": "juneja-2026-cost-ordered-feasibility", "title": "Cost-Ordered Feasibility for Multi-Armed Bandits with Cost Subsidy", "authors": [ "Ishank Juneja", "Carlee Joe-Wong", "Osman Yagan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07171", "code_url": null, "topic_clusters": [ "C5", "C15" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "reward feedback", "known arm costs" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies multi-armed bandits with cost subsidy, where the goal is to minimize cost while keeping reward within a tolerance of the unknown best arm. Derives instance-dependent lower bounds and the Cost-Ordered Feasibility algorithm, which pools samples across arms to test whether a cheap arm is good enough.", "key_idea": "Bandit algorithm for picking the cheapest arm that is near-best in quality.", "relevance_score": 3, "relevance_rationale": "Exactly the objective of cheap-model-first online routing; generic bandit theory.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "kang-2026-thunderagent", "title": "ThunderAgent: A Simple, Fast and Program-Aware Agentic Inference System", "authors": [ "Hao Kang", "Ziyang Li", "Weili Xu", "Xinyu Yang", "Yinfang Chen", "Junxiong Wang", "Beidi Chen", "Tushar Krishna", "Chenfeng Xu", "Simran Arora" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.13692", "code_url": "https://github.com/Agentic-Kinetics/ThunderAgent", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "program-level state" ], "cost_reduction_reported": "1.5-3.6x serving throughput, 1.8-3.9x RL rollout throughput and up to 4.2x disk memory savings vs state-of-the-art inference systems", "benchmarks": [ "coding", "routing", "scientific discovery agents" ], "models_studied": [], "production_evidence": false, "summary": "Program-aware agentic inference system that manages memory imbalance and prepares environments asynchronously, for both serving and RL rollouts.", "key_idea": "Program-aware scheduling for agent serving and rollouts.", "relevance_score": 3, "relevance_rationale": "Agent serving throughput.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "luo-2025-autellix" ] }, { "id": "katz-2026-cursor-dynamic-context-discovery", "title": "Dynamic context discovery", "authors": [ "Jediah Katz" ], "year": 2026, "venue_or_source": "Cursor blog", "source_type": "blog", "url": "https://cursor.com/blog/dynamic-context-discovery", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "A/B test: loading only needed MCP tools reduced total agent tokens by 46.9% in runs that called an MCP tool (statistically significant, high variance by number of MCPs installed; vendor-reported)", "benchmarks": [ "Cursor production A/B test" ], "models_studied": [], "production_evidence": true, "summary": "Describes providing less context upfront and letting the agent pull what it needs: long tool outputs written to files, chat history referenced during summarisation, Agent Skills, lazily loaded MCP tools and terminal sessions as files. Reports an A/B test of lazy MCP tool loading.", "key_idea": "Replace static context with files the agent discovers on demand.", "relevance_score": 3, "relevance_rationale": "Production A/B evidence for a context-efficiency lever.", "found_via": "direct: cursor.com/blog index", "cites_or_cited_by": [ "katz-2026-cursor-token-efficiency" ] }, { "id": "khailo-2026-keepalive-economics", "title": "Keeping the Cache Warm Pays: Keepalive Economics for Agentic Workloads", "authors": [ "Maxim Khailo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.19214", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "idle gap duration", "cache TTL", "read/write price ratio" ], "cost_reduction_reported": "a client-side keepalive keeps provider prefixes warm through tool/approval pauses; breaks even against re-prefill at idle gaps of ~46 min (Anthropic) and ~36 min (OpenAI, DeepSeek)", "benchmarks": [ "measurements on Anthropic, OpenAI, Google and DeepSeek APIs" ], "models_studied": [], "production_evidence": false, "summary": "Agents pause for tools or human approval long enough for provider caches to evict their prefix, forcing full prefill on resume. The paper measures that periodically replaying the prefix during pauses keeps it cached and derives when this pays off, and argues providers will start metering cache residency.", "key_idea": "Keepalive pings preserve prompt caches across agent pauses.", "relevance_score": 3, "relevance_rationale": "Concrete cost optimization for agent loops with long tool waits.", "found_via": "\"Don't Break the Cache\" prompt caching agentic tasks evaluation", "cites_or_cited_by": [ "anthropic-2024-prompt-caching", "lumer-2026-dont-break-the-cache" ] }, { "id": "khanal-2026-beyond-pass1", "title": "Beyond pass@1: A Reliability Science Framework for Long-Horizon LLM Agents", "authors": [ "Aaditya Khanal", "Yangyang Tao", "Junxiu Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.29231", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "repeated-attempt success" ], "cost_reduction_reported": null, "benchmarks": [ "396-task benchmark across four duration buckets" ], "models_studied": [ "10 models" ], "production_evidence": false, "summary": "Shows that capability (single-attempt success) and reliability (consistent repeated success) diverge as task duration grows, and proposes reliability decay curves, variance amplification, graceful degradation and meltdown-onset metrics over 23,392 episodes.", "key_idea": "Reliability decays with horizon length in ways pass@1 misses.", "relevance_score": 3, "relevance_rationale": "Reliability measurement for long-horizon agents.", "found_via": "web search: agent reliability metrics", "cites_or_cited_by": [] }, { "id": "kim-2026-agenticcache", "title": "AgenticCache: Cache-Driven Asynchronous Planning for Embodied AI Agents", "authors": [ "Hojoon Kim", "Yuheng Wu", "Thierry Tambe" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.24039", "code_url": "https://github.com/hojoonleokim/MLSys26_AgenticCache", "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "cached plan transitions", "asynchronous LLM validation" ], "cost_reduction_reported": "22% average task-success improvement across 12 configurations, 65% lower simulation latency and 50% lower token usage", "benchmarks": [ "four multi-agent embodied benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Embodied agents reuse cached plan transitions instead of calling the LLM planner at every step, while a background updater asynchronously asks the LLM to validate and refine cache entries.", "key_idea": "Cache-driven asynchronous planning takes the LLM off the critical path.", "relevance_score": 3, "relevance_rationale": "Plan-caching evidence with token savings, in embodied multi-agent settings.", "found_via": "snowball: cited-by zhang-2025-agentic-plan-caching", "cites_or_cited_by": [ "zhang-2025-agentic-plan-caching" ] }, { "id": "koh-2026-graft", "title": "Global Optimization and Inference-Time Region Grafting for Agentic Workflows", "authors": [ "Donghyeok Koh", "Gyuwan Kim", "Jinyeong Bak", "Seung-Hoon Na", "Tao Yang", "Haneol Jang", "Cheoneum Park" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.02353", "code_url": null, "topic_clusters": [ "C4", "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "label-free execution-quality signals" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "GRAFT keeps a globally optimized workflow but replaces selected regions per input at inference time using label-free quality signals, improving over MaAS by 3.85 points on average.", "key_idea": "Inference-time local region replacement in a globally optimized workflow.", "relevance_score": 3, "relevance_rationale": "Runtime adaptation of optimized workflows using label-free signals.", "found_via": "search: ScoreFlow MaAS agentic supernet arXiv", "cites_or_cited_by": [ "zhang-2025-maas-agentic-supernet" ] }, { "id": "kondapalli-2026-slms-as-multi-agent-routers", "title": "SLMs as Multi-Agent Routers: A Progressive SFT and Reinforcement Learning Approach", "authors": [ "Gayathri V Kondapalli", "Alexander Ng", "Hirsh Pithadia", "Rahul Monish", "Harvey Yorke", "Amir Kayhani" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.00030", "code_url": null, "topic_clusters": [ "C1", "C3", "C8" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "retrieval relevance reward", "topic alignment" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Trains a small language model with SFT then RL to select which specialized retrieval agent to call and to generate its tool parameters, rewarding retrieval relevance rather than inferred intent. The SLM router learns which agents reliably return relevant results.", "key_idea": "A small RL-trained model routes among retrieval agents.", "relevance_score": 3, "relevance_rationale": "SLM-based routing among agents.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "kovacs-2026-squeez", "title": "Squeez: Task-Conditioned Tool-Output Pruning for Coding Agents", "authors": [ "Ádám Kovács" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.04979", "code_url": null, "topic_clusters": [ "C8", "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "task-conditioned relevance of tool-output blocks" ], "cost_reduction_reported": "removes 92% of tool-output input tokens at 0.86 recall / 0.80 F1, beating zero-shot Qwen 3.5 35B-A3B by 11 recall points", "benchmarks": [ "SWE-bench-derived tool-output pruning benchmark (11,477 examples)" ], "models_studied": [ "Qwen 3.5 2B (LoRA)", "Qwen 3.5 35B-A3B" ], "production_evidence": false, "summary": "Fine-tunes a 2B model to prune tool outputs for coding agents down to the evidence the agent should inspect next, and releases a benchmark built from SWE-bench interactions.", "key_idea": "A tiny fine-tuned model truncates tool outputs for the frontier agent.", "relevance_score": 3, "relevance_rationale": "Tool-output truncation by a small specialized model.", "found_via": "tool output observation compression LLM agent tokens arXiv", "cites_or_cited_by": [ "wang-2026-swe-pruner" ] }, { "id": "kulkarni-2026-cache-eviction-study", "title": "Which Eviction Policy Should an LLM Cache Use? A Systematic Study Across Workloads, Capacities, and Encoders", "authors": [ "Yash Kulkarni", "Shubham Harkare", "Arvind Yogesh Suresh Babu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.20280", "code_url": null, "topic_clusters": [ "C10", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "embedding similarity", "eviction policy" ], "cost_reduction_reported": "raw semantic-cache hit rates of 51-60% on LMSYS and QQP fall to quality-adjusted rates of 1.1-2.2% when only answer-substitutable hits are counted", "benchmarks": [ "LMSYS", "QQP" ], "models_studied": [ "MiniLM and other embedding models", "cross-encoders" ], "production_evidence": false, "summary": "Systematic study of eviction policies for semantic LLM caches across workloads and capacities. Its most important finding is an audit showing that most \"hits\" at common thresholds are not answer-substitutable, so reported hit rates hugely overstate useful savings; LFU is a strong default.", "key_idea": "Most semantic-cache hits at standard thresholds are not valid substitutes.", "relevance_score": 3, "relevance_rationale": "Critical evidence on semantic-cache false hits and overstated savings.", "found_via": "arXiv search: semantic cache LLM", "cites_or_cited_by": [ "schroeder-2025-vcache" ] }, { "id": "kumar-2026-cooperative-profiles", "title": "Cooperative Profiles Predict Multi-Agent LLM Team Performance in AI for Science Workflows", "authors": [ "Shivani Kumar", "Adarsh Bharathwaj", "David Jurgens" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.20658", "code_url": null, "topic_clusters": [ "C6", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "behavioral-economics game profiles" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "35 open-weight LLMs" ], "production_evidence": false, "summary": "Benchmarks 35 open-weight LLMs on six behavioral economics games and shows that the resulting cooperative profiles predict how well teams of those models perform on AI-for-science multi-agent tasks.", "key_idea": "Cheap game-based profiles predict multi-agent team performance.", "relevance_score": 3, "relevance_rationale": "Performance prediction for choosing models in multi-agent teams without running full tasks.", "found_via": "query: predict whether LLM agent will solve task", "cites_or_cited_by": [] }, { "id": "kumar-2026-craft", "title": "CRAFT: Cost-aware Refinement And Front-aware Tuning of Prompts", "authors": [ "Shanu Kumar", "Shubhanshu Khandelwal", "Akhila Yesantarao Venkata", "Parag Agrawal", "Yova Kementchedjhieva", "Manish Gupta" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.04661", "code_url": null, "topic_clusters": [ "C4", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "accuracy", "prompt-token cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CRAFT searches the Pareto front of accuracy and prompt-token cost, spending scarce validation calls on candidates near the optimistic front and keeping a spread population via NSGA-II, avoiding the 'scalarization collapse' of weighted-sum objectives.", "key_idea": "Pareto-front prompt optimization with validation-budget-aware acquisition.", "relevance_score": 3, "relevance_rationale": "Cost-aware prompt optimization; complements model selection.", "found_via": "search: cost-aware prompt optimization LLM token cost Pareto arXiv", "cites_or_cited_by": [ "zehle-2025-capo" ] }, { "id": "kumar-2026-tvcache", "title": "TVCACHE: A Stateful Tool-Value Cache for Post-Training LLM Agents", "authors": [ "Abhishek Vijaya Kumar", "Bhaskar Kataria", "Byungsoo Oh", "Emaad Manzoor", "Rachee Singh" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.10986", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "longest-prefix match on tool-call history" ], "cost_reduction_reported": "cache hit rates up to 70% and median tool-call execution time reduced up to 6.9x, with no degradation in post-training reward", "benchmarks": [ "terminal tasks", "SQL generation", "video understanding" ], "models_studied": [], "production_evidence": false, "summary": "A stateful tool-value cache for RL post-training of agents: tool outputs are reused only when the full sequence of prior tool calls matches, guaranteeing identical environment state. It shows why naive tool-output caching is unsafe.", "key_idea": "Cache tool results keyed by the whole tool-call prefix to keep state correct.", "relevance_score": 3, "relevance_rationale": "Correct tool-output caching; relevant for agent training cost and for state-aware caching design.", "found_via": "semantic cache LLM agents tool calls caching arXiv", "cites_or_cited_by": [] }, { "id": "laddha-2026-slmjury", "title": "SLMJury: Can Small Language Models Judge as Well as Large Ones?", "authors": [ "Anish Laddha", "Nitesh Pradhan", "Gaurav Srivastava" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.07810", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "small judge" ], "cost_reduction_reported": null, "benchmarks": [ "8 closed-ended tasks", "SummEval", "MT-Bench" ], "models_studied": [ "16 SLMs 0.6B-14B" ], "production_evidence": false, "summary": "Benchmarks 16 small language models as judges; short 10-token verdicts often match extended reasoning on math judging, with domain-dependent effects.", "key_idea": "When small judges suffice.", "relevance_score": 3, "relevance_rationale": "Guides cheap judge selection.", "found_via": "hf-papers search: small judge model evaluation", "cites_or_cited_by": [] }, { "id": "lai-2026-dars", "title": "From Sampled Outcomes to Capability Distributions: Rethinking Supervision for LLM Routing", "authors": [ "Guannan Lai", "Haoran Hu", "Long Chen", "Zhenguo Li", "Han-Jia Ye" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.06924", "code_url": null, "topic_clusters": [ "C1", "C6" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "repeated sampling", "query rewrites", "variance" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "DARS builds routing supervision from repeated observations over rewrites and decoding, summarizing expected quality, cost and variability, which improves routing over single-sample labels.", "key_idea": "Distribution-aware routing labels.", "relevance_score": 3, "relevance_rationale": "Label construction.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "lai-2026-routefm", "title": "Pretrain Once, Route Anywhere: Towards a Foundation Model for LLM Routing", "authors": [ "Guannan Lai", "Han-Jia Ye" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.37362", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "behavioral context of anonymous models", "episodic pretraining" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RouteFM pretrains a reusable routing model across many environments so it characterizes anonymous candidates from behavioral context and adapts to new pools through context alone.", "key_idea": "Foundation model for routing.", "relevance_score": 3, "relevance_rationale": "Very recent.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "lan-2026-searchswarm", "title": "SearchSwarm: Towards Delegation Intelligence in Agentic LLMs for Long-Horizon Deep Research", "authors": [ "Xiaochong Lan", "Quan Chen", "Kun Tao", "Xinyu Tang", "Tianshu Wang", "Qianggang Cao", "Xinyu Kong", "Zujie Wen", "Zhiqiang Zhang", "Jun Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.09730", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "harness-guided delegation trajectories" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "SearchSwarm trains a main agent to decompose deep-research tasks and decide when and what to delegate to subagents that return summarized results, using harness-guided trajectories as SFT data.", "key_idea": "Learn delegation intelligence for main-agent/subagent splits.", "relevance_score": 3, "relevance_rationale": "Learned delegation to subagents; cost framed as context budget.", "found_via": "arXiv query ti:delegation AND ti:llm", "cites_or_cited_by": [] }, { "id": "lenton-2026-distilling-repeated-work-into-code", "title": "Distilling repeated work into code", "authors": [ "Dan Lenton" ], "year": 2026, "venue_or_source": "Unify blog", "source_type": "blog", "url": "https://unify.ai/blog/distilling-repeated-work-into-code", "code_url": null, "topic_clusters": [ "C8", "C10", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "execution exceptions" ], "cost_reduction_reported": "Hourly triage task, 8 runs x 12 inquiries: first run 6 model calls and ~251,000 tokens; each later run one call of ~645 tokens (~390x fewer), all 96 inquiries correct; one-off setup and review cost 1.2M tokens (vendor-reported)", "benchmarks": [ "internal hourly triage task" ], "models_studied": [], "production_evidence": false, "summary": "Argues that agents repeating a task should distil it into a stored function with fixed control flow and narrow model calls only where judgement is needed, rather than reasoning afresh each time or freezing a brittle script. The function is repaired in place by a bounded loop that sees the exception, the source and a read-only probe of the external interface.", "key_idea": "Compile recurring agent trajectories into code with targeted model calls and self-repair.", "relevance_score": 3, "relevance_rationale": "Concrete cost lever for recurring agent workloads with measured token reduction.", "found_via": "direct: unify.ai/blog index", "cites_or_cited_by": [ "lenton-2026-continual-arc" ] }, { "id": "li-2026-bavt-budget-aware-value-tree", "title": "Spend Less, Reason Better: Budget-Aware Value Tree Search for LLM Agents", "authors": [ "Yushu Li", "Wenlong Deng", "Jiajin Li", "Xiaoxiao Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.12634", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "step-level value estimate", "remaining budget ratio" ], "cost_reduction_reported": "Under strict low budgets, surpasses baseline performance obtained at 4x the resource allocation", "benchmarks": [ "four multi-hop QA benchmarks" ], "models_studied": [ "two model families" ], "production_evidence": false, "summary": "BAVT models multi-hop agent reasoning as a search tree guided by step-level residual value estimates, with a budget-conditioned node-selection rule that shifts from exploration to exploitation as budget depletes. It prunes redundant tool calls and beats parallel sampling at a quarter of the budget.", "key_idea": "Budget-conditioned tree search pruning low-value agent steps.", "relevance_score": 3, "relevance_rationale": "Per-step compute allocation within agents.", "found_via": "OpenAlex search: agent pruning", "cites_or_cited_by": [] }, { "id": "li-2026-coca-confidence-before-answering", "title": "Confidence Before Answering: A Paradigm Shift for Efficient LLM Uncertainty Estimation", "authors": [ "Changcheng Li", "Jiancan Wu", "Hengheng Zhang", "Zhengsu Chen", "Guo An", "Junxiang Qiu", "Xiang Wang", "Qi Tian" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.05881", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "query", "learning_regime": "RL-online", "signals_used": [ "verbalized pre-answer confidence" ], "cost_reduction_reported": null, "benchmarks": [ "math", "code", "factual QA" ], "models_studied": [], "production_evidence": false, "summary": "CoCA trains models with GRPO to output a calibrated confidence before answering, jointly optimizing calibration and accuracy.", "key_idea": "Confidence-first outputs trained with RL.", "relevance_score": 3, "relevance_rationale": "Pre-answer confidence can drive routing/escalation.", "found_via": "snowball: cited-by pacchiardi-2025-predictaboard", "cites_or_cited_by": [] }, { "id": "li-2026-e2-explainer-topology", "title": "Discovering Efficient and Explainable Communication Topologies for LLM-based Multi-Agent Systems via Causal Inference", "authors": [ "Junzhi Li", "Peng He", "Qirui Ji", "Wei Wang", "Lixiang Liu", "Chuxiong Sun" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.12921", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "Granger-style edge masking effects" ], "cost_reduction_reported": null, "benchmarks": [ "reasoning and coding benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "E2-Explainer attributes MAS performance to compact communication subgraphs by measuring how masking each channel changes outcomes, then distills this into an amortized explainer. The identified subgraphs can be run directly to prune redundant edges.", "key_idea": "Causal attribution of communication edges enables pruning.", "relevance_score": 3, "relevance_rationale": "Explainable MAS pruning.", "found_via": "arXiv search: cost-efficient LLM agent model selection", "cites_or_cited_by": [] }, { "id": "li-2026-morse", "title": "MoRSE: Task-Oriented Multi-Agent System with Mixture of Role-Subtask Experts", "authors": [ "Peiwen Li", "Shiyang Zhang", "Yangtian Zhang", "Sizhuang He", "David van Dijk", "Rex Ying" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.09251", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "prototype-based semantic router" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MoRSE decomposes tasks into subtask DAGs with explicit (role, subtask) assignments and adds a dynamic mixture of LoRA experts per (role, subtask) on a shared LLM, with a semantic router and hierarchical RL. It gives parameter-level specialization cheaply without separate full models.", "key_idea": "Mixture of role-subtask LoRA experts on a shared backbone.", "relevance_score": 3, "relevance_rationale": "Cheap role specialization alternative to heterogeneous models.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [] }, { "id": "li-2026-same-model-not-same-service", "title": "When Is the Same Model Not the Same Service? A Measurement Study of Hosted Open-Weight LLM APIs", "authors": [ "Haorui Li", "Zhenghui He", "Xuanzi Liu", "Yang Xu", "Dongsheng Liu", "Jiakang Ma", "Lupan Wu", "Yangjie Wu", "Xiongchao Tang", "Tianhui Shi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.02821", "code_url": "https://github.com/haoruilee/llm_api_measurement_study", "topic_clusters": [ "C5", "C13", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "request logs", "latency measurements", "pricing snapshots" ], "cost_reduction_reported": "In two counterfactuals, routing lowers Qwen3-32B cost by 37.8% and raises DeepSeek-V3.2 throughput ~90% relative to direct official access", "benchmarks": [], "models_studied": [ "Qwen3-32B", "DeepSeek-V3.2" ], "production_evidence": true, "summary": "A measurement study of hosted open-weight LLM APIs using sampled request logs, provider metadata and continuous latency measurements from a routing platform in Q4 2025. Shows that the same model behaves as different, time-varying services across providers and that routing over provider-model-task-time tuples can lower cost.", "key_idea": "Hosted models are time-varying services; routing must track provider-level drift.", "relevance_score": 3, "relevance_rationale": "Production-log evidence on non-stationarity of the action set a router chooses from.", "found_via": "query: abs:\"logged\" AND abs:\"counterfactual\" AND abs:\"LLM\"", "cites_or_cited_by": [] }, { "id": "li-2026-test-time-scaling-agents", "title": "Evaluating Test-Time Scaling of General LLM Agents", "authors": [ "Xiaochuan Li", "Ryan Ming", "Pranav Setlur", "Abhijay Paladugu", "Andy Tang", "Hao Kang", "Shuai Shao", "Rong Jin", "Chenyan Xiong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.18998", "code_url": "https://github.com/cxcscmu/General-AgentBench", "topic_clusters": [ "C9", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "General-AgentBench (search, coding, reasoning, tool use)" ], "models_studied": [ "ten leading LLM agents" ], "production_evidence": false, "summary": "Unified benchmark across agent domains and a study of sequential and parallel test-time scaling; neither axis yields consistent gains in realistic settings because of a scaling plateau and a verification gap.", "key_idea": "More test-time compute does not reliably help general agents.", "relevance_score": 3, "relevance_rationale": "Evidence on diminishing returns of spending more compute in agents.", "found_via": "web search: tau2-bench dual-control", "cites_or_cited_by": [] }, { "id": "li-2026-toolprmbench", "title": "ToolPRMBench: Evaluating and Advancing Process Reward Models for Tool-using Agents", "authors": [ "Dawei Li", "Yuguang Yao", "Zhen Tan", "Huan Liu", "Ruocheng Guo" ], "year": 2026, "venue_or_source": "Findings of ACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.12294", "code_url": "https://github.com/David-Li0406/ToolPRMBench", "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "process reward" ], "cost_reduction_reported": null, "benchmarks": [ "ToolPRMBench" ], "models_studied": [], "production_evidence": false, "summary": "Benchmark converting tool-use trajectories into step-wise test cases to evaluate whether PRMs distinguish correct and incorrect actions, with multi-LLM verification for label quality.", "key_idea": "PRM benchmark for tool-using agents.", "relevance_score": 3, "relevance_rationale": "Evaluates step-level signals for tool agents.", "found_via": "hf-papers search: process reward model agents tool use", "cites_or_cited_by": [] }, { "id": "lim-2026-lodestar", "title": "Lodestar: An Online-Learning LLM Inference Router", "authors": [ "Gangmuk Lim", "Wanyu Zhao", "Brighten Godfrey", "Jiaxin Shan", "Le Xu", "Liguang Xie" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.00946", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "instance state", "request features", "observed TTFT" ], "cost_reduction_reported": "1.41x lower average TTFT and 1.47x lower P99 TTFT vs prefix-cache/load-aware heuristic (public cloud GPU cluster)", "benchmarks": [ "public cloud GPU cluster experiments" ], "models_studied": [ "vLLM-served LLMs" ], "production_evidence": false, "summary": "Lodestar routes inference requests to GPU instances using an online-trained reward predictor over per-request cluster snapshots, adapting continuously to workload and hardware changes; it integrates with vLLM and learns good policies within about 5 minutes.", "key_idea": "Online-learned request-to-instance routing for LLM serving.", "relevance_score": 3, "relevance_rationale": "Serving-level online routing (not model choice); supporting systems technique.", "found_via": "OpenAlex title: online routing LLM", "cites_or_cited_by": [] }, { "id": "lin-2026-bagen-budget-aware-agents", "title": "BAGEN: Are LLM Agents Budget-Aware?", "authors": [ "Yuxiang Lin", "Zihan Wang", "Mengyang Liu", "Yuxuan Shan", "Longju Bai", "Junyao Zhang", "Xing Jin", "Boshan Chen", "Jinyan Su", "Xingyao Wang", "Jiaxin Pei", "Manling Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.00198", "code_url": null, "topic_clusters": [ "C7", "C9", "C12" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "remaining-budget interval estimates" ], "cost_reduction_reported": "Early stop saves 28-64% tokens on failed trajectories", "benchmarks": [ "four environments" ], "models_studied": [ "five frontier agents" ], "production_evidence": false, "summary": "Defines budget awareness as step-wise interval estimation of remaining cost and finds frontier agents are over-optimistic and keep spending on doomed tasks; budget signals are trainable with SFT+RL and early stopping saves a large share of wasted tokens.", "key_idea": "Agents should predict remaining budget and stop doomed runs early.", "relevance_score": 3, "relevance_rationale": "Budget signal for per-step control.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "bai-2026-agents-spend-your-money" ] }, { "id": "litellm-2026-auto-routing", "title": "[Beta] Auto Routing (LiteLLM proxy docs: complexity, semantic and adaptive routing)", "authors": [ "LiteLLM (BerriAI)" ], "year": 2026, "venue_or_source": "LiteLLM documentation", "source_type": "docs", "url": "https://docs.litellm.ai/docs/proxy/auto_routing", "code_url": "https://github.com/BerriAI/litellm", "topic_clusters": [ "C1", "C5", "C10", "C14" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "heuristic complexity score", "LLM classifier", "keyword/semantic rules", "custom classifier plugin", "reward feedback (Thompson sampling)", "session id / cacheable prefix" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "LiteLLM's beta auto router classifies each request into tiers (e.g., SIMPLE/MEDIUM/COMPLEX/REASONING) with a sub-millisecond heuristic scorer, a small LLM classifier, keyword rules or a custom plugin, then serves a pinned model, a random pool, or a Thompson-sampled pool per tier that learns from reward feedback. Optional session affinity and prefix-keyed deployment pinning preserve prompt caches, and per-decision cause markers plus classifier-cost accounting support spend attribution. It replaces the deprecated embedding-based semantic auto router.", "key_idea": "Open-source gateway combining tiered complexity classification with per-tier bandits and cache-preserving session pinning.", "relevance_score": 3, "relevance_rationale": "Widely deployed OSS gateway implementing query-level routing with online bandits; no evaluation.", "found_via": "direct: docs.litellm.ai", "cites_or_cited_by": [] }, { "id": "liu-2026-cost-utility-alignment-trajectories", "title": "Cost-Utility Alignment in LLM Agent Trajectories:Profiling,Attribution,Diagnosis,Adaptation,and Evaluation", "authors": [ "Dan Liu", "Jian Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.26195", "code_url": null, "topic_clusters": [ "C6", "C12", "C13", "C15" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "utility attribution", "cost profiling", "token cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "n/a" ], "production_evidence": false, "summary": "A framework and survey for judging whether an agent trajectory's resource spend is justified by its task contribution, organized into cost profiling, utility attribution, misalignment diagnosis, targeted adaptation and evaluation.", "key_idea": "Treat cost and utility as dual ledgers over agent trajectories.", "relevance_score": 3, "relevance_rationale": "Framework for per-step cost-value analysis.", "found_via": "snowball: cited-by kapoor-2024-ai-agents-that-matter | web search: coding agents spend tokens", "cites_or_cited_by": [ "bai-2026-agents-spend-your-money", "dang-2025-puppeteer-evolving-orchestration", "zhang-2025-maas-agentic-supernet", "zhang-2026-hera-device-cloud" ] }, { "id": "liu-2026-e3-orch", "title": "E$^3$-Orch: Towards Effective, Efficient, and Extensible Agentic Orchestration with Reinforcement Learning", "authors": [ "Junnan Liu", "Linhao Luo", "Thuy-Trang Vu", "Gholamreza Haffari" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.04588", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "milestone feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "E3-Orch trains an orchestration policy with RL around a milestone-plan-act workflow: for each milestone it plans dependencies, assigns subtasks to suitable executors and runs independent ones in parallel.", "key_idea": "Milestone-scoped RL orchestration with executor assignment.", "relevance_score": 3, "relevance_rationale": "Learned executor assignment.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "dang-2025-puppeteer-evolving-orchestration", "ke-2026-mas-orchestra", "su-2025-daao", "yue-2025-masrouter", "zhang-2024-g-designer" ] }, { "id": "liu-2026-ems-majority-then-stopping", "title": "EMS: Multi-Agent Voting via Efficient Majority-then-Stopping", "authors": [ "Yiqing Liu", "Hantao Yao", "Wu Liu", "Yongdong Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.02863", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "task-conditioned reliability ordering", "incremental vote" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "EMS calls agents in order of retrieved historical reliability for similar queries and stops voting once the leading answer cannot be overturned, avoiding redundant agent calls in majority voting.", "key_idea": "Reliability-ordered agent calls with early stopping of votes.", "relevance_score": 3, "relevance_rationale": "Cuts multi-agent voting cost.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems", "yue-2025-masrouter" ] }, { "id": "liu-2026-masprism", "title": "MASPrism: Lightweight Failure Attribution for Multi-Agent Systems Using Prefill-Stage Signals", "authors": [ "Yang Liu", "Hongjiang Feng", "Junsong Pu", "Zhuangbin Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07509", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "token NLL", "attention weights (prefill)" ], "cost_reduction_reported": null, "benchmarks": [ "Who&When", "TRAIL" ], "models_studied": [ "Qwen3-0.6B" ], "production_evidence": false, "summary": "MASPrism attributes failures in multi-agent executions to source steps using only two prefill passes of a 0.6B model, reading token NLL and attention instead of decoding, and achieves the best results on three of four Who&When/TRAIL subsets.", "key_idea": "Cheap SLM prefill signals for step-level failure attribution.", "relevance_score": 3, "relevance_rationale": "Low-cost step attribution; a building block for step-level routing decisions.", "found_via": "arXiv query ti:\"failure attribution\" AND abs:agent", "cites_or_cited_by": [] }, { "id": "liu-2026-multi-agent-protocol-distillation", "title": "From Proprietary to Open-Source: Bridging the Distribution Gap via Multi-Agent Protocol Distillation in Agentic Search", "authors": [ "Junlin Liu", "Jiangwang Chen", "Zixin Song", "Shuaiyu Zhou", "Chunji Lv", "Hank Wu", "Kailin Jiang", "Jinyang Wu", "Bohan Yu", "Chenxi Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.24280", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "multi-agent teacher protocol" ], "cost_reduction_reported": "Average success 39.4% on Qwen3-1.7B and 44.4% on Qwen3-4B across seven QA benchmarks", "benchmarks": [ "seven QA benchmarks" ], "models_studied": [ "Qwen3-1.7B", "Qwen3-4B" ], "production_evidence": false, "summary": "MAPD distills agentic search behavior from proprietary teachers into small open models through a multi-agent protocol bridging distribution gaps, beating distillation and RL baselines.", "key_idea": "Protocol-based distillation of search agents.", "relevance_score": 3, "relevance_rationale": "Small agent distillation.", "found_via": "hf-papers search: distill multi-agent system into single model", "cites_or_cited_by": [] }, { "id": "liu-2026-search-do-not-guess", "title": "Search, Do not Guess: Teaching Small Language Models to Be Effective Search Agents", "authors": [ "Yizhou Liu", "Qi Sun", "Yulin Chen", "Siyue Zhang", "Chen Zhao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.04651", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "search-behavior training" ], "cost_reduction_reported": "Improves 17.3 on Bamboogle and 15.3 on HotpotQA over agent distillation from LLMs", "benchmarks": [ "Bamboogle", "HotpotQA" ], "models_studied": [], "production_evidence": false, "summary": "Teaches small language models to rely on search rather than guessing from parametric memory, beating standard agent distillation.", "key_idea": "Train SLMs to search instead of guess.", "relevance_score": 3, "relevance_rationale": "Small search agents at LLM level.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "liu-2026-trouter", "title": "Task-Aware LLM Routing with Multi-Level Task-Profile-Guided Data Synthesis for Cold-Start Scenarios", "authors": [ "Hui Liu", "Bin Zou", "Kecheng Chen", "Jie Liu", "Wenya Wang", "Haoliang Li" ], "year": 2026, "venue_or_source": "ACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2604.09377", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "latent task types", "synthesized task taxonomy data" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Addresses cold-start routing without in-domain data by synthesizing QA pairs from a hierarchical task taxonomy and training TRouter with latent task-type variables.", "key_idea": "Taxonomy-guided data synthesis for cold-start routing.", "relevance_score": 3, "relevance_rationale": "Cold start is common in enterprise.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "liu-2026-utility-guided-orchestration", "title": "Utility-Guided Agent Orchestration for Efficient LLM Tool Use", "authors": [ "Boyan Liu", "Gongming Zhao", "Hongli Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.19896", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "estimated gain", "step cost", "uncertainty", "redundancy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Formulates tool-using agent orchestration as an explicit decision among respond, retrieve, call tool, verify and stop, scored by estimated gain, step cost, uncertainty and redundancy. Compares against direct answering, fixed workflows and ReAct to study quality-cost trade-offs.", "key_idea": "Explicit utility-based policy over agent actions including stopping.", "relevance_score": 3, "relevance_rationale": "Controllable per-step cost policy.", "found_via": "arXiv title search: tool-use efficiency agent", "cites_or_cited_by": [] }, { "id": "lodha-2026-less-context-better-agents", "title": "Less Context, Better Agents: Efficient Context Engineering for Long-Horizon Tool-Using LLM Agents", "authors": [ "Abhilasha Lodha", "Mahsa Pahlavikhah Varnosfaderani", "Abir Chakraborty", "Abhinav Mithal" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.10209", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "recent tool-interaction window", "compact summarization" ], "cost_reduction_reported": null, "benchmarks": [ "enterprise expense-processing tool-use workflow" ], "models_studied": [ "Claude Sonnet 4.5" ], "production_evidence": false, "summary": "For an enterprise expense-processing tool-use workflow, selective retention of recent tool interactions plus compact summaries improves reliability and efficiency over full-history retention, with confidence intervals and cross-model checks.", "key_idea": "Recent-window plus summary beats full history in enterprise tool-use agents.", "relevance_score": 3, "relevance_rationale": "Enterprise-style evidence on context pruning.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [ "lindenbauer-2025-complexity-trap" ] }, { "id": "lu-2026-meta-policy-delegation", "title": "History Matters: Meta-policy Delegation with Heterogeneous Multi-agent Reinforcement Learning", "authors": [ "Ziqing Lu", "Avinash Mudireddy", "Sarra M. Alqahtani", "Weiyu Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.03833", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "RL-online", "signals_used": [ "interaction history", "execution cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies MARL-trained delegation among heterogeneous agents with different capabilities and operating costs, so that tasks are completed under resource constraints. Policies depend on interaction history, and the paper also covers prescribed delegation topologies.", "key_idea": "MARL delegation among heterogeneous-cost agents.", "relevance_score": 3, "relevance_rationale": "Cost-aware delegation formulation; not LLM-specific benchmarks in abstract.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "lukassen-2026-cafe", "title": "CAFE: A Compound-AI Factorial Evaluation Framework", "authors": [ "Fabian Lukassen", "Christoph Weisser", "Thomas Kneib", "Alexander Silbersdorff" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.10380", "code_url": null, "topic_clusters": [ "C4", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judge", "human ratings", "mixed-effects models" ], "cost_reduction_reported": null, "benchmarks": [ "HotpotQA" ], "models_studied": [], "production_evidence": false, "summary": "CAFE applies factorial design of experiments to compound AI systems, attributing quality variance to swappable components (retriever, model, prompt) and their interactions and reporting cost and latency trade-offs.", "key_idea": "Factorial DoE for compound AI configuration effects.", "relevance_score": 3, "relevance_rationale": "Statistically rigorous config evaluation.", "found_via": "snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [] }, { "id": "lyu-2026-agenticqwen", "title": "AgenticQwen: Training Small Agentic Language Models with Dual Data Flywheels for Industrial-Scale Tool Use", "authors": [ "Yuanjie Lyu", "Chengyu Wang", "Haonan Zheng", "Yuanhao Yue", "Junbing Yan", "Ming Wang", "Jun Huang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.21590", "code_url": "https://github.com/haruhi-sudo/data_synth_and_rl", "topic_clusters": [ "C8", "C14" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [], "cost_reduction_reported": "Closes the gap with much larger models on search and data-analysis tasks in an industrial agent system", "benchmarks": [ "agentic benchmarks", "industrial agent system" ], "models_studied": [ "AgenticQwen family" ], "production_evidence": true, "summary": "Alibaba trains small agentic Qwen models with multi-round reasoning and agentic RL on synthetic data from dual data flywheels that raise difficulty and turn linear workflows into branching behavior trees. Deployed in an industrial agent system, they approach much larger models on search and data analysis.", "key_idea": "Small agent models trained with difficulty-escalating data flywheels.", "relevance_score": 3, "relevance_rationale": "Production evidence for small agent models.", "found_via": "arXiv title search: small language models agentic", "cites_or_cited_by": [] }, { "id": "mahmood-2026-routing-user-choice", "title": "Routing, Cascades, and User Choice for LLMs", "authors": [ "Rafid Mahmood" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.09902", "code_url": null, "topic_clusters": [ "C1", "C2", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "Stackelberg game", "user re-prompt/abandon behavior", "expected user utility", "latency" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Models a provider routing between a standard and a reasoning model and a user who may re-prompt or abandon as a Stackelberg game. The optimal policy is usually static with no cascading, and provider-optimal routing can be misaligned with users.", "key_idea": "Game-theoretic analysis of routing with user behavior.", "relevance_score": 3, "relevance_rationale": "Theory.", "found_via": "snowball: cited-by ong-2024-routellm | arXiv title search: cascade LLM", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "markovic-voronov-2026-batch-routing", "title": "Robust Batch-Level Query Routing for Large Language Models under Cost and Capacity Constraints", "authors": [ "Jelena Markovic-Voronov", "Kayhan Behdin", "Yuanda Xu", "Zhengze Zhou", "Zhipeng Wang", "Rahul Mazumder" ], "year": 2026, "venue_or_source": "CAIS 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2603.26796", "code_url": null, "topic_clusters": [ "C1", "C11" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted performance", "GPU capacity", "robust optimization" ], "cost_reduction_reported": "Batch-level routing outperforms per-query methods by up to 24% under adversarial batching (abstract)", "benchmarks": [ "two multi-task benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Routes batches of queries jointly under cost, GPU and concurrency constraints, with a robust variant accounting for predictor uncertainty and an offline instance-allocation step.", "key_idea": "Robust batch-level routing under capacity constraints.", "relevance_score": 3, "relevance_rationale": "Serving-aware routing (LinkedIn authors).", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "mccleary-2026-bcas", "title": "Quantifying the Accuracy and Cost Impact of Design Decisions in Budget-Constrained Agentic LLM Search", "authors": [ "Kyle McCleary", "James Ghawaly" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.08877", "code_url": null, "topic_clusters": [ "C3", "C9", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "remaining budget" ], "cost_reduction_reported": null, "benchmarks": [ "three QA benchmarks" ], "models_studied": [ "six LLMs" ], "production_evidence": false, "summary": "BCAS is a harness that exposes remaining budget to agentic RAG systems and gates tool use. It measures how search depth, retrieval strategy and completion budget affect accuracy and cost across six LLMs; accuracy saturates after a few searches and hybrid retrieval with light re-ranking helps most.", "key_idea": "Measurement harness for budget-constrained agentic search design.", "relevance_score": 3, "relevance_rationale": "Budgeted agent-execution evidence.", "found_via": "arXiv query ti:\"budget\" AND ti:agent AND ti:llm", "cites_or_cited_by": [] }, { "id": "mehta-2026-behavioral-consistency", "title": "When Agents Disagree With Themselves: Behavioral Consistency as an Uncertainty Signal for LLM Agents", "authors": [ "Aman Mehta" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.11619", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "action-sequence consistency across runs" ], "cost_reduction_reported": null, "benchmarks": [ "HotpotQA" ], "models_studied": [], "production_evidence": false, "summary": "Shows that the number of distinct action sequences an agent produces across repeated runs is a black-box uncertainty signal: consistent tasks reach 82-87% accuracy versus 41-65% for inconsistent ones. Selective answering when three runs agree gives 87-88% accuracy at 54-62% coverage.", "key_idea": "Run-to-run behavioral consistency as an agent success predictor.", "relevance_score": 3, "relevance_rationale": "Cheap success signal usable for agent-level deferral.", "found_via": "query: abs:\"predict\" AND abs:\"trajectory\" AND abs:\"failure\" AND abs:\"LLM agent\"", "cites_or_cited_by": [] }, { "id": "merchant-2026-temporal-semantic-cache-mcp", "title": "Evaluating Temporal Semantic Caching and Workflow Optimization in Agentic Plan-Execute Pipelines", "authors": [ "Alimurtaza Mustafa Merchant", "Krish Veera", "Sajal Kumar Goyla", "Shambhawi Bhure", "Dhaval Patel", "Kaoutar El Maghraoui" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.20630", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "temporal semantic similarity", "tool-discovery cache", "dependency-aware parallelism" ], "cost_reduction_reported": "MCP workflow optimizations gave 1.67x speedup and ~40% lower median end-to-end latency; temporal cache achieved 30.6x median speedup on hits", "benchmarks": [ "industrial MCP plan-execute benchmark" ], "models_studied": [], "production_evidence": false, "summary": "Evaluates a temporal semantic cache and MCP workflow optimizations (caching tool discovery, parallelizing independent steps) in plan-execute agent pipelines, and documents a failure mode where pure semantic caching returns wrong answers for parameter-rich queries.", "key_idea": "Caching and parallelism help plan-execute agents but semantic caching breaks on parameter-rich queries.", "relevance_score": 3, "relevance_rationale": "Practical evidence on caching correctness in tool-heavy agents.", "found_via": "semantic cache LLM agents tool calls caching arXiv", "cites_or_cited_by": [ "zhang-2025-agentic-plan-caching" ] }, { "id": "mieczkowski-2026-latte-adaptive-task-graphs", "title": "Improving the Efficiency of Language Agent Teams with Adaptive Task Graphs", "authors": [ "Elizabeth Mieczkowski", "Alexander Ku", "Tiwalayo Eisape", "Dilip Arumugam", "John Matters", "Katherine M. Collins", "Ilia Sucholutsky", "Thomas L. Griffiths" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06320", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "shared coordination graph" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "LATTE lets an LLM team co-build and maintain a shared task graph of subtask dependencies, assignments and progress, reducing wasted time, tokens and file operations compared with unstructured teams.", "key_idea": "Shared evolving task graph to cut MAS coordination waste.", "relevance_score": 3, "relevance_rationale": "MAS efficiency via coordination structure.", "found_via": "snowball: cited-by cemri-2025-mast-why-mas-fail", "cites_or_cited_by": [ "kim-2025-science-of-scaling-agent-systems", "zhang-2024-agentprune" ] }, { "id": "mo-2026-entworld", "title": "EntWorld: A Holistic Environment and Benchmark for Verifiable Enterprise GUI Agents", "authors": [ "Ying Mo", "Yu Bai", "Dapeng Sun", "Yuqian Shi", "Yukai Miao", "Li Chen", "Dan Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.17722", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "SQL state verification" ], "cost_reduction_reported": null, "benchmarks": [ "EntWorld (1,756 tasks: CRM, ITIL, ERP, etc.)" ], "models_studied": [ "GPT-4.1", "others" ], "production_evidence": false, "summary": "Enterprise GUI benchmark of 1,756 tasks across six domains including ITIL, generated from database schemas and verified by SQL state checks; GPT-4.1 reaches 47.61% success.", "key_idea": "Schema-grounded enterprise (incl. ITIL) GUI tasks.", "relevance_score": 3, "relevance_rationale": "ITIL/ITSM workflow coverage.", "found_via": "web search: OSWorld benchmark", "cites_or_cited_by": [] }, { "id": "moon-2026-price-of-thinking", "title": "The Price of Thinking: Reasoning Effort as a Model-Specific API Contract", "authors": [ "Yeabin Moon" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.16956", "code_url": null, "topic_clusters": [ "C9", "C13", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "reasoning-effort API parameter" ], "cost_reduction_reported": "Explicit-high effort cost $0.01031 more per call than omitted effort on Sonnet 5 (30 AIME 2026 items x5); no detected accuracy difference (+0.0133); cost per correct $0.08665 vs $0.07662", "benchmarks": [ "AIME 2026 (30 items)" ], "models_studied": [ "Sonnet 5" ], "production_evidence": false, "summary": "Preregistered paired study treating reasoning effort as part of a dated API contract. Explicit high effort cost more with no detectable accuracy gain on the tested items, and omission semantics vary by model.", "key_idea": "Reasoning-effort defaults are model-specific contract terms with cost impact.", "relevance_score": 3, "relevance_rationale": "Rigorous but narrow evidence on API effort settings.", "found_via": "arXiv title search: reasoning effort", "cites_or_cited_by": [] }, { "id": "morandi-2026-selection-bias-sparse-feedback", "title": "Correcting Selection Bias in Sparse User Feedback for Large Language Model Quality Estimation: A Multi-Agent Hierarchical Bayesian Approach", "authors": [ "Andrea Morandi", "Mahesh Viswanathan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.12177", "code_url": null, "topic_clusters": [ "C6", "C7", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "thumbs feedback", "topic clusters" ], "cost_reduction_reported": null, "benchmarks": [ "UltraFeedback (simulated selection bias)" ], "models_studied": [], "production_evidence": false, "summary": "Shows that averaging thumbs feedback in production can misestimate system quality by 40-50 points because feedback arrives mainly from the tails. Proposes a hierarchical Bayesian, topic-stratified correction that estimates per-topic selection and quality, plus drift signals, and evaluates it under simulated biases on UltraFeedback.", "key_idea": "Correct thumbs-feedback selection bias with topic-stratified hierarchical Bayes before using it as a quality metric.", "relevance_score": 3, "relevance_rationale": "Practical guide to making sparse thumbs data usable for monitoring or route evaluation.", "found_via": "query: implicit user feedback LLM production", "cites_or_cited_by": [] }, { "id": "ndzomga-2026-efficient-benchmarking-agents", "title": "Efficient Benchmarking of AI Agents", "authors": [ "Franck Ndzomga" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.23749", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "historical pass rates" ], "cost_reduction_reported": "Mid-difficulty task filter cuts evaluation tasks by 44-70% while preserving rank fidelity", "benchmarks": [ "eight benchmarks", "33 scaffolds", "70+ model configurations", "8 agent benchmarks" ], "models_studied": [ "33 scaffolds, 70+ model configs" ], "production_evidence": false, "summary": "Shows that small task subsets preserve agent rankings even under scaffold-driven distribution shift, and proposes evaluating new agents only on tasks with 30-70% historical pass rates. This cuts evaluation cost by 44-70% with reliable rankings.", "key_idea": "Evaluate on mid-difficulty tasks to rank agents cheaply.", "relevance_score": 3, "relevance_rationale": "Evaluation cost reduction; useful for config search budgets.", "found_via": "web search: HAL | web search: enterprise agent benchmark cost efficiency", "cites_or_cited_by": [ "kapoor-2025-hal-holistic-agent-leaderboard" ] }, { "id": "nguyen-2026-cupid-model-zoo", "title": "CUPID in the Model Zoo: Online Matchmaking for Selecting Your Dream LLM", "authors": [ "S. Nguyen", "Xin-Yu Liu", "Ransalu Senanayake" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.00846", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "pairwise user preference", "cost budget" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Treats choosing an LLM for a user as interactive active learning: a dueling-bandit algorithm shows pairs of model responses, collects the user's preference, and updates a belief over latent preferences. A belief-aware UCB rule trades off exploration against user-specified cost and time budgets, validated with simulations and human studies.", "key_idea": "Dueling-bandit matchmaking that learns which LLM a user prefers from pairwise feedback under a budget.", "relevance_score": 3, "relevance_rationale": "Arena-style pairwise feedback used online for model selection; per-user rather than per-query.", "found_via": "snowball: cited-by tsiourvas-2025-causal-routing", "cites_or_cited_by": [ "tsiourvas-2025-causal-routing" ] }, { "id": "ni-2026-topas", "title": "TOPAS: Workflow-Aware Prefix-State Scheduling for Multi-Agent LLM Serving", "authors": [ "Hongqiu Ni", "Han Tian", "Chi Zhang", "Guopeng Li", "Haisheng Tan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.25523", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "workflow DAG", "prefix state" ], "cost_reduction_reported": "mean/p99 JCT reduced up to 39.8%/49.4% on synthetic DAGs; mean JCT reduced 9.8% (MetaGPT-SOP) and mean/p99 22.0%/26.6% (MetaGPT-TL)", "benchmarks": [ "synthetic DAGs", "MetaGPT workflows" ], "models_studied": [], "production_evidence": false, "summary": "Workflow-aware prefix-state scheduling for multi-agent serving in SGLang, with preemption and task-level aging.", "key_idea": "Schedule multi-agent DAGs to exploit prefix state.", "relevance_score": 3, "relevance_rationale": "Multi-agent serving scheduler.", "found_via": "serving engine LLM agents agentic workflows scheduling arXiv 2025", "cites_or_cited_by": [ "pan-2025-kvflow" ] }, { "id": "nie-2026-enumgrpo", "title": "Cost-Aware Optimization for Agentic Query Execution", "authors": [ "Lunyiu Nie", "Yilin Xia", "Yiren Liu", "Christopher Jermaine", "Swarat Chaudhuri" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.03152", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "RL-offline", "signals_used": [ "quality-cost feedback" ], "cost_reduction_reported": "35.4% execution accuracy at $0.011 per query, ~317x cost reduction over hybrid query baseline with 18% relative accuracy improvement (SWAN)", "benchmarks": [ "SWAN" ], "models_studied": [], "production_evidence": false, "summary": "Formalizes agentic query execution, where the placement, ordering and granularity of LLM operators jointly set dollar cost and quality. EnumGRPO enumerates plans and distills quality-cost feedback into planning heuristics via in-context RL.", "key_idea": "Cost-aware optimization of where LLM operators run in agentic query plans.", "relevance_score": 3, "relevance_rationale": "Per-operator cost-quality planning.", "found_via": "arXiv query ti:\"cost-aware\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "nie-2026-generative-opt-challenges", "title": "Understanding the Challenges in Iterative Generative Optimization with LLMs", "authors": [ "Allen Nie", "Xavier Daull", "Zhiyi Kuang", "Abhinav Akkiraju", "Anish Chaudhuri", "Max Piasevoli", "Ryan Rong", "YuCheng Yuan", "Prerit Choudhary", "Shannon Xiao", "Rasool Fakoor", "Adith Swaminathan", "Ching-An Cheng" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.23994", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "execution feedback" ], "cost_reduction_reported": null, "benchmarks": [ "MLAgentBench", "Atari", "BigBench Extra Hard" ], "models_studied": [], "production_evidence": false, "summary": "Argues LLM-driven generative optimization is brittle because of hidden design choices (starting artifact, credit horizon, batching of evidence), notes only 9% of surveyed agents use automated optimization, and demonstrates these choices decide success.", "key_idea": "Hidden design choices determine whether generative optimization works.", "relevance_score": 3, "relevance_rationale": "Practical guidance on making C4 optimizers reliable.", "found_via": "snowball: cited-by conway-2025-syftr", "cites_or_cited_by": [ "cheng-2024-trace" ] }, { "id": "nie-2026-rivet-internalizing-experts", "title": "From Collaboration to Capability: Internalizing Routed LLM Experts into Compact Reasoners", "authors": [ "Frank Nie", "Shuyao Wang", "Ethan B. Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.12578", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "seven competition-math benchmarks", "GPQA-Diamond" ], "models_studied": [ "RIVET-1.7B", "RIVET-4B" ], "production_evidence": false, "summary": "Trains a compact controller that routes to stronger experts, then internalizes the collaboration via expert-augmented RL and verified trajectory distillation so it works without any external LLM, gaining 6.49 points after expert removal.", "key_idea": "Distill a router's expert consultations into the router itself.", "relevance_score": 3, "relevance_rationale": "From routing to distillation of routed experts.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "su-2025-toolorchestra" ] }, { "id": "nilayam-2026-heterogeneous-debate-adversarial", "title": "Heterogeneous LLM Debate Under Adversarial Peers: Honest Gains, Replacement Costs, and Resilience", "authors": [ "Prashanti Nilayam", "Kiran Kumar Ramanna", "Prashil Tumbade", "Sankalp Nayak" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.19826", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "revision behavior" ], "cost_reduction_reported": null, "benchmarks": [ "MATH-hard", "three reasoning benchmarks" ], "models_studied": [ "Llama-3.1-70B", "four model families" ], "production_evidence": false, "summary": "Measures how a heterogeneous peer changes honest agents' revisions in debate. An honest cross-family peer cuts harmful revision on MATH-hard from 89% to 35% for Llama-3.1-70B defenders, while an adversarial one returns it to 90%.", "key_idea": "Heterogeneous debate peers help when honest but amplify harm when adversarial.", "relevance_score": 3, "relevance_rationale": "Heterogeneous debate dynamics; no cost numbers.", "found_via": "arXiv query ti:debate AND ti:heterogeneous", "cites_or_cited_by": [] }, { "id": "niu-2026-armor-mad", "title": "ARMOR-MAD: Adaptive Routing for Heterogeneous Multi-Agent Debate in Large Language Model Reasoning", "authors": [ "Fuqiang Niu", "Bowen Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.13197", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "round-0 agreement", "convergence" ], "cost_reduction_reported": null, "benchmarks": [ "MATH Level 5", "GSM8K", "MMLU", "MMLU-Pro" ], "models_studied": [], "production_evidence": false, "summary": "ARMOR-MAD treats heterogeneous multi-agent debate as conditional computation: skip debate when initial answers agree, stop early after convergence, and down-weight outliers. It beats fixed-round heterogeneous debate with the same pool.", "key_idea": "Agreement-gated debate over heterogeneous models.", "relevance_score": 3, "relevance_rationale": "Conditional MAS computation.", "found_via": "arXiv title search: model routing agentic", "cites_or_cited_by": [] }, { "id": "okamoto-2026-topaz-explainable-routing", "title": "Explainable Model Routing for Agentic Workflows", "authors": [ "Mika Okamoto", "Ansel Kaplan Erol", "Mark Riedl" ], "year": 2026, "venue_or_source": "CHI 2026 Workshop (HCXAI)", "source_type": "paper", "url": "https://arxiv.org/abs/2604.03527", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "skill-based capability profiles", "budget" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Topaz makes subtask-to-model routing in agentic workflows auditable, using skill-based capability profiles, traceable budget-based and multi-objective routing, and natural-language explanations so developers can tune cost-quality trade-offs.", "key_idea": "Interpretable, auditable subtask-level model routing.", "relevance_score": 3, "relevance_rationale": "Governance for step-level routing decisions.", "found_via": "arXiv title search: model routing agentic", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "openai-reasoning-effort-docs", "title": "Reasoning models: reasoning effort (OpenAI API guide)", "authors": [ "OpenAI" ], "year": 2026, "venue_or_source": "OpenAI API documentation", "source_type": "docs", "url": "https://developers.openai.com/api/docs/guides/reasoning", "code_url": null, "topic_clusters": [ "C9", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-6 family", "OpenAI reasoning models" ], "production_evidence": false, "summary": "Documents the reasoning.effort parameter (model-dependent values from none/minimal through low, medium, high, xhigh and max) that trades reasoning-token usage and latency against answer quality, noting that models also adapt reasoning length to task difficulty within a level. Covers reasoning-token accounting and context handling for reasoning items.", "key_idea": "A per-request effort knob is the main production control for reasoning-token spend.", "relevance_score": 3, "relevance_rationale": "The concrete API knob that step-level effort routing research (e.g., adaptive effort selection) targets.", "found_via": "direct: developers.openai.com docs", "cites_or_cited_by": [ "anthropic-effort-docs", "google-gemini-thinking-docs" ] }, { "id": "pan-2026-smoothagent", "title": "SmoothAgent: Efficient Long-Horizon LLM-Based Agent Serving with Lookahead Context Engineering", "authors": [ "Zaifeng Pan", "Qianxu Wang", "Zhengding Hu", "Chang Chen", "Yue Guan", "Yanbo Zhou", "Steven Swanson", "Yufei Ding" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.00151", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "lookahead context transformation" ], "cost_reduction_reported": "eliminates context-transformation overhead and reduces TTFT by up to 11.9x", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Context engineering (summarization, pruning) in long-horizon agents invalidates caches and adds latency; SmoothAgent performs these transformations ahead of time asynchronously with a lookahead-aware scheduler.", "key_idea": "Do context engineering asynchronously ahead of need.", "relevance_score": 3, "relevance_rationale": "Makes context compaction cheap at serving time.", "found_via": "Context Engineering for AI Agents: Lessons from Building Manus KV-cache hit rate", "cites_or_cited_by": [ "pan-2025-kvflow" ] }, { "id": "patel-2026-locus", "title": "LOCUS: Low-Dimensional Model Embeddings for Efficient Model Exploration, Comparison, and Selection", "authors": [ "Shivam Patel", "William Cocke", "Gauri Joshi" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.21082", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "attention-based model embeddings", "correctness predictor" ], "cost_reduction_reported": "Needs up to 4.8x fewer query evaluation samples than baselines to produce informative embeddings (abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "LOCUS produces low-dimensional model embeddings via a forward pass over query encodings and scores, enabling retrain-free addition of models and state-of-the-art routing accuracy.", "key_idea": "Sample-efficient model embeddings.", "relevance_score": 3, "relevance_rationale": "Cheaper model profiling.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "patel-2026-six-sigma-agent", "title": "The Six Sigma Agent: Achieving Enterprise-Grade Reliability in LLM Systems Through Consensus-Driven Decomposed Execution", "authors": [ "Khush Patel", "Siva Surendira", "Jithin George", "Shreyas Kapale" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.22290", "code_url": null, "topic_clusters": [ "C3", "C12", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "consensus voting across diverse LLMs" ], "cost_reduction_reported": "14,700x reliability improvement over single-agent execution while reducing costs by 80% (three enterprise use cases)", "benchmarks": [ "three enterprise use cases" ], "models_studied": [], "production_evidence": false, "summary": "Six Sigma Agent decomposes tasks into atomic actions, runs each n times across diverse cheaper LLMs, and takes a consensus vote with dynamic scaling. Theory shows error falls exponentially in n.", "key_idea": "Redundant cheap-model micro-agents with voting instead of one expensive model.", "relevance_score": 3, "relevance_rationale": "Cheap-model redundancy as a cost lever; enterprise claims are from an evaluation, not live traffic.", "found_via": "arXiv query ti:enterprise AND ti:agent AND abs:\"model\" AND abs:\"cost\"", "cites_or_cited_by": [] }, { "id": "peng-2026-hexagent", "title": "HexAGenT: Efficient Agentic LLM Serving via Workflow- and Heterogeneity-Aware Scheduling", "authors": [ "You Peng", "Youhe Jiang", "Wenshuang Li", "Xu Xu", "Ke Zhou", "Jiawei Jiang", "Chen Wang", "Binhang Yuan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.16637", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "workflow deadline risk", "heterogeneous GPU tiers" ], "cost_reduction_reported": "SLO scale needed for timely workflow completion reduced by 20.1% at 95% and 33.0% at 99% attainment on average (max 45.0% / 80.5%)", "benchmarks": [ "representative agentic workloads" ], "models_studied": [], "production_evidence": false, "summary": "Workflow- and heterogeneity-aware scheduling that jointly chooses prefill and decode placement and queue priority on mixed A100/H100/H200 clusters, accounting for KV capacity and transfer cost.", "key_idea": "Place workflow stages on heterogeneous GPUs by deadline risk.", "relevance_score": 3, "relevance_rationale": "Cost-aware heterogeneous hardware use for agent workflows.", "found_via": "serving engine LLM agents agentic workflows scheduling arXiv 2025", "cites_or_cited_by": [ "zhong-2024-distserve" ] }, { "id": "piriyakulkij-2026-subagents-vs-skills", "title": "Subagents vs Agent Skills: Executing Reusable Knowledge for Long-Horizon Agentic Tasks", "authors": [ "Wasu Top Piriyakulkij", "Rachel Lawrence", "Alicia Curth", "Sushrut Karmalkar", "Niranjani Prasad" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.09233", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "long-horizon agentic tasks" ], "models_studied": [], "production_evidence": false, "summary": "Compares loading skill instructions into the main context against invoking skill packages as subagents with fresh context windows. Subagents win when skills have clear input-output contracts, at the price of extra coordination tokens.", "key_idea": "When to run reusable knowledge as subagents vs in-context skills.", "relevance_score": 3, "relevance_rationale": "Architecture trade-off relevant to subagent design.", "found_via": "arXiv title search: subagent", "cites_or_cited_by": [] }, { "id": "pona-2026-calibrate-then-delegate", "title": "Calibrate-Then-Delegate: Safety Monitoring with Risk and Budget Guarantees via Model Cascades", "authors": [ "Edoardo Pona", "Milad Kazemi", "Mehran Hosseini", "Yali Du", "David Watson", "Osvaldo Simeone", "Nicola Paoletti" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.14251", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "delegation-value probe on hidden states", "multiple hypothesis testing" ], "cost_reduction_reported": null, "benchmarks": [ "four safety datasets" ], "models_studied": [], "production_evidence": false, "summary": "For safety monitoring, a probe predicts the benefit of calling an expensive expert (delegation value) rather than just uncertainty, and thresholds are calibrated for finite-sample cost or safety guarantees.", "key_idea": "Delegate on predicted value of the expert call.", "relevance_score": 3, "relevance_rationale": "Better escalation objective with guarantees.", "found_via": "snowball: cited-by AutoMix (2310.12963)", "cites_or_cited_by": [ "aggarwal-2023-automix" ] }, { "id": "qiao-2026-verifymas", "title": "VerifyMAS: Hypothesis Verification for Failure Attribution in LLM Multi-Agent Systems", "authors": [ "Hezhe Qiao", "Hanghang Tong", "Ee-Peng Lim", "Bing Liu", "Guansong Pang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.17467", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "hypothesis verification" ], "cost_reduction_reported": null, "benchmarks": [ "Aegis-Bench", "Who&When" ], "models_studied": [ "Qwen", "GPT" ], "production_evidence": false, "summary": "Error-first failure attribution that validates trajectory-level error hypotheses before localizing agents, improving attribution across backbones without sacrificing efficiency.", "key_idea": "Verify error hypotheses, then localize agent.", "relevance_score": 3, "relevance_rationale": "Failure attribution.", "found_via": "hf-papers search: agent failure detection", "cites_or_cited_by": [ "zhang-2025-which-agent-when" ] }, { "id": "raghavendra-2026-swe-atlas", "title": "SWE Atlas: Benchmarking Coding Agents Beyond Issue Resolution", "authors": [ "Mohit Raghavendra", "Soham Dan", "Miguel Romero Calvo", "Yannis Yiming He", "Johannes Baptist Mols", "Gautam Anand", "Cole McCollum", "Edgar Arakelyan", "Vijay Bharadwaj", "Andrew Park", "Jeff Da", "MohammadHossein Rezaei", "Bing Liu", "Brad Kenstler", "Yunzhong He" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.08366", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "rubrics", "programmatic checks" ], "cost_reduction_reported": "Reports cost-per-task Pareto from ~$0.35/task (~15% pass) to ~$1.90/task (~40% pass) (per search-result excerpt)", "benchmarks": [ "SWE Atlas (Codebase Q&A, Test Writing, Refactoring)" ], "models_studied": [ "GPT-5.4", "Opus 4.7", "Gemini 3 Flash", "GPT 5.3 Codex" ], "production_evidence": false, "summary": "Coding-agent benchmark across codebase Q&A, test writing and refactoring with rubric-based quality assessment, including a cost-per-task analysis from scaffold logs.", "key_idea": "Coding benchmark with cost-per-task Pareto.", "relevance_score": 3, "relevance_rationale": "Cost-per-task reporting on coding agents.", "found_via": "web search: SWE-bench cost per task", "cites_or_cited_by": [] }, { "id": "raj-2026-consistency-testable", "title": "Consistency as a Testable Property: Statistical Methods to Evaluate AI Agent Reliability", "authors": [ "Harsh Raj", "Niranjan Orkat", "Suvrorup Mukherjee", "Aritra Guha", "Cheryl Flynn", "Subhabrata Majumdar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.10516", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "U-statistics", "trajectory kernels" ], "cost_reduction_reported": null, "benchmarks": [ "multiple agent tasks" ], "models_studied": [ "LLM agents" ], "production_evidence": false, "summary": "Develops a statistical measurement framework for agent consistency under meaning-preserving perturbations, using U-statistics for outputs and kernel metrics for trajectories, and shows high accuracy can hide severe consistency failures.", "key_idea": "Statistical tests for agent consistency.", "relevance_score": 3, "relevance_rationale": "Reliability evaluation methodology.", "found_via": "web search: agent reliability metrics", "cites_or_cited_by": [] }, { "id": "richards-2026-agentlogs", "title": "AgentLogs: A Dataset for Opening the Black Box of GitHub's Cloud Agent", "authors": [ "Jonan Richards", "Kosei Horikawa", "Youmei Fan", "Yutaro Kashiwa", "Mairieli Wessel" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.29204", "code_url": null, "topic_clusters": [ "C6", "C13", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "step-level agent session logs", "token usage" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GitHub Copilot cloud agent" ], "production_evidence": true, "summary": "Releases AgentLogs: 307,416 tasks and 549,239 sessions of GitHub Copilot cloud agent activity with 64M step-level log entries including prompts, tool calls and token usage, enabling studies of agent efficiency, cost and failure.", "key_idea": "Large real-world agent trace dataset with per-step token usage.", "relevance_score": 3, "relevance_rationale": "Rare public production agent logs suitable for offline analysis of cost and success prediction.", "found_via": "query: offline RL multi-agent orchestration logs", "cites_or_cited_by": [] }, { "id": "rognvaldsson-2026-lean-prover-cost-quality", "title": "Optimizing the Cost-Quality Tradeoff of Agentic Theorem Provers in Lean", "authors": [ "Kári Rögnvaldsson", "Chenhao Sun", "Jasper Dekoninck", "Martin Vechev" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.04883", "code_url": null, "topic_clusters": [ "C3", "C4", "C9" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "failed Lean attempts", "success likelihood", "attempt cost", "estimated success likelihood" ], "cost_reduction_reported": "Decreases cost by 28.9% over a fixed-step baseline on average on a PutnamBench subset, preserving performance", "benchmarks": [ "PutnamBench" ], "models_studied": [], "production_evidence": false, "summary": "An action-routing agent for Lean theorem proving has a control plane that reads failed attempts, estimates success probability and cost of another try, and decides whether to continue or restart with a new decomposition. It cuts cost by about 29% at equal performance.", "key_idea": "Control plane decides continue-vs-restart from failure signals and cost.", "relevance_score": 3, "relevance_rationale": "Cost-aware control of agent steps.", "found_via": "arXiv search: agent model routing cost step | snowball: cited-by chen-2025-llmselector", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "rokon-2026-autorelannotator", "title": "AutoRelAnnotator: Calibrated Model Cascades for Cost-Efficient Relevance Evaluation in Sponsored Search", "authors": [ "Md Omar Faruk Rokon", "Shasvat Desai", "Hong Yao", "Kuang-chih Lee" ], "year": 2026, "venue_or_source": "SIGIR 2026 E-commerce workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2606.25871", "code_url": null, "topic_clusters": [ "C2", "C8", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "per-class isotonic calibrated confidence" ], "cost_reduction_reported": "Cascading is roughly accuracy-neutral but halves compute cost; production deployment across six use cases with 150M+ annotations", "benchmarks": [ "sponsored-search relevance annotation" ], "models_studied": [ "fine-tuned classifiers of increasing size" ], "production_evidence": true, "summary": "A calibrated cascade of progressively larger fine-tuned classifiers for offline relevance annotation in sponsored search. Separates gains: fine-tuning gives accuracy, cascading halves cost, calibration adds a small reliable gain.", "key_idea": "Fine-tuning for accuracy, cascading for cost, calibration on top.", "relevance_score": 3, "relevance_rationale": "Production cascade case study.", "found_via": "arXiv title search: model cascading", "cites_or_cited_by": [] }, { "id": "rosset-2026-universal-verifier-cua", "title": "The Art of Building Verifiers for Computer Use Agents", "authors": [ "Corby Rosset", "Pratyusha Sharma", "Andrew Zhao", "Miguel Gonzalez-Fernandez", "Ahmed Awadallah" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.06240", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "rubrics", "process vs outcome rewards" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Lessons from building a universal verifier for web computer-use agents: non-overlapping rubrics, separate process and outcome rewards, controllable vs uncontrollable failures.", "key_idea": "Design principles for CUA verifiers.", "relevance_score": 3, "relevance_rationale": "Practical guidance for trajectory judges.", "found_via": "hf-papers search: step-level verifier web agent", "cites_or_cited_by": [] }, { "id": "saha-2026-llm-judge-on-budget", "title": "LLM-as-Judge on a Budget", "authors": [ "Aadirupa Saha", "Aniket Wagde", "Branislav Kveton" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.15481", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "bandit-online", "signals_used": [ "judge score variance" ], "cost_reduction_reported": null, "benchmarks": [ "Summarize-From-Feedback", "HelpSteer2" ], "models_studied": [], "production_evidence": false, "summary": "Adaptive allocation of repeated LLM-judge queries across prompt-response pairs based on score variance, with near-optimal error bounds for a fixed budget.", "key_idea": "Budget-optimal judge query allocation.", "relevance_score": 3, "relevance_rationale": "Makes judge-based signals cheaper.", "found_via": "hf-papers search: LLM judge cost efficient", "cites_or_cited_by": [] }, { "id": "satija-2026-fastworkflow", "title": "fastWorkflow: Closing the Performance Gap Between Small and Frontier Language Models for Conversational Agents", "authors": [ "Sanchit Satija", "Aditya Bhatt", "Priyanshu Jani", "Dharmesh D. Rawal" ], "year": 2026, "venue_or_source": "CAIS 2026", "source_type": "paper", "url": "https://doi.org/10.1145/3786335.3813158", "code_url": null, "topic_clusters": [ "C3", "C8", "C13" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "cascaded NLU intent detection" ], "cost_reduction_reported": null, "benchmarks": [ "tau-bench" ], "models_studied": [], "production_evidence": false, "summary": "Builds a taxonomy of SLM failure modes in conversational tool-use agents (NLU, tool management, decomposition, reasoning, context) measured on tau-bench, then proposes a dual-mode architecture with a cascaded NLU pipeline to close the gap to frontier models.", "key_idea": "Characterize and engineer around SLM failure modes in agent workflows.", "relevance_score": 3, "relevance_rationale": "Informs where small models fail inside agents.", "found_via": "snowball: cited-by belcak-2025 / su-2025-toolorchestra", "cites_or_cited_by": [] }, { "id": "satish-2026-beyond-token-savings", "title": "Beyond Token Savings: A Systematic Study of Context Compression in LLM Agents", "authors": [ "Ritul Satish", "Prasoon Sinha", "Akiho Kawada", "Neeraja J. Yadwadkar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.32961", "code_url": null, "topic_clusters": [ "C10", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "compression policy" ], "cost_reduction_reported": "on Terminal-Bench with Qwen, policies using roughly one-third of the tokens can take 20-80% longer than the uncompressed agent", "benchmarks": [ "Terminal-Bench" ], "models_studied": [ "Qwen" ], "production_evidence": false, "summary": "Systematic study of context compression policies in agents measuring success, tokens, latency and estimated cost: fewer tokens need not mean cheaper or faster execution, and policies with similar success solve different tasks.", "key_idea": "Compression effects must be judged on execution, not token counts.", "relevance_score": 3, "relevance_rationale": "Evidence that compression gains are workload- and model-specific.", "found_via": "tool output observation compression LLM agent tokens arXiv", "cites_or_cited_by": [ "kang-2025-acon", "weinberger-2026-token-reduction-not-cost" ] }, { "id": "shen-2026-dynbranch", "title": "DynBranch: Speculative Subgraph Reuse for Dynamic Agentic LLM Serving", "authors": [ "Junyi Shen", "Noppanat Wadlom", "Zhengyuan Su", "Yao Lu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.31047", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "expected reuse benefit vs load price" ], "cost_reduction_reported": "mean latency reduced up to 32% over the strongest prior system per workload and 46-66% vs no reuse, preserving workflow results", "benchmarks": [ "four agentic workloads" ], "models_studied": [ "Qwen3-32B", "Qwen3-8B" ], "production_evidence": false, "summary": "Speculatively reuses subgraphs of dynamic agentic workflows at the model-API boundary when expected benefit exceeds the load it adds, requiring no harness or engine changes.", "key_idea": "Speculative reuse of workflow subgraphs at the API boundary.", "relevance_score": 3, "relevance_rationale": "Workflow-level reuse without harness changes.", "found_via": "snowball: cited-by zhang-2025-agentic-plan-caching", "cites_or_cited_by": [ "shen-2025-halo", "zhang-2025-agentic-plan-caching" ] }, { "id": "shi-2026-dynsteer", "title": "DynSTEER: Dynamic Stage-wise Trajectory Evaluation and Execution-time Review for Agents", "authors": [ "Zhichao Shi", "Xuhui Jiang", "Wenjie Zhang", "Xiaojun Wu", "Cehao Yang", "Chengjin Xu", "Jian Guo", "Yuanzhuo Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.14637", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "stage-wise trajectory evaluation" ], "cost_reduction_reported": "Improves evaluation discriminability by over 85% vs whole-trajectory evaluation and saves 17.74% of execution steps", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Evaluates long agent trajectories at a dynamic stage granularity (between whole-trajectory and atomic steps) and reviews during execution.", "key_idea": "Stage-level trajectory evaluation at execution time.", "relevance_score": 3, "relevance_rationale": "Mid-granularity quality signal that also saves steps.", "found_via": "WebSearch: agent trajectory evaluation step-level reward benchmark", "cites_or_cited_by": [] }, { "id": "shi-2026-lamas-latency-aware-mas", "title": "Learning Latency-Aware Orchestration for Multi-Agent Systems", "authors": [ "Xi Shi", "Mengxin Zheng", "Qian Lou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.10560", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "critical execution path" ], "cost_reduction_reported": "Over 50% end-to-end latency reduction at competitive or better accuracy", "benchmarks": [ "four benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "LAMaS learns latency-aware MAS execution graphs with critical-path-aware credit assignment and adds a runtime controller that eliminates redundant future agent interactions. It halves latency without losing accuracy.", "key_idea": "Optimize the critical path, not just total cost, in MAS.", "relevance_score": 3, "relevance_rationale": "Latency counterpart to MAS cost optimization.", "found_via": "snowball: cited-by zhang-2025-maas-agentic-supernet", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "shim-2026-when2think", "title": "When2Think: Learning When and How Much to Reason", "authors": [ "Jaejun Shim", "HyunJin Kim", "Young Jin Kim", "JinYeong Bak" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.19671", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "cached reference success and token cost" ], "cost_reduction_reported": "On AIME24, +10.0 pp Pass@3 with 27.9% fewer tokens vs the backbone", "benchmarks": [ "AIME24" ], "models_studied": [], "production_evidence": false, "summary": "RLVR post-training that jointly decides whether to think and how much, using instance-level difficulty-aware efficiency bonuses from cached reference statistics.", "key_idea": "Joint control of whether and how much to reason.", "relevance_score": 3, "relevance_rationale": "Adaptive reasoning post-training.", "found_via": "arXiv title search: when to reason", "cites_or_cited_by": [] }, { "id": "shivam-2026-multi-model-agentic-taxonomy", "title": "Architecting Multi-Model Agentic AI Systems: A Taxonomy and Reference Architecture for Specialized LLMs in Production Agents", "authors": [ "Sandeep Shivam", "Milan Parikh", "Venkat Nutalapati", "Vinay R Soni", "Vijaykumar Venganti" ], "year": 2026, "venue_or_source": "ICAISET 2026", "source_type": "paper", "url": "https://doi.org/10.1109/ICAISET66439.2026.11541520", "code_url": null, "topic_clusters": [ "C3", "C15" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes a taxonomy of six model classes (general LLMs, MoE, VLMs, reasoning models, SLMs, large action models) and a four-layer reference architecture with explicit model-to-layer assignments for production agents.", "key_idea": "Reference architecture for heterogeneous model assignment in production agents.", "relevance_score": 3, "relevance_rationale": "Conceptual framework for role-level model assignment.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [] }, { "id": "singh-2026-krites", "title": "Asynchronous Verified Semantic Caching for Tiered LLM Architectures", "authors": [ "Asmit Kumar Singh", "Haozhe Wang", "Laxmi Naga Santosh Attaluri", "Tak Chiam", "Weihua Zhu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.13165", "code_url": null, "topic_clusters": [ "C7", "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "static-cache nearest-neighbor similarity", "asynchronous LLM-judge verification" ], "cost_reduction_reported": "up to 3.9x more requests served with curated static answers for conversational and search-style traffic vs tuned baselines, with unchanged critical-path latency", "benchmarks": [ "trace-driven conversational and search workloads" ], "models_studied": [ "LLM judge" ], "production_evidence": false, "summary": "Krites keeps a standard static-threshold semantic cache on the critical path but, for near-miss prompts, asynchronously asks an LLM judge whether a curated static answer is acceptable; approved matches are promoted into a dynamic cache for future reuse.", "key_idea": "Off-critical-path LLM-judge verification grows the reach of a curated cache safely.", "relevance_score": 3, "relevance_rationale": "Combines judges and caching to raise safe hit rate in tiered LLM deployments.", "found_via": "MeanCache user-centric semantic cache LLM", "cites_or_cited_by": [ "schroeder-2025-vcache" ] }, { "id": "singh-2026-total-cost-of-agency", "title": "Total Cost of Agency: Exact Attribution of Memory Injection Cost in Multi-Agent LLM Workflows", "authors": [ "Vivek Kumar Singh", "Preeti Priyam", "Gautam Bhowmick" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.23790", "code_url": null, "topic_clusters": [ "C10", "C12", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "exact injected-token attribution" ], "cost_reduction_reported": "Memory injection is ~12% of full billed cost, rising to 27.6% at workflow depth six; shrinking retrieval window 32 to 2 cuts injected tokens 28.7%", "benchmarks": [ "200-task enterprise benchmark on real model APIs" ], "models_studied": [], "production_evidence": false, "summary": "Decomposes multi-agent workflow cost into base prompt, inference, memory injection, miss penalty and context accumulation, with an exact two-pass token attribution. On an enterprise benchmark, injected memory is a growing share of cost with workflow depth and is controllable.", "key_idea": "Attribute MAS bills to memory injection and control it.", "relevance_score": 3, "relevance_rationale": "Cost accounting for MAS in production-like settings.", "found_via": "arXiv title search: multi-agent cost", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "singla-2026-agentweave", "title": "AgentWeave: Routing Before Reasoning for Efficient Function Calling in Tool-Rich Language Models", "authors": [ "Saurav Singla", "Aarav Singla", "Advik Gupta", "Parnika Gupta" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.23078", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "eligibility", "capability", "routing signals" ], "cost_reduction_reported": "70.18% fewer tools exposed, 61.70% fewer input tokens and 50.95% lower local-model latency vs all-tools exposure on BFCL V4 tasks", "benchmarks": [ "BFCL V4" ], "models_studied": [ "Hammer2.1-1.5b" ], "production_evidence": false, "summary": "AgentWeave is a deterministic pre-inference layer that cuts the candidate tool set before a function-calling model runs, leaving the downstream model unchanged. With a 1.5B model on 48 BFCL tasks it raises success from 0/48 to 6/48.", "key_idea": "Deterministic tool pre-routing to shrink the action space for small function-calling models.", "relevance_score": 3, "relevance_rationale": "Tool routing as a cost lever; small evaluation.", "found_via": "arXiv query ti:\"routing\" AND ti:\"tool\"", "cites_or_cited_by": [] }, { "id": "smekal-2026-task-spec-token-spend", "title": "Can your AI agent be cheaper? Investigating the effects of task specifications on token spend in agentic coding tasks", "authors": [ "Jakub Smékal" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.25399", "code_url": null, "topic_clusters": [ "C9", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "single cheap probe run", "task specification", "thinking effort" ], "cost_reduction_reported": "Reducing a full task spec to a bare user story raises token spend by 29.7%; predictor prices spec/effort configurations within 36% from one probe", "benchmarks": [ "agentic coding tasks (2,700 runs)" ], "models_studied": [ "Kimi K3" ], "production_evidence": false, "summary": "Measures how task specification detail and thinking effort change token spend in agentic coding across 2,700 runs, finding sparse specs raise spend by about 30% and sensitivity varies by task. A simple predictor estimates spend distributions from a single cheap probe.", "key_idea": "Spec quality drives agent token spend and is predictable from a probe.", "relevance_score": 3, "relevance_rationale": "Token-consumption analysis for coding agents.", "found_via": "web search: token consumption coding agents | web search: SWE-bench cost per task", "cites_or_cited_by": [ "bai-2026-agents-spend-your-money", "kapoor-2024-ai-agents-that-matter" ] }, { "id": "sui-2026-paste", "title": "Parallelizing Tool Execution and LLM Generation for Low-Latency Agent Serving", "authors": [ "Yifan Sui", "Han Zhao", "Rui Ma", "Zhiyuan He", "Hao Wang", "Jianxun Li", "Kaiqiang Xu", "Kai Chen", "Yuqing Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.18897", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "recurring tool-call patterns", "data dependencies" ], "cost_reduction_reported": "average task completion time reduced by 43.5% and observed tool latency lowered 1.8x across deep research, coding and scientific agents", "benchmarks": [ "deep research", "coding", "scientific-agent workloads" ], "models_studied": [], "production_evidence": false, "summary": "PASTE predicts concrete upcoming tool invocations from application-level control-flow patterns and runs them speculatively while the LLM is still generating, isolating results until confirmed and co-scheduling tool execution with LLM sessions.", "key_idea": "Pattern-aware speculative tool execution overlapped with generation.", "relevance_score": 3, "relevance_rationale": "Cuts agent latency by overlapping tools and LLM decoding.", "found_via": "speculative actions agent tool calls parallel execution latency LLM agents", "cites_or_cited_by": [ "xu-2024-conveyor", "ye-2025-speculative-actions" ] }, { "id": "sun-2026-routeguard", "title": "RouteGuard: Certifying Routing Gain in LLM Multi-Agent Systems When Complementarity Is Not Enough", "authors": [ "Anchen Sun", "Kaiqi Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.07583", "code_url": null, "topic_clusters": [ "C1", "C6", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "conditional-regret functional" ], "cost_reduction_reported": null, "benchmarks": [ "RouterBench", "OpenRCA" ], "models_studied": [ "11 cross-family models", "three Gemini advisors" ], "production_evidence": false, "summary": "RouteGuard certifies before deployment whether routing among model-backed advisors in a MAS will yield a real gain, decomposing gain and giving finite-sample certification with matching lower bounds. It withholds certification when gains rest on few workload cells or advisors are redundant.", "key_idea": "Certify routing gain statistically before shipping a router.", "relevance_score": 3, "relevance_rationale": "Guardrail for deciding whether routing is worth deploying.", "found_via": "arXiv title search: routing multi-agent system LLM", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "suwannapichat-2026-rga-designer", "title": "Reward-Guided Autoregressive Graph Generation for Efficient Multi-Agent Communication Topology Design", "authors": [ "Poomphob Suwannapichat", "Boonyarit Changaival", "Caesar Wu", "Pascal Bouvry" ], "year": 2026, "venue_or_source": "ICONIP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2608.20099", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "reward model for correctness and compactness" ], "cost_reduction_reported": "Preserves ARG-Designer accuracy while reducing token consumption by an average of 20.5%", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RGA-Designer fine-tunes an autoregressive topology generator with a reward model that scores both correctness and structural compactness, RLHF-style. It keeps accuracy while using about 20% fewer tokens than ARG-Designer.", "key_idea": "Reward-guided sparsification of generated MAS topologies.", "relevance_score": 3, "relevance_rationale": "Incremental topology-cost work.", "found_via": "web search: agent topology pruning", "cites_or_cited_by": [] }, { "id": "talluri-2026-rgao-topology-budget", "title": "Retrieval-Conditioned Topology Selection with Provable Budget Conservation for Multi-Agent Code Generation", "authors": [ "Abhijit Talluri", "Pujith Anne", "Bhagavan Choudary Pendiyala", "Raghavendra Chilukuri" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.05657", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "structural code complexity vector", "budget vectors" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RGAO extracts a structural-complexity vector from a code index before choosing the orchestration topology for a multi-agent coding system, whose subagents operate under formal six-dimensional budget contracts. The claimed contribution is provable budget conservation under dynamic topology selection.", "key_idea": "Complexity-conditioned topology routing with formal subagent budgets.", "relevance_score": 3, "relevance_rationale": "Budgeted subagent orchestration; mainly a formal/architectural contribution.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "tan-2026-atod", "title": "ATOD: Annealed Turn-aware On-policy Distillation for Multi-turn Autonomous Agents", "authors": [ "Qitai Tan", "Zefang Zong", "Yang Li", "Peng Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.27814", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "annealed on-policy distillation plus RL" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Anneals from on-policy distillation early to RL later for training small multi-turn agents, with turn-aware weighting.", "key_idea": "Schedule OPD then RL for small agents.", "relevance_score": 3, "relevance_rationale": "Training recipe for small agents.", "found_via": "hf-papers search: on-policy distillation agents", "cites_or_cited_by": [] }, { "id": "tang-2026-jisi", "title": "Beyond Gemini-3-Pro: Revisiting LLM Routing and Aggregation at Scale", "authors": [ "Shengji Tang", "Weihao Lin", "Peng Ye", "Jingqi Ye", "Hao Li", "Yiqun Zhang", "Xiaosong Wang", "Bo Zhang", "Shuyue Hu", "Tao Chen", "Lei Bai", "Wanli Ouyang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.01330", "code_url": null, "topic_clusters": [ "C1", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query-response mixed routing", "support-set aggregator selection" ], "cost_reduction_reported": "Surpasses Gemini-3-Pro with only 47% of the cost by orchestrating ten open-source LLMs (abstract)", "benchmarks": [ "nine benchmarks" ], "models_studied": [ "Gemini-3-Pro", "ten open-source LLMs" ], "production_evidence": false, "summary": "JiSi combines routing that uses both semantics and difficulty, support-set-based aggregator selection, and an adaptive switch between routing and aggregation to beat Gemini-3-Pro with open models.", "key_idea": "Adaptive routing-aggregation switch.", "relevance_score": 3, "relevance_rationale": "Cost vs frontier comparison.", "found_via": "search: query difficulty LLM routing", "cites_or_cited_by": [] }, { "id": "team-2026-dr-venus", "title": "DR-Venus: Towards Frontier Edge-Scale Deep Research Agents with Only 10K Open Data", "authors": [ "Venus Team", "Sunhao Dai", "Yong Deng", "Jinzhen Lin", "Yusheng Song", "Guoqing Wang", "Xiaofeng Wu", "Yuqi Zhou", "Shuo Yang", "Zhenzhe Ying", "Zhanwei Zhang", "Changhua Meng", "Weiqiang Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.19859", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "10K open data" ], "cost_reduction_reported": null, "benchmarks": [ "deep research benchmarks" ], "models_studied": [ "4B model" ], "production_evidence": false, "summary": "Trains a 4B edge-scale deep-research agent from about 10K open-data samples, beating prior sub-9B agentic models and narrowing the gap to 30B systems.", "key_idea": "Strong small research agent from little data.", "relevance_score": 3, "relevance_rationale": "Small-agent feasibility evidence.", "found_via": "hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [] }, { "id": "tian-2026-pinterest-decision-quality", "title": "Decision Quality Evaluation Framework at Pinterest", "authors": [ "Yuqi Tian", "Robert Paine", "Attila Dobi", "Kevin O'Sullivan", "Aravindh Manickavasagam", "Faisal Farooq" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.15809", "code_url": null, "topic_clusters": [ "C6", "C13", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "expert golden set", "propensity-score sampling" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Describes a deployed framework at Pinterest for evaluating human and LLM moderation decisions, built around an expert golden set expanded with propensity-score-based sampling. Used to benchmark cost-performance trade-offs among LLM agents, optimize prompts and track policy evolution.", "key_idea": "Production evaluation pipeline using propensity sampling to compare LLM agents on cost and quality.", "relevance_score": 3, "relevance_rationale": "Production example of logged-data evaluation infrastructure for choosing LLM configurations.", "found_via": "query: off-policy evaluation prompt policy", "cites_or_cited_by": [] }, { "id": "tiwari-2026-cachewise", "title": "CacheWise: Understanding Workloads and Optimizing KVCache Management for Efficiently Serving LLM Coding Agents", "authors": [ "Shubham Tiwari", "Tapan Chugh", "Nash Rickert", "Simon Peter", "Ratul Mahajan", "Haiying Shen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.16824", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "tool-call metadata predictions", "prefix awareness" ], "cost_reduction_reported": "KV cache evictions reduced 2-2.6x and total agent session completion time improved up to 3.5x on collected coding-assistant traces", "benchmarks": [ "real-world coding assistant traces" ], "models_studied": [], "production_evidence": false, "summary": "Collects real coding-assistant traces showing repeated large prefixes and sustained KV pressure, then combines prefix-aware scheduling with reuse-aware eviction guided by predictions from tool-call metadata in vLLM.", "key_idea": "Use tool-call metadata to predict KV reuse in coding-agent serving.", "relevance_score": 3, "relevance_rationale": "Coding-agent-specific KV management with trace-based evaluation.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "li-2025-continuum" ] }, { "id": "tolochinsky-2026-valid-best-model-identification", "title": "Valid Best-Model Identification for LLM Evaluation via Low-Rank Factorization", "authors": [ "Elad Tolochinsky", "Yaniv Tenzer", "Yaniv Romano" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.10405", "code_url": null, "topic_clusters": [ "C5", "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "low-rank predicted scores", "doubly robust estimates" ], "cost_reduction_reported": "Reduces the number of required evaluations while still identifying the best model (qualitative)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Combines multi-armed bandit evaluation with low-rank predictions of the model-example score matrix, using doubly robust estimators to keep finite-sample confidence intervals valid while selecting the best model with fewer evaluations.", "key_idea": "Bandit best-model identification with doubly robust use of predicted scores.", "relevance_score": 3, "relevance_rationale": "Cheap, valid model selection when onboarding new models.", "found_via": "query: doubly robust estimator LLM evaluation", "cites_or_cited_by": [] }, { "id": "vllm-2026-agentx-agentic-serving", "title": "vLLM x AgentX: Optimizing for Real-World Agentic Serving", "authors": [ "vLLM Team", "Inferact" ], "year": 2026, "venue_or_source": "vLLM blog", "source_type": "blog", "url": "https://blog.vllm.ai/blog/2026-09-08-vllm-agentx", "code_url": "https://github.com/vllm-project/vllm", "topic_clusters": [ "C10", "C11", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Up to 130K total tokens per GPU-second on DeepSeek V4 Pro; 14.6x-106x serving-cost advantage over Opus 5 API pricing across DeepSeek V4 Pro, MiniMax M3, Kimi K3 on SemiAnalysis AgentX (project-reported)", "benchmarks": [ "SemiAnalysis AgentX" ], "models_studied": [ "DeepSeek V4 Pro", "MiniMax M3", "Kimi K3" ], "production_evidence": false, "summary": "Characterises agentic coding traffic using AgentX, a public benchmark built from real agentic coding traces: median 43 turns per session, median 142K input vs 444 output tokens, prefix-cache hit rate above 96%, and 44% of sessions spawning subagents. Describes vLLM's KV-cache management, hierarchical offloading, context parallelism and prefill/decode disaggregation choices for this workload and reports tokens-per-dollar against interactivity.", "key_idea": "Agentic workloads are prefill- and cache-dominated; serving must optimise the latency-cost frontier around prefix reuse.", "relevance_score": 3, "relevance_rationale": "Serving-layer cost lever with workload statistics from real traces; self-hosting cost comparison.", "found_via": "direct: blog.vllm.ai index", "cites_or_cited_by": [ "liu-2026-vllm-sr-session-aware-agentic-routing", "vllm-2026-tiered-kv-offloading" ] }, { "id": "wadlom-2026-helium", "title": "Efficient LLM Serving for Agentic Workflows: A Data Systems Perspective", "authors": [ "Noppanat Wadlom", "Junyi Shen", "Yao Lu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.16104", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "proactive caching", "cache-aware scheduling" ], "cost_reduction_reported": "up to 1.56x speedup over state-of-the-art agent serving systems", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Treats LLM invocations as first-class query operators and applies proactive caching and cache-aware scheduling across prompts, KV states and workflows.", "key_idea": "Data-systems view of agent serving.", "relevance_score": 3, "relevance_rationale": "End-to-end workflow reuse.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "shen-2025-halo" ] }, { "id": "wang-2026-amro-s-ant-colony-routing", "title": "Efficient and Interpretable Multi-Agent LLM Routing via Ant Colony Optimization", "authors": [ "Xudong Wang", "Chaoning Zhang", "Jiaquan Zhang", "Chenghao Li", "Qigan Sun", "Sung-Ho Bae", "Peng Wang", "Ning Xie", "Jie Zou", "Yang Yang", "Hengtao Shen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.12933", "code_url": null, "topic_clusters": [ "C3", "C5" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "SFT small-LM intent inference", "pheromone memory", "quality gate" ], "cost_reduction_reported": "Improves quality-cost trade-off over strong routing baselines (no single number in abstract)", "benchmarks": [ "five public benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "AMRO-S treats MAS routing as semantic-conditioned path selection, using a fine-tuned small LM for intent, task-specific ant-colony pheromone memories, and quality-gated asynchronous updates that learn without adding latency. It improves the quality-cost trade-off with interpretable routing traces.", "key_idea": "Ant-colony path selection over agents with asynchronous learning.", "relevance_score": 3, "relevance_rationale": "Online-updated MAS routing.", "found_via": "OpenAlex search: heterogeneous LLMs multi-agent", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "wang-2026-budgeted-act-or-defer", "title": "Budgeted Act-or-Defer Multi-Agent LLM Deliberation with Local Reliability Bounds", "authors": [ "Mengdie Flora Wang", "Haochen Xie", "Guanghui Wang", "Devin Zhang", "Jae Oh Woo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.29654", "code_url": null, "topic_clusters": [ "C2", "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "debate-prefix state", "kNN lower confidence bound" ], "cost_reduction_reported": "Uses 9-12% of the pre-declared wrong-action budget, reaching up to 84% automation and 96% acted-on accuracy on six benchmarks", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Frames multi-agent debate deployment as deciding, round by round, whether to act or defer to human review under a user-declared wrong-action budget. A kNN lower confidence bound on correctness, computed from calibration data, sets the act threshold with stated assumptions and diagnostics.", "key_idea": "Per-round act-or-defer with a declared risk budget for multi-agent deliberation.", "relevance_score": 3, "relevance_rationale": "Budgeted per-round stopping in MAS; deferral is to humans, not to models.", "found_via": "arXiv query ti:\"budget\" AND ti:agent AND ti:llm", "cites_or_cited_by": [] }, { "id": "wang-2026-causalrm", "title": "CausalRM: Causal-Theoretic Reward Modeling for RLHF from Observational User Feedbacks", "authors": [ "Hao Wang", "Licheng Pan", "Zhichao Chen", "Chunyuan Zheng", "Zhixuan Chu", "Xiaoxi Li", "Yuan Lu", "Xinggao Liu", "Haoxuan Li", "Zhouchen Lin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.18736", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "clicks", "copies", "upvotes" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defines observational reward modeling from user feedback such as clicks, copies and upvotes and identifies two problems: label noise and self-selection of which responses receive feedback. CausalRM adds a noise-aware surrogate loss and propensity-style reweighting to learn unbiased reward models.", "key_idea": "Debias reward models trained on observational user feedback with causal corrections.", "relevance_score": 3, "relevance_rationale": "Selection-bias correction is needed to train routing quality predictors from production thumbs data.", "found_via": "query: counterfactual learning router logged bandit feedback", "cites_or_cited_by": [] }, { "id": "wang-2026-confidently-wrong-finqa", "title": "Confidently Wrong: Detecting Hallucinations in Financial Question Answering from LLM Internal States", "authors": [ "Richard Zhe Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.11414", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state probe" ], "cost_reduction_reported": null, "benchmarks": [ "FinQA" ], "models_studied": [ "Qwen3-8B", "Llama-3.1-8B", "Gemma-2-9B" ], "production_evidence": false, "summary": "Shows 15-23% of fully self-consistent answers are wrong on FinQA and that hidden-state probes detect such confident errors (0.68-0.77 AUROC vs 0.55-0.63 baselines), proposing probes for routing to human review.", "key_idea": "Probes catch confident errors that consistency misses.", "relevance_score": 3, "relevance_rationale": "Probe-based escalation in an enterprise domain.", "found_via": "WebSearch: probes hidden states LLM correctness prediction routing escalation", "cites_or_cited_by": [] }, { "id": "wang-2026-forecasting-agents-reliability-routing", "title": "When Should Forecasting Agents Reason? Behavioral Stress Tests for Reliability Routing", "authors": [ "Yufeng Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.28475", "code_url": "https://github.com/louiswang524/forcastagent", "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "historical coverage", "market prior", "evidence strength", "horizon" ], "cost_reduction_reported": null, "benchmarks": [ "ForecastBench-style binary forecasting" ], "models_studied": [ "16 LLM vintages" ], "production_evidence": false, "summary": "Routes forecasting-agent behaviour between retrieval, reasoning, market priors and historical analogs using reliability features; gains are modest, and more reasoning is not always better.", "key_idea": "Route agent behaviours by evidence-source reliability.", "relevance_score": 3, "relevance_rationale": "Behaviour-level routing in agents.", "found_via": "arXiv title search: when to reason", "cites_or_cited_by": [] }, { "id": "wang-2026-forkkv", "title": "ForkKV: Scaling Multi-LoRA Agent Serving via Copy-on-Write Disaggregated KV Cache", "authors": [ "Shao Wang", "Rui Ren", "Lin Gui" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.06370", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "copy-on-write shared KV" ], "cost_reduction_reported": "up to 3.0x throughput of state-of-the-art multi-LoRA serving systems with negligible quality impact", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Copy-on-write disaggregated KV cache for multi-LoRA agents: agents share a base cache and keep only their small unique parts, with a kernel reconstructing KV in on-chip SRAM.", "key_idea": "Fork-style KV sharing for multi-LoRA agents.", "relevance_score": 3, "relevance_rationale": "Multi-adapter agent serving efficiency.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "jeon-2026-lragent" ] }, { "id": "wang-2026-heterogeneous-vuln-detection", "title": "Strategic Heterogeneous Multi-Agent Architecture for Cost-Effective Code Vulnerability Detection", "authors": [ "Zhaohui Geoffrey Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.21282", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "77.2% F1 at $0.002 per sample vs single-expert LLM F1 71.4% (262 NIST Juliet samples)", "benchmarks": [ "NIST Juliet Test Suite" ], "models_studied": [ "DeepSeek-V3", "Qwen3-8B" ], "production_evidence": false, "summary": "A \"3+1\" architecture runs three cloud DeepSeek-V3 experts with complementary perspectives in parallel and a local Qwen3-8B verifier that adversarially validates at zero marginal cost.", "key_idea": "Cloud experts plus a local small verifier.", "relevance_score": 3, "relevance_rationale": "Role-level big/small split with cost per sample.", "found_via": "arXiv query abs:\"multi-agent\" AND abs:\"heterogeneous\" AND abs:\"cost\" AND abs:\"LLMs\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "wang-2026-implicitrm", "title": "ImplicitRM: Unbiased Reward Modeling from Implicit Preference Data for LLM alignment", "authors": [ "Hao Wang", "Haochen Yang", "Li-Cheng Pan", "Lei Shen", "Xiaoxi Li", "Yi-Nuo Wang", "Zhichao Chen", "Yuan Lu", "Hao-Xuan Li", "Zhou-Chen Lin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.23184", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "implicit clicks and copies" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Learns reward models from implicit feedback that has no definite negatives and whose propensity to be observed varies by response. Stratifies samples into latent groups and derives an unbiased learning objective.", "key_idea": "Unbiased reward learning from positive-only, propensity-biased implicit feedback.", "relevance_score": 3, "relevance_rationale": "Same issue arises when using production signals to train routers.", "found_via": "query: implicit user feedback LLM production", "cites_or_cited_by": [] }, { "id": "wang-2026-latency-aware-mas-gpus", "title": "Latency-Aware Orchestration for Multi-Agent LLM Workflows on Heterogeneous GPUs", "authors": [ "Jinghao Wang", "Yifeng Zhang", "Xiao Zhou", "Yao Lu", "Yihui Zhang", "Xiaoyang Sun", "Tianyu Wo", "Xu Wang", "Chunming Hu", "Renyu Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.03335", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "predicted activation latency", "memory", "model-loading cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A prediction-guided runtime for concurrent multi-agent workflows on heterogeneous GPU pools: it forecasts per-device latency, memory and model-loading cost through workflow dependencies, then jointly chooses fusion, placement and order.", "key_idea": "Workflow-aware placement and scheduling of multi-model agents on heterogeneous GPUs.", "relevance_score": 3, "relevance_rationale": "Serving support for multi-model agents.", "found_via": "arXiv query ti:heterogeneous AND ti:\"multi-agent\" AND ti:llm", "cites_or_cited_by": [] }, { "id": "wang-2026-random-routing-sparse-debate", "title": "Do We Need Complex Topology Control? Distinct-Peer Random Routing Improves Cost-Efficiency in Sparse Multi-Agent Debate", "authors": [ "Boxuan Wang", "Zhuoyun Li", "Xiaowei Huang", "Yi Dong" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.27150", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Finds that a simple random-without-replacement peer routing, where each agent debates two new distinct peers per round, is a strong baseline that improves the accuracy-cost trade-off of sparse debate, and that light stopping rules cut cost further. Argues learned topology control must beat these simple baselines.", "key_idea": "Simple random sparse routing plus stopping rivals learned topologies.", "relevance_score": 3, "relevance_rationale": "Strong cheap baseline for MAS topology work.", "found_via": "arXiv title search: agent routing", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer" ] }, { "id": "wang-2026-scale-query-level-workflows", "title": "Do We Always Need Query-Level Workflows? Rethinking Agentic Workflow Generation for Multi-Agent Systems", "authors": [ "Zixu Wang", "Bingbing Xu", "Yige Yuan", "Huawei Shen", "Xueqi Cheng" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.11147", "code_url": null, "topic_clusters": [ "C4", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "self-predicted evaluation with few-shot calibration" ], "cost_reduction_reported": "Average 0.61% degradation while cutting token usage by up to 83%", "benchmarks": [ "multiple datasets" ], "models_studied": [], "production_evidence": false, "summary": "Shows that a small set of top task-level workflows covers as many queries as per-query workflow generation, and that exhaustive execution-based evaluation is costly and unreliable. SCALE replaces full validation runs with calibrated self-prediction, cutting optimization tokens up to 83%.", "key_idea": "Per-query workflow generation is often unnecessary; evaluate cheaply.", "relevance_score": 3, "relevance_rationale": "Questions the cost of query-level MAS design.", "found_via": "web search: MAS generation cost", "cites_or_cited_by": [ "zhang-2025-maas-agentic-supernet" ] }, { "id": "wang-2026-signed-rescue-routing", "title": "Signed Rescue Routing: Harm-Aware Cascades for Efficient LLM Inference", "authors": [ "Zheyuan Wang", "Siyu Li", "Peiqiao Song", "Sijia Chen", "Qianqian Song", "Qian Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.07786", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted rescue vs harm probabilities" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU", "HellaSwag", "ARC-Challenge" ], "models_studied": [ "Qwen3-4B", "Qwen3-8B" ], "production_evidence": false, "summary": "Argues escalation helps only when the large model fixes the small model and hurts when it breaks a correct answer, so it trains a two-head router to predict both and ranks requests by their difference, which is Bayes-optimal under a fixed escalation budget. The arXiv abstract still contains placeholder numbers.", "key_idea": "Route on signed expected gain of escalation, not on uncertainty.", "relevance_score": 3, "relevance_rationale": "Useful objective reformulation; results incomplete in current version.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "wang-2026-strong-weak-hmas", "title": "Guided Collaboration in Heterogeneous LLM-Based Multi-Agent Systems via Entropy-Based Understanding Assessment and Experience Retrieval", "authors": [ "Linlin Wang", "Tianqing Zhu", "Laiqiao Qin", "Longxiang Gao", "Wanlei Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.13639", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "multi-dimensional entropy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "In heterogeneous MAS, strong-weak pairings can underperform weak-weak pairings because of cognitive mismatch. The authors propose entropy-based assessment of the weak agent's understanding to adapt guidance, plus experience retrieval.", "key_idea": "Strong-weak agent pairs need adaptive guidance to beat homogeneous teams.", "relevance_score": 3, "relevance_rationale": "Evidence on pitfalls of mixing big and small models across roles.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [ "ye-2025-x-mas" ] }, { "id": "wang-2026-swe-pruner", "title": "SWE-Pruner: Self-Adaptive Context Pruning for Coding Agents", "authors": [ "Yuhang Wang", "Yuling Shi", "Mo Yang", "Rongrui Zhang", "Shilin He", "Heng Lian", "Yuting Chen", "Siyu Ye", "Kai Cai", "Xiaodong Gu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.16746", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "goal hint", "0.6B neural skimmer" ], "cost_reduction_reported": "23-54% token reduction on agent tasks such as SWE-Bench Verified while improving success rates; up to 14.84x compression on LongCodeQA", "benchmarks": [ "SWE-Bench Verified", "LongCodeQA" ], "models_studied": [ "0.6B skimmer" ], "production_evidence": false, "summary": "The coding agent states a goal for what it needs from surrounding code, and a small trained skimmer selects relevant lines, pruning long code contexts adaptively.", "key_idea": "Goal-conditioned line-level pruning by a tiny model.", "relevance_score": 3, "relevance_rationale": "Cheap small-model pruning of tool outputs in coding agents.", "found_via": "SWE-agent context management trajectory reduction cost coding agents arXiv", "cites_or_cited_by": [ "kovacs-2026-squeez", "xiao-2025-agentdiet-trajectory-reduction" ] }, { "id": "wei-2026-triage-trajectory-reuse", "title": "TRIAGE: Three-level Routing and Intelligent Agent Guidance for Efficient Execution", "authors": [ "Ruocan Wei" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.01428", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query similarity to stored trajectories" ], "cost_reduction_reported": "62.3% token savings on 1,007 security monitoring queries; 76.3% token reduction on ToolBench", "benchmarks": [ "1,007 security monitoring queries", "ToolBench (345 queries)" ], "models_studied": [], "production_evidence": false, "summary": "TRIAGE routes ReAct queries into three levels: exact reuse of a stored trajectory, skill substitution for similar queries via deterministic parameter substitution, or full ReAct for novel ones, storing results for reuse. Token cost falls sharply as the skill store fills.", "key_idea": "Reuse past trajectories as skills to skip LLM reasoning.", "relevance_score": 3, "relevance_rationale": "Plan/trajectory caching as an agent cost lever.", "found_via": "arXiv search: agent model routing cost step", "cites_or_cited_by": [] }, { "id": "white-2026-scepsy", "title": "Scepsy: Serving Agentic Workflows Using Aggregate LLM Pipelines", "authors": [ "Otto White", "Marcel Wagenländer", "Britannio Jarrett", "Xijin Zhou", "Yanda Tao", "Pedro Silvestre", "Guo Li", "Huanzhou Zhu", "Lluís Vilanova", "Peter Pietzuch" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.15186", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "aggregate LLM pipeline model", "GPU share/TP/replica search" ], "cost_reduction_reported": "up to 2.5x higher throughput before saturation and 1.0-3.3x lower latency than systems optimizing LLMs independently or using user-specified allocations", "benchmarks": [ "realistic agentic workflows" ], "models_studied": [], "production_evidence": false, "summary": "Schedules multi-LLM agentic workflows on a GPU cluster by modelling them as an aggregate pipeline and searching fractional GPU shares, tensor parallelism and replica counts, then placing the allocation to reduce fragmentation.", "key_idea": "Allocate GPUs across the multiple models of a workflow jointly.", "relevance_score": 3, "relevance_rationale": "Multi-model serving for agent workflows.", "found_via": "serving engine LLM agents agentic workflows scheduling arXiv 2025", "cites_or_cited_by": [ "chaudhry-2025-murakkab" ] }, { "id": "woo-2026-prefillshare", "title": "PrefillShare: A Shared Prefill Module for KV Reuse in Multi-LLM Disaggregated Serving", "authors": [ "Sunghyeon Woo", "Hoseung Kim", "S. Shim", "Minjung Jo", "Hyunjoon Jeong", "Jeongtae Lee", "Joonghoon Kim", "Sungjae Lee", "Jiwon Ryu", "Baeseong Park", "S. Kwon", "Dongsoo Lee" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.12029", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "PrefillShare lets multiple fine-tuned task-specific models in a multi-agent system share one frozen prefill module and its KV cache over a shared context in disaggregated serving, fine-tuning only decode modules.", "key_idea": "Share prefill/KV across specialized models in multi-agent serving.", "relevance_score": 3, "relevance_rationale": "Systems lever that makes multi-model agent pipelines cheaper.", "found_via": "snowball: cited-by su-2025-toolorchestra", "cites_or_cited_by": [ "su-2025-toolorchestra" ] }, { "id": "wu-2026-assign-tasks-capacity", "title": "Learning to Assign Prediction Tasks to Agents with Capacity Constraints", "authors": [ "Shanglin Wu", "Saatvik Kher", "Padhraic Smyth" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.27999", "code_url": null, "topic_clusters": [ "C2", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "task context", "observed agent outcomes" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "LLMs", "human decision-makers" ], "production_evidence": false, "summary": "Studies sequential learning of which agent (human or AI model) should handle each prediction task when each agent can take only a fraction of tasks. Gives a theoretical characterization and explore-exploit policy-learning algorithms that beat non-contextual baselines on tabular, image and text tasks.", "key_idea": "Contextual explore-exploit assignment of tasks to capacity-limited agents.", "relevance_score": 3, "relevance_rationale": "Capacity-constrained online routing, as with rate-limited frontier models.", "found_via": "snowball: cited-by taberner-miller-2026-paretobandit", "cites_or_cited_by": [ "taberner-miller-2026-paretobandit" ] }, { "id": "wu-2026-card-conditional-topology", "title": "CARD: Towards Conditional Design of Multi-agent Topological Structures", "authors": [ "Tongtong Wu", "Yanming Li", "Ziye Tang", "Chen Jiang", "Linhao Luo", "Guilin Qi", "Shirui Pan", "Gholamreza Haffari" ], "year": 2026, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2603.01089", "code_url": "https://github.com/Warma10032/CARD", "topic_clusters": [ "C3" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "environment signals (model capability, resource availability)" ], "cost_reduction_reported": null, "benchmarks": [ "HumanEval", "MATH", "MMLU" ], "models_studied": [], "production_evidence": false, "summary": "CARD generates multi-agent communication topologies conditioned on environmental signals such as model upgrades, tool changes or resource availability, using a conditional variational graph encoder. Topologies adapt at training and run time and stay robust when model capabilities shift.", "key_idea": "Condition MAS topology generation on the current model/resource environment.", "relevance_score": 3, "relevance_rationale": "Topology adaptation to changing model pools.", "found_via": "web search: G-Designer follow-ups", "cites_or_cited_by": [ "zhang-2024-g-designer" ] }, { "id": "wu-2026-differentiable-moa", "title": "Differentiable Mixture-of-Agents Incentivizes Swarm Intelligence of Large Language Models", "authors": [ "Xingjian Wu", "Junkai Lu", "Siyu Yan", "Xiangfei Qiu", "Jilin Hu", "Chenjuan Guo", "Bin Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.15706", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "predictive entropy", "recurrent context" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "DMoA dynamically activates a sparse subset of agents at each reasoning step through a differentiable, context-aware router, implicitly simulating many communication topologies. Predictive entropy serves as a self-supervised signal for test-time adaptation.", "key_idea": "Step-wise sparse agent activation via a differentiable router.", "relevance_score": 3, "relevance_rationale": "Step-level agent routing in MoA; efficiency claims are qualitative in the abstract.", "found_via": "snowball: cited-by lee-2025-steer", "cites_or_cited_by": [ "lee-2025-steer" ] }, { "id": "wu-2026-faq-efficient-evaluation", "title": "Efficient Evaluation of LLM Performance with Statistical Guarantees", "authors": [ "Skyler Wu", "Yash Nair", "Emmanuel J. Candès" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.20251", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "historical eval data", "Bayesian factor model" ], "cost_reduction_reported": "Up to 5x effective sample size gains; matches uniform-sampling CI width with up to 5x fewer queries", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Treats benchmarking as finite-population inference and selects questions adaptively using a Bayesian factor model on historical results, while keeping valid confidence intervals through a finite-population active inference extension.", "key_idea": "Active, statistically valid LLM evaluation reusing historical results.", "relevance_score": 3, "relevance_rationale": "Cuts the cost of evaluating candidate models or configs before swapping them.", "found_via": "query: active evaluation LLM selection budget", "cites_or_cited_by": [] }, { "id": "wu-2026-routerxbench", "title": "Towards Fair and Comprehensive Evaluation of Routers in Collaborative LLM Systems", "authors": [ "Wanxing Wu", "He Zhu", "Yixia Li", "Lei Yang", "Jiehui Zhao", "Hongru Wang", "Jian Yang", "Benyou Wang", "Bingyi Jing", "Guanhua Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.11877", "code_url": null, "topic_clusters": [ "C1", "C7", "C13" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden states", "Dirichlet probe" ], "cost_reduction_reported": "16.68% and 18.86% relative improvements over best baselines in router ability and high-accuracy scenarios (abstract)", "benchmarks": [ "RouterXBench" ], "models_studied": [], "production_evidence": false, "summary": "Proposes RouterXBench for evaluating local-vs-cloud routers on ability, scenario alignment and robustness, and ProbeDirichlet, which routes from internal hidden states aggregated with learnable Dirichlet distributions.", "key_idea": "Hidden-state probe routers and a fairer benchmark.", "relevance_score": 3, "relevance_rationale": "Evaluation and signals.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "wu-2026-tiny-critic-rag", "title": "Tiny-Critic RAG: Empowering Agentic Fallback with Parameter-Efficient Small Language Models", "authors": [ "Yichao Wu", "Penghao Liang", "Yafei Xiang", "Mengwei Yuan", "Jianan Liu", "Jing Yang", "Xianyou Li", "Weiran Yan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.00846", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "LoRA SLM binary gate" ], "cost_reduction_reported": "Routing accuracy comparable to GPT-4o-mini with an order of magnitude lower latency (noise-injected datasets)", "benchmarks": [], "models_studied": [ "GPT-4o-mini" ], "production_evidence": false, "summary": "Tiny-Critic RAG replaces a large LLM evaluator in agentic RAG with a LoRA-tuned small model that acts as a fast binary gatekeeper over retrieval quality, triggering fallback only when needed.", "key_idea": "Small LoRA critic as the routing gate in agentic RAG.", "relevance_score": 3, "relevance_rationale": "Small-model gate for a step in an agentic pipeline.", "found_via": "arXiv query ti:\"agentic\" AND ti:\"efficient\" AND ti:\"small language models\"", "cites_or_cited_by": [] }, { "id": "wunderlich-2026-multi-agent-pareto-tts", "title": "Multi-Agent Reasoning Improves Compute Efficiency: Pareto-Optimal Test-Time Scaling", "authors": [ "Florian Valentin Wunderlich", "Lars Benedikt Kaesberg", "Jan Philip Wahle", "Terry Ruas", "Bela Gipp" ], "year": 2026, "venue_or_source": "ACL 2026 SRW", "source_type": "paper", "url": "https://arxiv.org/abs/2605.01566", "code_url": null, "topic_clusters": [ "C9", "C12", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "number of agents, rounds, samples" ], "cost_reduction_reported": "Inference scaling gives up to +7.1 pp over CoT at 20x CoT compute on MMLU-Pro; at equal budget, debate and mixture-of-agents beat self-consistency by 1.3 and 2.7 pp", "benchmarks": [ "MMLU-Pro", "BBH" ], "models_studied": [], "production_evidence": false, "summary": "Pareto analysis of self-consistency, self-refinement, debate and mixture-of-agents across 34 configurations. Multi-agent methods keep improving where self-consistency saturates.", "key_idea": "Compute-matched Pareto fronts of multi-agent inference strategies.", "relevance_score": 3, "relevance_rationale": "Evidence on when multi-agent compute is worth it.", "found_via": "arXiv title search: efficient reasoning agent", "cites_or_cited_by": [] }, { "id": "xia-2026-mori", "title": "Idleness is Relative: Exploiting Tool-Call Idle Windows for Offloading in Agentic Systems with MORI", "authors": [ "Tian Xia", "Hanchen Li", "Zhifei Li", "Xiaokun Chen", "Hao Kang", "Yifan Qiao", "Yi Xu", "Ion Stoica" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.00866", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "relative idleness ranking" ], "cost_reduction_reported": "20-71% higher throughput and 18-43% lower TTFT than the best offloading baseline on Claude Code agent traces", "benchmarks": [ "coding agent workloads collected from Claude Code" ], "models_studied": [], "production_evidence": false, "summary": "Treats tool-call idleness as a continuous spectrum: ranks active agent programs by idleness, keeps the busiest in GPU memory and offloads the most idle to CPU DRAM, adjusting the boundary to hardware capacity.", "key_idea": "Tier KV memory by relative idleness of agent programs.", "relevance_score": 3, "relevance_rationale": "Tool-call idle-window exploitation for agent serving.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "bian-2025-tokencake", "li-2025-continuum", "liu-2026-copilot-traces" ] }, { "id": "xu-2026-agent-skills-slm-industrial", "title": "Agent Skill Framework: Perspectives on the Potential of Small to Medium Language Models in Industrial Environments", "authors": [ "Yangjie Xu", "Lujun Li", "Lama Sleem", "Niccolò Gentile", "Ye-Wei Song", "Yiqun Wang", "Siming Ji", "Wenbo Wu", "Radu State" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.16653", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "two open-source tasks", "insurance claims classification" ], "models_studied": [ "270M-80B open models" ], "production_evidence": false, "summary": "Studies whether agent Skills work with small and medium open models in resource-constrained industrial settings. Very small models struggle with skill selection, models around 30B-80B benefit, and thinking variants add GPU cost without much gain.", "key_idea": "Skills need ~30B+ models; very small models fail at skill selection.", "relevance_score": 3, "relevance_rationale": "Sizing guidance for open models in industrial agents.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "xuan-2026-confidence-dichotomy-tool-agents", "title": "The Confidence Dichotomy: Analyzing and Mitigating Miscalibration in Tool-Use Agents", "authors": [ "Weihao Xuan", "Qingcheng Zeng", "Heli Qi", "Yunze Xiao", "Junjue Wang", "Naoto Yokoya" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.07264", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "verbalized confidence", "calibration" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Analyzes miscalibration of tool-use agents' verbalized confidence across multi-turn tasks and proposes mitigation.", "key_idea": "Tool-use changes agent calibration.", "relevance_score": 3, "relevance_rationale": "Caveat on verbalized confidence in agents.", "found_via": "hf-papers search: verbalized confidence calibration", "cites_or_cited_by": [] }, { "id": "yan-2026-zerorouter", "title": "Breaking Model Lock-in: Cost-Efficient Zero-Shot LLM Routing via a Universal Latent Space", "authors": [ "Cheng Yan", "Wuyang Zhang", "Zhiyuan Ning", "Fan Xu", "Ziyang Tao", "Lu Zhang", "Bing Yin", "Yanyong Zhang" ], "year": 2026, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.06220", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "universal latent difficulty space", "context-aware predictor" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ZeroRouter decouples query characterization from model profiling via a model-agnostic latent difficulty space, allowing zero-shot onboarding of new models without full retraining, and optimizes accuracy, cost and latency.", "key_idea": "Universal latent query-difficulty space for zero-shot model onboarding.", "relevance_score": 3, "relevance_rationale": "Model-pool churn.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "yang-2026-fusion-moa-fleet", "title": "The Fleet Is the Model: Engineering Collective Intelligence with Fusion-MoA Pioneer R1", "authors": [ "Zong-You Yang", "Ying-Han Hou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.26080", "code_url": null, "topic_clusters": [ "C3", "C11", "C14" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "HMMT P1-P10", "Terminal-Bench 2.1" ], "models_studied": [], "production_evidence": false, "summary": "Fusion-MoA exposes a governed fleet of heterogeneous model \"Cells\" as one OpenAI-compatible model, with read-only Analysts and a single Executor holding tool authority. Members can be promoted or rolled back without changing the API. The collective solves 8/10 HMMT problems vs 6/10 for the best single Cell.", "key_idea": "Operate a heterogeneous model fleet as one governed, evolvable service.", "relevance_score": 3, "relevance_rationale": "Operational architecture for heterogeneous MoA; no cost numbers.", "found_via": "snowball: cited-by belcak-2025 / yue-2025-masrouter", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai", "yue-2025-masrouter" ] }, { "id": "yang-2026-star-agent-router", "title": "STAR: Failure-Aware Markovian Routing for Multi-Agent Spatiotemporal Reasoning", "authors": [ "Ruiyi Yang", "Lihuan Li", "Hao Xue", "Flora D. Salim" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.10057", "code_url": null, "topic_clusters": [ "C3", "C6" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "typed execution status", "failure state" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "STAR externalizes routing among heterogeneous specialist agents as a transition matrix conditioned on the current agent, task type and typed failure status. Nominal routes come from experts, and recovery transitions are learned from execution traces.", "key_idea": "Failure-aware inter-agent routing matrix learned from execution traces.", "relevance_score": 3, "relevance_rationale": "Routing among subagents learned from traces; not cost-focused.", "found_via": "arXiv query ti:routing AND ti:\"multi-agent\"", "cites_or_cited_by": [] }, { "id": "yao-2026-ace-router", "title": "ACE-Router: Generalizing History-Aware Routing from MCP Tools to the Agent Web", "authors": [ "Zhiyuan Yao", "Zishan Xu", "Yifu Guo", "Zhiguang Han", "Cheng Yang", "Shuo Zhang", "Weinan Zhang", "Xingshan Zeng", "Weiwen Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.08276", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "interaction history", "candidate graph" ], "cost_reduction_reported": null, "benchmarks": [ "MCP-Universe", "MCP-Mark" ], "models_studied": [], "production_evidence": false, "summary": "ACE-Router trains history-aware routers on synthesized multi-turn trajectories over a tool/agent dependency graph to select among very large MCP tool and agent pools. It generalizes to agent selection in multi-agent settings and scales to massive candidate spaces.", "key_idea": "History-aware router for tools and agents in large ecosystems.", "relevance_score": 3, "relevance_rationale": "Tool/agent routing inside trajectories.", "found_via": "web search: routing among agents", "cites_or_cited_by": [] }, { "id": "ye-2026-conditional-progressive-pruning-debate", "title": "Beyond Solo and Consistency: Vindicating Multi-Agent Debate via Conditional Progressive Pruning", "authors": [ "Ruosong Ye", "Caiqi Zhang", "Jiahao Li", "Haijun Wu", "Xiaolong Luo", "Huiyuan Chen", "Yu Wang", "Ying Chen", "Zhenting Wang", "Kai Mei", "Yang Zhou", "Dimitris N. Metaxas" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.33974", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Notes that existing debate frameworks fail to beat strong single-agent and self-consistency baselines under the same cost limit, and proposes Conditional Progressive Pruning, which it reports as the first debate method to beat consistency methods at equal cost.", "key_idea": "Prune debate progressively to beat equal-cost baselines.", "relevance_score": 3, "relevance_rationale": "Cost-matched MAS evaluation.", "found_via": "snowball: cited-by zhang-2024-agentprune", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer" ] }, { "id": "ye-2026-ecotab-stepwise-table-routing", "title": "Rethinking Stepwise Model Routing: A Cost-Efficient Table Reasoning Perspective", "authors": [ "Shenghao Ye", "Yuxiang Wang", "Yu Guo", "Dong Jin", "Shuangwu Chen", "Jian Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.29319", "code_url": null, "topic_clusters": [ "C3", "C9" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "table-token uncertainty", "text-token uncertainty" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "EcoTab adapts stepwise model routing to table reasoning, modeling the uncertainty of table-grounded tokens and surrounding text tokens separately because both predict errors in the next step.", "key_idea": "Table-aware uncertainty signals for stepwise small/large routing.", "relevance_score": 3, "relevance_rationale": "Stepwise routing variant; not agentic.", "found_via": "snowball: cited-by lee-2025-steer", "cites_or_cited_by": [ "lee-2025-steer" ] }, { "id": "yu-2026-olivia", "title": "OLIVIA: Online Learning via Inference-time Action Adaptation for Decision Making in LLM ReAct Agents", "authors": [ "Sheldon Yu", "Junda Wu", "Xintong Li", "Nikki Lijing Kuang", "Sizhe Zhou", "Tong Yu", "Jiawei Han", "Jingbo Shang", "Julian McAuley" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.11169", "code_url": null, "topic_clusters": [ "C3", "C5" ], "routing_granularity": "step", "learning_regime": "bandit-online", "signals_used": [ "frozen hidden states", "action-level feedback", "UCB" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Adds an explicit decision layer to ReAct agents, modeling final action selection as a contextual linear bandit over candidate actions with frozen hidden states as context. It updates online from action-level feedback during deployment and outperforms static ReAct and prompt-based adaptation on four benchmarks.", "key_idea": "Online bandit layer over agent action selection learned during deployment.", "relevance_score": 3, "relevance_rationale": "Deployment-time online learning at the step level, not cost-targeted.", "found_via": "query: online model selection LLM agents bandit tool", "cites_or_cited_by": [] }, { "id": "yu-2026-pythia", "title": "Pythia: Exploiting Workflow Predictability for Efficient Agent-Native LLM Serving", "authors": [ "Shan Yu", "Junyi Shu", "Yuanjiang Ni", "Kun Qian", "Xue Li", "Yang Wang", "Jinyuan Zhang", "Ziyi Xu", "Shuo Yang", "Lingjun Zhu", "Ennan Zhai", "Qingda Lu", "Jiarong Xing", "Youyou Lu", "Xin Jin", "Xuanzhe Liu", "Harry Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.25899", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "workflow semantics interface" ], "cost_reduction_reported": null, "benchmarks": [ "production serving platform and internal coding assistant traces" ], "models_studied": [], "production_evidence": true, "summary": "Analysis of a production serving platform and an internal coding assistant shows low prefix hit rates, contention from long-context requests and queuing from poor scaling; Pythia lets multi-agent apps declare workflow semantics to the serving layer to improve throughput and job completion time.", "key_idea": "Exploit the predictability of multi-agent workflow topology in serving.", "relevance_score": 3, "relevance_rationale": "Serving design informed by production agent traces.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "pan-2025-kvflow" ] }, { "id": "yu-2026-when-mas-help-information-bottleneck", "title": "When Do Multi-Agent Systems Help? An Information Bottleneck Perspective", "authors": [ "Wendi Yu", "Lianhao Zhou", "Xiangjue Dong", "Sai Sudarshan Barath", "Declan Staunton", "Byung-Jun Yoon", "Xiaoning Qian", "James Caverlee", "Shuiwang Ji" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.16133", "code_url": null, "topic_clusters": [ "C12", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "relay bandwidth" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Frames the MAS vs single-agent difference as an information bottleneck: a single agent keeps its full trace in one context while a MAS relays bounded messages between isolated contexts. MAS advantages arise only under bounded relays, where compression trades efficiency against loss of task-relevant information.", "key_idea": "MAS helps only when bounded-message compression is beneficial.", "relevance_score": 3, "relevance_rationale": "Theory for when MAS is worth it.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [ "gao-2025-single-or-multi-agent-why-not-both", "kapoor-2024-ai-agents-that-matter", "kim-2025-science-of-scaling-agent-systems", "tran-2026-single-vs-multi-agent-equal-budget" ] }, { "id": "yuan-2026-mixed-vendor-clinical-mas", "title": "Do Mixed-Vendor Multi-Agent LLMs Improve Clinical Diagnosis?", "authors": [ "G. C. Yuan", "Xiaoman Zhang", "Sung Eun Kim", "P. Rajpurkar" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.04421", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "RareBench", "DiagnosisArena" ], "models_studied": [ "o4-mini", "Gemini-2.5-Pro", "Claude-4.5-Sonnet" ], "production_evidence": false, "summary": "Compares single-LLM, single-vendor and mixed-vendor multi-agent conversations for clinical diagnosis. Mixed-vendor teams consistently win because they pool complementary inductive biases.", "key_idea": "Vendor diversity beats homogeneous teams in diagnostic MAS.", "relevance_score": 3, "relevance_rationale": "Heterogeneous-model MAS evidence (quality only).", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [ "ye-2025-x-mas" ] }, { "id": "yue-2026-workflow-opt-survey", "title": "From Static Templates to Dynamic Runtime Graphs: A Survey of Workflow Optimization for LLM Agents", "authors": [ "Ling Yue", "Kushal Raj Bhandari", "Ching-Yun Ko", "Dhaval Patel", "Shuxin Lin", "Nianjun Zhou", "Jianxi Gao", "Pin-Yu Chen", "Shaowu Pan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.22386", "code_url": null, "topic_clusters": [ "C4", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of workflow optimization for LLM agents distinguishing static templates, run-specific graphs and execution traces, classifying methods by when structure is fixed, what is optimized and which signals guide it, and proposing structure- and cost-aware evaluation.", "key_idea": "Unified vocabulary and taxonomy for agent workflow optimization.", "relevance_score": 3, "relevance_rationale": "Recent survey of C4 methods including cost-aware evaluation.", "found_via": "search: agentic workflow optimization survey automated generation 2025 arXiv", "cites_or_cited_by": [] }, { "id": "zehle-2026-cantante", "title": "CANTANTE: Optimizing Agentic Systems via Contrastive Credit Attribution", "authors": [ "Tom Zehle" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.13295", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "contrastive rollouts of joint configurations" ], "cost_reduction_reported": null, "benchmarks": [ "MBPP", "GSM8K", "HotpotQA" ], "models_studied": [], "production_evidence": false, "summary": "CANTANTE treats configuring multi-agent systems as credit assignment, decomposing system-level rewards into per-agent signals by contrasting rollouts of multiple joint configurations on the same query. Instantiated for prompt optimization, it gets the best average rank vs GEPA and MIPROv2.", "key_idea": "Contrastive per-agent credit attribution for optimizing agentic configs.", "relevance_score": 3, "relevance_rationale": "Credit attribution that could extend to per-agent model choice.", "found_via": "arXiv query abs:\"credit assignment\" AND abs:\"multi-agent\" AND abs:\"LLM\" AND abs:cost", "cites_or_cited_by": [] }, { "id": "zeng-2026-relope", "title": "ReLope: KL-Regularized LoRA Probes for Multimodal LLM Routing", "authors": [ "Yaopei Zeng", "Congchao Wang", "Blake JianHang Chen", "Lu Lin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.24787", "code_url": null, "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state probe", "LoRA" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows probe routing that predicts small-model correctness degrades for multimodal LLMs and proposes KL-regularized LoRA probes to improve hidden-state quality for routing.", "key_idea": "Improve hidden states to make probe routing work for MLLMs.", "relevance_score": 3, "relevance_rationale": "Probe routing extension.", "found_via": "WebSearch: probes hidden states LLM correctness prediction routing escalation", "cites_or_cited_by": [] }, { "id": "zhang-2026-calibration-not-control", "title": "Calibration Is Not Control: Why LLM-Agent Oversight Needs Intervention", "authors": [ "Chubin Zhang", "Zhenglin Wan", "Xingrui Yu", "Jingxuan Wu", "Qi Wen", "Pengfei Zhou", "Wangbo Zhao", "Ivor Tsang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.21399", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "risk estimates", "prefix branching" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Argues that agent oversight should target the expected gain from intervening rather than failure probability, since prefixes with equal risk may differ in recoverability. Introduces prefix branching to measure intervention advantage and shows the mismatch with risk prediction.", "key_idea": "Estimate intervention advantage, not failure risk, to decide when to intervene in agents.", "relevance_score": 3, "relevance_rationale": "Relevant to deciding when to escalate an agent step to a stronger model.", "found_via": "query: abs:\"predict\" AND abs:\"trajectory\" AND abs:\"failure\" AND abs:\"LLM agent\"", "cites_or_cited_by": [] }, { "id": "zhang-2026-credit-without-ground-truth", "title": "Credit Without Ground Truth: Auditing Step-Level Credit Assignment in LLM Agents Against Executed Replay", "authors": [ "Haiyue Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.19760", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "LLM-judge step scores", "logprob ratios", "policy confidence", "executed replay" ], "cost_reduction_reported": null, "benchmarks": [ "ALFWorld" ], "models_studied": [], "production_evidence": false, "summary": "Audits step-level credit signals for LLM agents against ground truth obtained by executed replay (resampling alternatives at each step and rolling forward) in ALFWorld. Finds none of the judge, logprob or confidence signals reliably beats shuffled controls.", "key_idea": "Executed replay as ground truth reveals step-level credit signals are unreliable.", "relevance_score": 3, "relevance_rationale": "Cautions against using judge scores to credit steps when learning step-level routing from traces.", "found_via": "query: replay evaluation agent model change", "cites_or_cited_by": [] }, { "id": "zhang-2026-duotrace-cascade-attribution", "title": "Detect Before You Attribute: Cascade Failure Attribution for Multi-Agent Systems", "authors": [ "Jiayi Zhang", "Zexin Wang", "Degang Sun", "Changhua Pei", "Fei Sun", "Gaogang Xie", "Jingjing Li (7654)" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.29646", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "failure detection", "attribution" ], "cost_reduction_reported": null, "benchmarks": [ "Who&When" ], "models_studied": [], "production_evidence": false, "summary": "Proposes detect-before-attribute cascade failure attribution (DuoTrace) for agent systems, improving agent- and step-level attribution accuracy by 8.7% and 7.0% over six baselines.", "key_idea": "Detect failures first, then attribute.", "relevance_score": 3, "relevance_rationale": "Failure attribution advance.", "found_via": "WebSearch: failure attribution multi-agent", "cites_or_cited_by": [ "zhang-2025-which-agent-when" ] }, { "id": "zhang-2026-metaagentx", "title": "MetaAgent-X : Breaking the Ceiling of Automatic Multi-Agent Systems via End-to-End Reinforcement Learning", "authors": [ "Yaolun Zhang", "Yujie Zhao", "Nan Wang", "Yiran Wu", "Jiayu Chang", "Yizhao Chen", "Qingyun Wu", "Jishen Zhao", "Huazheng Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.14212", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "task reward" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MetaAgent-X trains both a MAS designer and executor end to end with RL, using hierarchical rollouts and stagewise co-evolution, outperforming automatic MAS baselines by up to 21.7%.", "key_idea": "End-to-end RL co-training of designer and executor agents.", "relevance_score": 3, "relevance_rationale": "Automated MAS design; quality-focused.", "found_via": "search: ScoreFlow MaAS agentic supernet arXiv", "cites_or_cited_by": [] }, { "id": "zhang-2026-orchslm", "title": "OrchSLM: Probing the Dynamics of Small Language Model Orchestration", "authors": [ "Chengxi Zhang", "Yu Yao" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.13470", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "cached candidate samples", "consensus" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "OrchSLM studies non-interactive orchestration of heterogeneous small models, where each SLM generates candidates independently and a router selects among cached samples, exposing design choices like pool composition and consensus as controllable knobs.", "key_idea": "Probe of how routing over cached SLM outputs behaves.", "relevance_score": 3, "relevance_rationale": "SLM orchestration design space.", "found_via": "arXiv search: heterogeneous LLM multi-agent cost", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "zhang-2026-rl-orchestration-traces", "title": "Reinforcement Learning for LLM-based Multi-Agent Systems through Orchestration Traces", "authors": [ "Chenchen Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.02801", "code_url": null, "topic_clusters": [ "C3", "C6", "C15" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "orchestration traces" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Surveys RL for LLM multi-agent systems through orchestration traces (spawning, delegation, communication, aggregation and stopping), organizing work along reward design, credit-assignment units and orchestration learning. Notes that counterfactual message-level credit remains rare.", "key_idea": "Orchestration traces as the unit for learning multi-agent policies offline or online.", "relevance_score": 3, "relevance_rationale": "Maps how logged multi-agent traces can train orchestration and configuration policies.", "found_via": "query: abs:\"logged\" AND abs:\"counterfactual\" AND abs:\"LLM\"", "cites_or_cited_by": [] }, { "id": "zhao-2026-tcandon-router", "title": "TCAndon-Router: Adaptive Reasoning Router for Multi-Agent Collaboration", "authors": [ "Jiuzhou Zhao", "Chunrong Chen", "Chenqi Qiao", "Lebin Zheng", "Minqi Han", "Yanchi Liu Yongzhou Xu Xiaochuan Xu Min Zhang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.04544", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "natural-language reasoning chain", "candidate agent set" ], "cost_reduction_reported": null, "benchmarks": [ "public datasets", "enterprise data" ], "models_studied": [], "production_evidence": false, "summary": "A reasoning router for multi-agent systems that writes a reasoning chain, selects a set of expert agents, and has a refining agent merge their answers; supports adding new agents. Evaluated on public and enterprise data.", "key_idea": "Reasoning-based multi-label routing to expert agents.", "relevance_score": 3, "relevance_rationale": "Task routing in enterprise MAS; cost not the main target.", "found_via": "arXiv title search: reasoning router", "cites_or_cited_by": [] }, { "id": "zheng-2026-pbkv", "title": "Efficient Serving for Dynamic Agent Workflows with Prediction-based KV-Cache Management", "authors": [ "Haoyu Zheng", "Fangcheng Fu", "Jia Wu", "Binhang Yuan", "Yongqiang Zhang", "Hao Wang", "Yuanyuan Zhu", "Xiao Yan", "Jiawei Jiang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.06472", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "predicted next agent invocations from historical workflows" ], "cost_reduction_reported": "up to 1.85x speedup over LRU on dynamic workflows and up to 1.26x over KVFlow on static workflows", "benchmarks": [ "three workflow benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Prediction-based KV management for dynamic agent workflows: predicts future reuse from historical workflows and current context, keeping high-potential entries on GPU while using predictions conservatively.", "key_idea": "Predict dynamic workflow reuse to guide KV eviction and prefetch.", "relevance_score": 3, "relevance_rationale": "Extends workflow-aware KV caching to dynamic agent graphs.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "pan-2025-kvflow", "zhang-2026-cachescout" ] }, { "id": "zhong-2026-sod-onpolicy-distillation", "title": "SOD: Step-wise On-policy Distillation for Small Language Model Agents", "authors": [ "Qiyong Zhong", "Mao Zheng", "Mingyang Song", "Xin Lin", "Jie Sun", "Houcheng Jiang", "Xiang Wang", "Junfeng Fang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07725", "code_url": "https://github.com/YoungZ365/SOD", "topic_clusters": [ "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "step-wise on-policy teacher guidance" ], "cost_reduction_reported": "Up to 20.86% over second-best baseline; 0.6B student reaches 26.13% on AIME 2025", "benchmarks": [ "AIME 2025", "science", "code benchmarks" ], "models_studied": [ "0.6B student" ], "production_evidence": false, "summary": "Step-wise on-policy distillation for tool-integrated reasoning in small models, providing dense teacher signals beyond sparse RL rewards.", "key_idea": "Dense step-wise on-policy distillation for SLM agents.", "relevance_score": 3, "relevance_rationale": "Recent SLM agent distillation technique.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "zhong-2026-zclawbench-trajectory-reliability", "title": "Trajectory Reliability Prediction for Generalist AI Agents: Tool-Use Failure Analysis and Success Forecasting on ZClawBench", "authors": [ "Z. Zhong", "Chen-Yu Li", "Hengning Rao" ], "year": 2026, "venue_or_source": "Journal of Technology Informatics and Engineering", "source_type": "paper", "url": "https://doi.org/10.51903/jtie.v5i1.539", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "tool-call volume", "replanning", "tool errors", "invalid-action ratio", "looping indicators", "trajectory length" ], "cost_reduction_reported": null, "benchmarks": [ "ZClawBench" ], "models_studied": [ "six model families" ], "production_evidence": false, "summary": "Tests whether operational trajectory signals such as tool-call counts, replanning, tool errors and looping predict task success on ZClawBench (696 trajectories across 116 tasks and six model families). Success labels are reconstructed from official per-category score distributions, which the authors flag as a modeling assumption.", "key_idea": "Predict agent task success from operational trace features before manual evaluation.", "relevance_score": 3, "relevance_rationale": "Agent success prediction from traces; label construction is weak, so evidence is limited.", "found_via": "query: predicting agent success trajectory", "cites_or_cited_by": [] }, { "id": "aggarwal-2025-l1-lcpo", "title": "L1: Controlling How Long A Reasoning Model Thinks With Reinforcement Learning", "authors": [ "Pranjal Aggarwal", "Sean Welleck" ], "year": 2025, "venue_or_source": "COLM 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2503.04697", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "RL-offline", "signals_used": [ "user-specified length constraint" ], "cost_reduction_reported": "1.5B L1 model surpasses GPT-4o at equal reasoning lengths; outperforms S1 for length control", "benchmarks": [], "models_studied": [ "GPT-4o" ], "production_evidence": false, "summary": "Length Controlled Policy Optimization trains a reasoning model to hit a length target given in the prompt, enabling smooth cost-accuracy trade-offs. Also yields short reasoning models with long-CoT-like patterns.", "key_idea": "RL to obey a prompt-specified thinking length.", "relevance_score": 3, "relevance_rationale": "Controllable reasoning budget per call.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "alexandru-2025-atla-selene-mini", "title": "Atla Selene Mini: A General Purpose Evaluation Model", "authors": [ "Andrei Alexandru", "Antonia Calvi", "Henry Broomfield", "Jackson Golden", "Kyle Dai", "Mathias Leys", "Maurice Burger", "Max Bartolo", "Roman Engeler", "Sashank Pisupati", "Toby Drane", "Young Sun Park" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.17195", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "small judge" ], "cost_reduction_reported": null, "benchmarks": [ "RewardBench", "11 OOD benchmarks" ], "models_studied": [ "Llama-3.1-8B" ], "production_evidence": false, "summary": "8B small-language-model judge that outperforms GPT-4o-mini and other small judges across 11 OOD benchmarks and tops 8B generative models on RewardBench.", "key_idea": "Strong 8B general judge.", "relevance_score": 3, "relevance_rationale": "Cheap judge.", "found_via": "hf-papers search: small judge model evaluation", "cites_or_cited_by": [] }, { "id": "alomrani-2025-reasoning-on-a-budget", "title": "Reasoning on a Budget: A Survey of Adaptive and Controllable Test-Time Compute in LLMs", "authors": [ "Mohammad Ali Alomrani", "Yingxue Zhang", "Derek Li", "Qianyi Sun", "Soumyasundar Pal", "Zhanguang Zhang", "Yaochen Hu", "Rohan Deepak Ajwani", "Antonios Valkanas", "Raika Karimi", "Peng Cheng", "Yunzhou Wang", "Pengyi Liao", "Hanrui Huang", "Bin Wang", "Jianye Hao", "Mark Coates", "et al." ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.02076", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of adaptive and controllable test-time compute, distinguishing fixed-budget control from difficulty- or confidence-adaptive scaling, and benchmarking proprietary LLMs on performance vs token use.", "key_idea": "L1-controllability vs L2-adaptiveness taxonomy.", "relevance_score": 3, "relevance_rationale": "Budget control survey.", "found_via": "search: survey efficiency LLM agents | web search: Token-Budget-Aware LLM Reasoning (related results)", "cites_or_cited_by": [] }, { "id": "anthropic-2025-advanced-tool-use", "title": "Introducing advanced tool use on the Claude Developer Platform", "authors": [ "Anthropic" ], "year": 2025, "venue_or_source": "Anthropic engineering blog", "source_type": "blog", "url": "https://www.anthropic.com/engineering/advanced-tool-use", "code_url": null, "topic_clusters": [ "C10", "C12", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "Tool Search Tool: ~85% reduction in upfront tool-definition tokens (~77K to ~8.7K for 50+ MCP tools); MCP-eval accuracy Opus 4 49%->74%, Opus 4.5 79.5%->88.1%. Programmatic tool calling: average tokens 43,588->27,297 (37% reduction) on complex research tasks. Tool use examples: 72%->90% on complex parameter handling (all vendor-reported internal testing)", "benchmarks": [ "internal MCP evaluations", "GIA", "internal knowledge retrieval" ], "models_studied": [ "Claude Opus 4", "Claude Opus 4.5", "Claude Sonnet 4.5" ], "production_evidence": false, "summary": "Anthropic introduces three beta features: a Tool Search Tool that discovers tools on demand instead of loading all definitions, Programmatic Tool Calling in which Claude orchestrates tools via code so intermediate results skip the context, and Tool Use Examples. It reports token and accuracy numbers from internal evaluations and gives guidance on when each feature pays off (e.g., tool definitions over 10K tokens).", "key_idea": "Deferred tool loading and code-orchestrated tool calls cut agent context tokens while improving tool-selection accuracy.", "relevance_score": 3, "relevance_rationale": "Quantified context-efficiency levers for large-toolset agents; vendor internal numbers.", "found_via": "direct: anthropic.com/engineering", "cites_or_cited_by": [ "anthropic-2025-code-execution-with-mcp" ] }, { "id": "anthropic-2025-code-execution-with-mcp", "title": "Code execution with MCP: building more efficient AI agents", "authors": [ "Anthropic" ], "year": 2025, "venue_or_source": "Anthropic engineering blog", "source_type": "blog", "url": "https://www.anthropic.com/engineering/code-execution-with-mcp", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "Example workflow: tool-related context reduced from 150,000 tokens to 2,000 tokens (98.7% saving) by loading tool definitions on demand as code files (vendor-reported, illustrative example)", "benchmarks": [], "models_studied": [ "Claude" ], "production_evidence": false, "summary": "Anthropic describes how directly exposing all MCP tool definitions and routing every intermediate tool result through the model inflates context and cost. Instead, agents write code that calls MCP servers as a code API in a sandbox, loading only needed tool definitions and filtering data in the execution environment before it reaches the model. Cloudflare's similar 'Code Mode' is cited.", "key_idea": "Present tools as code APIs and let the agent execute code so tool definitions and intermediate results stay out of the context window.", "relevance_score": 3, "relevance_rationale": "Concrete context/token-cost lever for tool-heavy agents; single illustrative number.", "found_via": "direct: anthropic.com/engineering", "cites_or_cited_by": [ "anthropic-2025-advanced-tool-use", "varda-2025-cloudflare-code-mode" ] }, { "id": "anthropic-2025-context-management", "title": "Managing context on the Claude Developer Platform (context editing and memory tool)", "authors": [ "Anthropic" ], "year": 2025, "venue_or_source": "Claude blog; Claude Platform docs (Context editing)", "source_type": "product", "url": "https://claude.com/blog/context-management", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "token-count threshold" ], "cost_reduction_reported": "Memory tool + context editing improved performance 39% over baseline; context editing alone 29%; in a 100-turn web-search evaluation, context editing cut token consumption by 84% (vendor-reported internal evaluation)", "benchmarks": [ "internal agentic search evaluation (100-turn web search)" ], "models_studied": [ "Claude Sonnet 4.5" ], "production_evidence": false, "summary": "Announces server-side context editing, which automatically clears stale tool results (and later thinking blocks) as a conversation approaches limits, and a file-based memory tool that persists information outside the context window. The docs (platform.claude.com/docs/en/build-with-claude/context-editing) describe configurable clearing strategies and position server-side compaction as the primary strategy for long conversations.", "key_idea": "Rule-based clearing of old tool results plus external memory keeps long agent loops within context at far lower token cost.", "relevance_score": 3, "relevance_rationale": "Production API lever for long-horizon agent token cost with quantified internal results.", "found_via": "direct: claude.com/blog", "cites_or_cited_by": [ "anthropic-2026-compaction-docs" ] }, { "id": "arora-2025-reason-efficiently", "title": "Training Language Models to Reason Efficiently", "authors": [ "Daman Arora", "Andrea Zanette" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.04463", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "task complexity (implicit)", "length-penalised reward" ], "cost_reduction_reported": "Significant reductions in inference cost while preserving most of the accuracy on two open-weight reasoning models (no single number in abstract)", "benchmarks": [], "models_studied": [ "two open-weight large reasoning models" ], "production_evidence": false, "summary": "Uses RL to train reasoning models to allocate inference compute according to task complexity, yielding a family of models at different efficiency levels controlled by one hyperparameter.", "key_idea": "One-knob RL family of efficiency levels for reasoning models.", "relevance_score": 3, "relevance_rationale": "Gives a menu of cost/accuracy operating points to route among.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "balachandran-2025-inference-time-scaling-complex-tasks", "title": "Inference-Time Scaling for Complex Tasks: Where We Stand and What Lies Ahead", "authors": [ "Vidhisha Balachandran", "Jingya Chen", "Lingjiao Chen", "Shivam Garg", "Neel Joshi", "Yash Lara", "John Langford", "Besmira Nushi", "Vibhav Vineet", "Yue Wu", "Safoora Yousefi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.00294", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "repeated calls", "verifier feedback" ], "cost_reduction_reported": null, "benchmarks": [ "math/STEM", "calendar planning", "NP-hard problems", "navigation", "spatial reasoning" ], "models_studied": [ "GPT-4o", "o1", "nine models total" ], "production_evidence": false, "summary": "Study comparing conventional and reasoning models under repeated independent and sequential calls on eight tasks. Gains from inference-time scaling vary by task and shrink with complexity; more tokens do not reliably mean higher accuracy.", "key_idea": "Inference-time scaling gains are task-dependent and diminish on hard tasks.", "relevance_score": 3, "relevance_rationale": "Evidence on when reasoning-model spend pays off vs conventional models plus verifiers.", "found_via": "snowball: related work of snell-2024-test-time-compute-optimal", "cites_or_cited_by": [ "snell-2024-test-time-compute-optimal" ] }, { "id": "baranowski-2025-instacart-maple", "title": "Simplifying Large-Scale LLM Processing across Instacart with Maple", "authors": [ "Paul Baranowski" ], "year": 2025, "venue_or_source": "Instacart tech blog", "source_type": "blog", "url": "https://tech.instacart.com/simplifying-large-scale-llm-processing-across-instacart-with-maple-63df4508d5be", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Company-reported: up to 50% LLM cost savings vs real-time calls; some processes cut from hundreds of thousands to thousands of dollars per year", "benchmarks": [], "models_studied": [ "provider batch APIs" ], "production_evidence": true, "summary": "Instacart's internal service that wraps provider batch APIs for jobs of 10M+ prompts, handling batching, retries, file management and cost tracking on Temporal and S3.", "key_idea": "Internal platform to route offline workloads to batch APIs.", "relevance_score": 3, "relevance_rationale": "Production cost reduction via batch lever.", "found_via": "seed list: Instacart case study", "cites_or_cited_by": [] }, { "id": "barrak-2025-cargo", "title": "CARGO: A Framework for Confidence-Aware Routing of Large Language Models", "authors": [ "Amine Barrak", "Yosr Fourati", "Michael Olchawa", "Emna Ksontini", "Khalil Zoghlami" ], "year": 2025, "venue_or_source": "CASCON 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2509.14899", "code_url": null, "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "embedding regressor on LLM-judged pairwise comparisons", "uncertainty-triggered classifier" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-4o", "Claude 3.5 Sonnet", "DeepSeek V3", "Perplexity Sonar" ], "production_evidence": false, "summary": "CARGO trains an embedding regressor on LLM-judged pairwise comparisons, calling a binary classifier only when uncertain, with optional category-specific regressors. It routes among four frontier APIs with 76.4% top-1 accuracy.", "key_idea": "Judge-supervised two-stage confidence-aware router.", "relevance_score": 3, "relevance_rationale": "Uses judges instead of human labels.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "behera-2025-multi-llm-inference-survey", "title": "Towards Efficient Multi-LLM Inference: Characterization and Analysis of LLM Routing and Hierarchical Techniques", "authors": [ "Adarsh Prasad Behera", "Jaya Prakash Champati", "Roberto Morabito", "Sasu Tarkoma", "James Gross" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.06579", "code_url": null, "topic_clusters": [ "C1", "C2", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "query complexity", "confidence" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey characterizing routing and cascading (hierarchical inference) for efficient multi-LLM inference, especially for edge and cost-sensitive settings. Compares approaches that use light models for easy queries and escalate only when needed.", "key_idea": "Survey of routing vs hierarchical inference.", "relevance_score": 3, "relevance_rationale": "Survey for C15.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "bouchard-2025-uqlm-suite", "title": "Uncertainty Quantification for Language Models: A Suite of Black-Box, White-Box, LLM Judge, and Ensemble Scorers", "authors": [ "Dylan Bouchard", "Mohit Singh Chauhan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.19254", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "black-box UQ", "white-box UQ", "LLM judge", "ensemble" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Standardizes black-box, white-box and LLM-judge UQ into 0-1 confidence scores and a tunable ensemble that outperforms individual components for hallucination detection.", "key_idea": "Ensemble of UQ scorers.", "relevance_score": 3, "relevance_rationale": "Practical toolkit for confidence scoring.", "found_via": "hf-papers search: hallucination detection single pass", "cites_or_cited_by": [] }, { "id": "chase-2025-how-when-multi-agent", "title": "How and when to build multi-agent systems", "authors": [ "Harrison Chase" ], "year": 2025, "venue_or_source": "LangChain blog", "source_type": "blog", "url": "https://www.langchain.com/blog/how-and-when-to-build-multi-agent-systems", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Harrison Chase reconciles Anthropic's multi-agent research post with Cognition's 'don't build multi-agents' argument: multi-agent systems suit read-heavy, breadth-first, parallelizable tasks whose value justifies high token spend, while write-heavy tasks with shared context and many interdependencies are poor fits. Context engineering is framed as the main job.", "key_idea": "Multi-agent fits parallel read-heavy tasks worth the token cost; avoid it for tightly coupled writes.", "relevance_score": 3, "relevance_rationale": "Practitioner synthesis on when MAS is worth it.", "found_via": "web search: Anthropic multi-agent research system", "cites_or_cited_by": [] }, { "id": "chen-2025-llm-ensemble-survey", "title": "Harnessing Multiple Large Language Models: A Survey on LLM Ensemble", "authors": [ "Zhijun Chen", "Xiaodong Lu", "Jingzheng Li", "Pengpeng Chen", "Zhuoran Li", "Kai Sun", "Yuankai Luo", "Qianren Mao", "Ming Li", "Likang Xiao", "Dingqi Yang", "Xiao Huang", "Yikun Ban", "Hailong Sun", "Philip S. Yu" ], "year": 2025, "venue_or_source": "IJCAI 2026 (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2502.18036", "code_url": "https://github.com/junchenzhi/Awesome-LLM-Ensemble", "topic_clusters": [ "C1", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Systematic review of LLM ensemble methods grouped as ensemble-before-inference (routing), during-inference and after-inference, with benchmarks and applications.", "key_idea": "Taxonomy placing routing within LLM ensembling.", "relevance_score": 3, "relevance_rationale": "Seed survey on routing and ensembling.", "found_via": "seed", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "chen-2025-llm-slm-collab-survey", "title": "A Survey on Collaborative Mechanisms Between Large and Small Language Models", "authors": [ "Yi Chen", "JiaHao Zhao", "HaoHao Han" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.07460", "code_url": null, "topic_clusters": [ "C1", "C2", "C3", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of large-small LM collaboration mechanisms (pipeline, routing, auxiliary, distillation, fusion), mostly motivated by on-device constraints.", "key_idea": "LLM-SLM collaboration taxonomy.", "relevance_score": 3, "relevance_rationale": "Covers routing/cascades.", "found_via": "snowball: cited-by ding-2024-hybrid-llm | web search: big-little collaboration", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "narayan-2025-minions" ] }, { "id": "chhikara-2025-mem0", "title": "Mem0: Building Production-Ready AI Agents with Scalable Long-Term Memory", "authors": [ "Prateek Chhikara", "Dev Khant", "Saket Aryan", "Taranjeet Singh", "Deshraj Yadav" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.19413", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "extracted memory facts", "graph memory" ], "cost_reduction_reported": "91% lower p95 latency and more than 90% token-cost savings vs the full-context method, with gains in LLM-as-a-Judge score over OpenAI memory on LOCOMO", "benchmarks": [ "LOCOMO" ], "models_studied": [], "production_evidence": false, "summary": "Mem0 extracts, consolidates and retrieves salient facts from conversations (optionally as a graph) so agents answer from compact memory instead of full history, reporting large latency and token savings.", "key_idea": "Extract-and-retrieve memory instead of full-context replay.", "relevance_score": 3, "relevance_rationale": "Quantified token-cost savings from memory for production agents.", "found_via": "Mem0 scalable long-term memory production AI agents token cost", "cites_or_cited_by": [ "packer-2023-memgpt", "xu-2025-a-mem" ] }, { "id": "dai-2025-s-grpo", "title": "S-GRPO: Early Exit via Reinforcement Learning in Reasoning Models", "authors": [ "Muzhi Dai", "Chenxu Yang", "Qingyi Si" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.07686", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "RL-offline", "signals_used": [ "decaying reward by exit position" ], "cost_reduction_reported": "35.4%-61.1% shorter sequences with 0.72%-6.08% absolute accuracy gains on GSM8K, AIME 2024, AMC 2023, MATH-500, GPQA Diamond", "benchmarks": [ "GSM8K", "AIME 2024", "AMC 2023", "MATH-500", "GPQA Diamond" ], "models_studied": [ "Qwen3", "DeepSeek-distill" ], "production_evidence": false, "summary": "Serial-group RL samples one reasoning path and exits at several positions, rewarding earlier correct exits more, so the model learns when its reasoning is sufficient.", "key_idea": "RL rewards for earlier correct exits.", "relevance_score": 3, "relevance_rationale": "Learned early exit.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "deshpande-2025-trail", "title": "TRAIL: Trace Reasoning and Agentic Issue Localization", "authors": [ "Darshan Deshpande", "Varun Gangal", "Hersh Mehta", "Jitin Krishnan", "Anand Kannappan", "Rebecca Qian" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.08638", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judge", "error taxonomy" ], "cost_reduction_reported": null, "benchmarks": [ "TRAIL" ], "models_studied": [ "Gemini-2.5-pro" ], "production_evidence": false, "summary": "Introduces a taxonomy of agentic errors and 148 human-annotated traces from single- and multi-agent systems; long-context LLMs perform poorly at trace debugging (best model around 11% joint accuracy).", "key_idea": "Benchmark for locating errors in agent traces.", "relevance_score": 3, "relevance_rationale": "Measures readiness of LLM judges for trace-level monitoring.", "found_via": "keyword: failure detection in agents", "cites_or_cited_by": [] }, { "id": "elumar-2025-camvo", "title": "Cost-aware LLM-based Online Dataset Annotation", "authors": [ "Eray Can Elumar", "Cem Tekin", "Osman Yagan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.15101", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "contextual embeddings", "confidence scores" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU", "IMDB" ], "models_studied": [ "multiple LLMs" ], "production_evidence": false, "summary": "CaMVo selects, per item, a subset of LLMs for majority-vote labeling with LinUCB and a Bayesian confidence estimator, matching full majority voting at lower cost without ground-truth labels.", "key_idea": "LinUCB subset selection for cheaper LLM ensembles.", "relevance_score": 3, "relevance_rationale": "Label-free online model-subset selection.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "li-2010-linucb" ] }, { "id": "erdogan-2025-plan-and-act", "title": "Plan-and-Act: Improving Planning of Agents for Long-Horizon Tasks", "authors": [ "Lutfi Eren Erdogan", "Nicholas Lee", "Sehoon Kim", "Suhong Moon", "Hiroki Furuta", "Gopala Anumanchipalli", "Kurt Keutzer", "Amir Gholami" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.09572", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "WebArena-Lite", "WebVoyager" ], "models_studied": [], "production_evidence": false, "summary": "Plan-and-Act separates a Planner model that writes structured high-level plans from an Executor model that turns them into environment actions, training the planner on synthetic plan annotations. It reaches 57.58% on WebArena-Lite and 81.36% on WebVoyager.", "key_idea": "Train a dedicated planner model separate from the executor.", "relevance_score": 3, "relevance_rationale": "Planner-executor split that enables separate model sizing per role.", "found_via": "arXiv API search", "cites_or_cited_by": [] }, { "id": "fan-2025-agent-gspo", "title": "Agent-GSPO: Communication-Efficient Multi-Agent Systems via Group Sequence Policy Optimization", "authors": [ "Yijia Fan", "Jusheng Zhang", "Jing Yang", "Keze Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.22477", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "verbosity penalty" ], "cost_reduction_reported": "State-of-the-art on seven reasoning benchmarks with a fraction of existing methods' token consumption", "benchmarks": [ "seven reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Agent-GSPO trains MAS agents with sequence-level RL on a communication-aware reward that penalizes verbosity, leading to emergent strategic silence and much lower token use.", "key_idea": "RL with verbosity penalties for token-economical MAS.", "relevance_score": 3, "relevance_rationale": "Training-based MAS cost reduction.", "found_via": "snowball: cited-by zhang-2024-agentprune", "cites_or_cited_by": [ "zhang-2024-agentprune" ] }, { "id": "fan-2025-core-cloud-local-mobile", "title": "CORE: Reducing UI Exposure in Mobile Agents via Collaboration Between Cloud and Local LLMs", "authors": [ "Gucongcong Fan", "Chaoyue Niu", "Chengfei Lyu", "Fan Wu", "Guihai Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.15455", "code_url": "https://github.com/Entropy-Fighter/CORE", "topic_clusters": [ "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "UI block ranking" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CORE splits mobile-agent decisions between a local LLM that ranks UI blocks and a cloud LLM that selects elements within the top block, with co-planning of sub-tasks. It reduces UI exposure by up to 55.6% while staying slightly below cloud-only success.", "key_idea": "Per-step local/cloud division of labour in mobile agents.", "relevance_score": 3, "relevance_rationale": "Step-level local/cloud split; motivated by privacy rather than cost.", "found_via": "arXiv query ti:\"cloud\" AND ti:\"local\" AND ti:\"agent\" AND abs:\"llm\"", "cites_or_cited_by": [] }, { "id": "fanconi-2025-cascaded-human-ai", "title": "Cascaded Language Models for Cost-effective Human-AI Decision-Making", "authors": [ "Claudio Fanconi", "Mihaela van der Schaar" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.11887", "code_url": null, "topic_clusters": [ "C2", "C5" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "confidence score", "human feedback (online)" ], "cost_reduction_reported": "Higher accuracy at reduced cost vs single-model baselines in most cases (no single number in abstract)", "benchmarks": [ "ARC-Easy", "ARC-Challenge", "MMLU", "MedQA", "MedMCQA" ], "models_studied": [], "production_evidence": false, "summary": "A three-tier cascade (small LLM, large LLM, human expert) with a confidence-based deferral policy and an abstention policy that hands off to humans. Thresholds adapt online from human feedback.", "key_idea": "Two-stage deferral plus abstention to humans with online threshold updates.", "relevance_score": 3, "relevance_rationale": "Human-in-the-loop escalation relevant to enterprise workflows.", "found_via": "arXiv search: model cascade language model", "cites_or_cited_by": [] }, { "id": "fang-2025-thinkless", "title": "Thinkless: LLM Learns When to Think", "authors": [ "Gongfan Fang", "Xinyin Ma", "Xinchao Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.13379", "code_url": "https://github.com/VainF/Thinkless", "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "learned control token (/)" ], "cost_reduction_reported": "Reduces long-chain thinking usage by 50%-90% on Minerva Algebra, MATH-500, GSM8K", "benchmarks": [ "Minerva Algebra", "MATH-500", "GSM8K" ], "models_studied": [], "production_evidence": false, "summary": "Trains a model with RL to emit a control token choosing short or long reasoning per query. Decoupled GRPO separates the mode-selection loss from the answer loss to avoid collapse.", "key_idea": "RL-learned mode token for when to think.", "relevance_score": 3, "relevance_rationale": "Model-internal think/no-think routing.", "found_via": "web search: routing between thinking and non-thinking modes", "cites_or_cited_by": [] }, { "id": "farinhas-2025-qe-deferral-translation", "title": "Translate Smart, not Hard: Cascaded Translation Systems with Quality-Aware Deferral", "authors": [ "António Farinhas", "Nuno M. Guerreiro", "Sweta Agrawal", "Ricardo Rei", "André F. T. Martins" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.12701", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "quality estimation metric" ], "cost_reduction_reported": "Matches larger-model performance while invoking it for 30%-50% of examples", "benchmarks": [ "machine translation" ], "models_studied": [], "production_evidence": false, "summary": "Uses reference-free quality-estimation metrics as the deferral rule in a translation cascade, validated with automatic and human evaluation.", "key_idea": "Off-the-shelf QE metrics as deferral rules.", "relevance_score": 3, "relevance_rationale": "Judge-style signals for escalation.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "fatemi-2025-concise-reasoning-rl", "title": "Concise Reasoning via Reinforcement Learning", "authors": [ "Mehdi Fatemi", "Banafsheh Rafiee", "Mingjie Tang", "Kartik Talamadupula" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.05185", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "solvable-problem RL phase" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Argues that long outputs from RL-trained reasoners are an optimisation artefact of PPO/GRPO on unsolvable problems, and shows a brief second RL phase on solvable problems shortens responses while keeping or improving accuracy.", "key_idea": "Verbosity is an RL artefact; fix it with a short solvable-set RL phase.", "relevance_score": 3, "relevance_rationale": "Explains and reduces reasoning length.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "feng-2025-efficient-reasoning-models-survey", "title": "Efficient Reasoning Models: A Survey", "authors": [ "Sicheng Feng", "Gongfan Fang", "Xinyin Ma", "Xinchao Wang" ], "year": 2025, "venue_or_source": "TMLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2504.10903", "code_url": "https://github.com/fscdc/Awesome-Efficient-Reasoning-Models", "topic_clusters": [ "C8", "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey organizing efficient reasoning into shorter (compressed CoT), smaller (distillation, compression, RL) and faster (efficient decoding) directions.", "key_idea": "Shorter/smaller/faster taxonomy.", "relevance_score": 3, "relevance_rationale": "Efficient reasoning survey.", "found_via": "snowball: cited-by ong-2024-routellm | GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "ong-2024-routellm", "sui-2025-stop-overthinking" ] }, { "id": "feng-2025-heterogeneous-swarms", "title": "Heterogeneous Swarms: Jointly Optimizing Model Roles and Weights for Multi-LLM Systems", "authors": [ "Shangbin Feng", "Zifeng Wang", "Palash Goyal", "Yike Wang", "Weijia Shi", "Huang Xia", "Hamid Palangi", "Luke Zettlemoyer", "Yulia Tsvetkov", "Chen-Yu Lee", "Tomas Pfister" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.04510", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "utility score", "JFK contribution score" ], "cost_reduction_reported": null, "benchmarks": [ "12 tasks" ], "models_studied": [], "production_evidence": false, "summary": "Heterogeneous Swarms jointly optimizes which LLMs play which roles in a DAG and their weights using particle swarm optimization, outperforming 15 role- or weight-based baselines by 18.5% on average.", "key_idea": "Swarm search over model roles (DAG) and weights.", "relevance_score": 3, "relevance_rationale": "Role assignment among heterogeneous models; quality-focused.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [ "zhang-2024-agentprune", "zhang-2024-g-designer" ] }, { "id": "fu-hinthorn-2025-benchmarking-multi-agent-architectures", "title": "Benchmarking Multi-Agent Architectures", "authors": [ "Will Fu-Hinthorn" ], "year": 2025, "venue_or_source": "LangChain blog", "source_type": "blog", "url": "https://www.langchain.com/blog/benchmarking-multi-agent-architectures", "code_url": "https://github.com/langchain-ai/langgraph-supervisor-py", "topic_clusters": [ "C10", "C12", "C13" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Supervisor architecture improved by nearly 50% on the tau-bench variant after removing handoff messages and letting sub-agents respond directly; single agent's token use grows with distractor domains while supervisor and swarm stay flat (qualitative token comparison, vendor-reported)", "benchmarks": [ "tau-bench retail (first 100 test examples) with added distractor domains" ], "models_studied": [], "production_evidence": false, "summary": "Compares a single tool-calling agent, a swarm (peer handoffs) and a supervisor architecture on tau-bench retail with up to several distractor domains of 19 tools each, measuring score and tokens. The single agent degrades sharply with two or more distractor domains and uses more tokens; the supervisor loses accuracy and tokens to 'telephone' translation, fixed by removing handoff messages from sub-agent context and forwarding sub-agent responses directly.", "key_idea": "Multi-agent decomposition pays off mainly when context grows; supervisor overhead comes from relaying messages.", "relevance_score": 3, "relevance_rationale": "Empirical architecture-level cost and quality comparison, small scale.", "found_via": "direct: langchain.com/blog", "cites_or_cited_by": [] }, { "id": "gao-2025-reasoning-efficiency-gap", "title": "How Far Are We from Optimal Reasoning Efficiency?", "authors": [ "Jiaxuan Gao", "Shu Yan", "Qixin Tan", "Lu Yang", "Shusheng Xu", "Wei Fu", "Zhiyu Mei", "Kaifeng Lyu", "Yi Wu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.07104", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "token budgets" ], "cost_reduction_reported": "REO-RL reduces REG by >=50 across evaluated LRMs and matches Qwen3-4B/8B efficiency frontiers under a 16K budget with minimal accuracy loss", "benchmarks": [ "mathematical benchmarks" ], "models_studied": [ "Qwen3-4B", "Qwen3-8B" ], "production_evidence": false, "summary": "Builds empirical reasoning-efficiency frontiers and a Reasoning Efficiency Gap metric, then proposes REO-RL that optimises over a sparse set of token budgets.", "key_idea": "Frontier-based metric and RL for reasoning efficiency.", "relevance_score": 3, "relevance_rationale": "Evaluation standard for length-reduction methods.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "gao-2025-turn-control-coding-agents", "title": "More with Less: An Empirical Study of Turn-Control Strategies for Efficient Coding Agents", "authors": [ "Pengfei Gao", "Chao Peng" ], "year": 2025, "venue_or_source": "ICSE 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2510.16786", "code_url": null, "topic_clusters": [ "C3", "C9", "C13" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "turn count" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-bench" ], "models_studied": [ "Claude 4 Sonnet", "Gemini 2.5 Pro", "GPT 4.1" ], "production_evidence": false, "summary": "An empirical study on SWE-bench of turn-control strategies for coding agents: unrestricted, fixed-turn limits with reminders, and dynamic limits. It frames total turn count as a primary lever on cost, since tokens grow with every turn.", "key_idea": "Turn budgets as a cost lever for coding agents.", "relevance_score": 3, "relevance_rationale": "Budget-constrained agent execution in coding agents; ICSE 2026.", "found_via": "Semantic Scholar query \"per-turn model routing coding agent\"", "cites_or_cited_by": [] }, { "id": "ge-2025-famas-spectrum-attribution", "title": "Who is Introducing the Failure? Automatically Attributing Failures of Multi-Agent Systems via Spectrum Analysis", "authors": [ "Yu Ge", "Linna Xie", "Zhong Li", "Yu Pei", "Tian Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.13782", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "spectrum analysis", "repeated executions" ], "cost_reduction_reported": null, "benchmarks": [ "Who&When" ], "models_studied": [], "production_evidence": false, "summary": "Proposes FAMAS, a spectrum-based failure attribution method that replays MAS executions repeatedly and estimates each agent action's suspiciousness from activation patterns across successful and failed runs.", "key_idea": "Software-engineering spectrum fault localization applied to multi-agent traces.", "relevance_score": 3, "relevance_rationale": "Alternative failure attribution signal not requiring a strong LLM judge.", "found_via": "snowball: cited-by which-agent-when", "cites_or_cited_by": [ "zhang-2025-which-agent-when" ] }, { "id": "gema-2025-inverse-scaling-test-time", "title": "Inverse Scaling in Test-Time Compute", "authors": [ "Aryo Pradipta Gema", "Alexander Hägele", "Runjin Chen", "Andy Arditi", "Jacob Goldman-Wetzler", "Kit Fraser-Taliente", "Henry Sleight", "Linda Petrini", "Julian Michael", "Beatrice Alex", "Pasquale Minervini", "Yanda Chen", "Joe Benton", "Ethan Perez" ], "year": 2025, "venue_or_source": "TMLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2507.14417", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "counting with distractors", "regression with spurious features", "deduction with constraint tracking", "AI-risk evaluations" ], "models_studied": [ "Claude models", "OpenAI o-series", "Claude Sonnet 4" ], "production_evidence": false, "summary": "Constructs tasks where longer reasoning lowers accuracy and identifies five failure modes, such as distraction and overfitting to framing, plus amplified concerning behaviours in some models.", "key_idea": "More test-time compute can reduce accuracy.", "relevance_score": 3, "relevance_rationale": "Argues for evaluating across reasoning lengths before raising budgets.", "found_via": "web search: Do NOT Think That Much (related results)", "cites_or_cited_by": [] }, { "id": "geuter-2025-robon", "title": "RoBoN: Routed Online Best-of-n for Test-Time Scaling with Multiple LLMs", "authors": [ "Jonathan Geuter", "Gregor Kornhardt" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.05542", "code_url": null, "topic_clusters": [ "C5", "C7", "C9" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "reward model score", "agreement signal" ], "cost_reduction_reported": "up to 3.4% absolute accuracy gain over single-model best-of-n at compute parity", "benchmarks": [ "MATH500", "OlympiadBench", "MinervaMath", "GSM8K", "MMLU" ], "models_studied": [ "multiple open LLMs" ], "production_evidence": false, "summary": "RoBoN routes each best-of-n generation across a suite of models sequentially, choosing the next model from reward-model scores and agreement among responses; it is training-free and beats single-model best-of-n.", "key_idea": "Online routing within best-of-n across multiple LLMs.", "relevance_score": 3, "relevance_rationale": "Online, in-query model routing for test-time scaling.", "found_via": "OpenAlex title: online routing LLM", "cites_or_cited_by": [] }, { "id": "ghosal-2025-mirage-test-time-scaling", "title": "Does Thinking More always Help? Mirage of Test-Time Scaling in Reasoning Models", "authors": [ "Soumya Suvra Ghosal", "Souradip Chakraborty", "Avinash Reddy", "Yifu Lu", "Mengdi Wang", "Dinesh Manocha", "Furong Huang", "Mohammad Ghavamzadeh", "Amrit Singh Bedi" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.04210", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "majority vote over parallel paths" ], "cost_reduction_reported": "Parallel thinking gives up to 20% higher accuracy than extended thinking at the same inference budget", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Finds extending thinking with prompts like \"Wait\" first helps then hurts, explained by rising output variance, and proposes spending the same budget on parallel paths with majority voting.", "key_idea": "Longer thinking is a mirage; parallel sampling uses the budget better.", "relevance_score": 3, "relevance_rationale": "Guidance on how to spend a thinking budget.", "found_via": "web search: s1 budget forcing (related results)", "cites_or_cited_by": [] }, { "id": "gim-2025-pie", "title": "Pie: A Programmable Serving System for Emerging LLM Applications", "authors": [ "In Gim", "Zhiyao Ma", "Seung-seob Lee", "Lin Zhong" ], "year": 2025, "venue_or_source": "SOSP 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2510.24051", "code_url": "https://github.com/pie-project/pie", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "application-defined inferlets" ], "cost_reduction_reported": "3-12% latency overhead on standard tasks and 1.3x-3.4x higher latency/throughput on agentic workflows", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Programmable serving where user WebAssembly \"inferlets\" control the generation loop with direct KV cache and forward-pass access, enabling application-specific agent optimizations without modifying the engine.", "key_idea": "Let applications program the serving loop.", "relevance_score": 3, "relevance_rationale": "Agent-specific serving optimizations via programmability.", "found_via": "Pie programmable serving system LLM agent inference loops", "cites_or_cited_by": [ "gim-2023-prompt-cache" ] }, { "id": "guerdan-2025-doubly-robust-llm-judge", "title": "Doubly-Robust LLM-as-a-Judge: Externally Valid Estimation with Imperfect Personas", "authors": [ "Luke Guerdan", "Justin Whitehouse", "Kimberly Truong", "Kenneth Holstein", "Zhiwei Steven Wu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.22957", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "persona-prompted LLM judge ratings", "human ratings" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Addresses the gap between lab evaluation samples and deployment distributions with a doubly robust estimator that combines human ratings and persona-conditioned LLM-judge ratings to estimate deployment-time system quality.", "key_idea": "Doubly robust estimation of deployment quality using imperfect LLM-judge personas.", "relevance_score": 3, "relevance_rationale": "Off-policy-style correction when eval traffic differs from production traffic.", "found_via": "query: doubly robust estimator LLM evaluation", "cites_or_cited_by": [] }, { "id": "han-2025-rluf", "title": "Reinforcement Learning from User Feedback", "authors": [ "Eric Han", "Jun Chen", "Karthik Abinav Sankararaman", "Xiao-Liang Peng", "Tengyu Xu", "Eryk Helenowski", "Kaiyan Peng", "Mrinal Kumar", "Si-Yuan Wang", "Han Fang", "Arya Talebzadeh" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.14946", "code_url": null, "topic_clusters": [ "C5", "C6", "C14" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "emoji Love reactions (implicit user feedback)" ], "cost_reduction_reported": "28% increase in Love Reactions in live A/B tests", "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Trains a reward model P[Love] to predict whether a response will receive a lightweight positive user reaction in production and uses it alongside helpfulness and safety objectives in policy optimization. Shows P[Love] predicts future user behavior offline and raises positive feedback in live A/B tests, while noting reward-hacking risks.", "key_idea": "Reward model trained on sparse binary production user reactions, validated by live A/B tests.", "relevance_score": 3, "relevance_rationale": "Production evidence that implicit user signals can be modeled offline and validated with A/B tests; same machinery could score routes.", "found_via": "query: implicit user feedback LLM production", "cites_or_cited_by": [] }, { "id": "han-2025-tale", "title": "Token-Budget-Aware LLM Reasoning", "authors": [ "Tingxu Han", "Zhenting Wang", "Chunrong Fang", "Shiyu Zhao", "Shiqing Ma", "Zhenyu Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.18547", "code_url": "https://github.com/GeniusHTX/TALE", "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "estimated per-problem token budget" ], "cost_reduction_reported": "Reduces token costs in CoT reasoning with only a slight performance reduction (no single number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows CoT can be compressed by stating a token budget in the prompt, but the chosen budget matters. TALE estimates a per-problem budget from reasoning complexity and uses it to guide generation; a post-trained variant internalises budget awareness.", "key_idea": "Estimate and impose a per-problem token budget.", "relevance_score": 3, "relevance_rationale": "Seed; prompt-level budget control easy to apply per agent step.", "found_via": "seed: token-budget-aware reasoning (TALE)", "cites_or_cited_by": [] }, { "id": "he-2025-thinkdial", "title": "ThinkDial: An Open Recipe for Controlling Reasoning Effort in Large Language Models", "authors": [ "Qianyu He", "Siyu Yuan", "Xuefeng Li", "Mingxuan Wang", "Jiangjie Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.18773", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "discrete budget mode" ], "cost_reduction_reported": "Medium mode: 50% token reduction with <10% performance degradation; Low mode: 75% token reduction with <15% degradation", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Open recipe for gpt-oss-style discrete reasoning modes (high/medium/low) via budget-mode SFT and budget-aware RL.", "key_idea": "Open training recipe for discrete reasoning-effort modes.", "relevance_score": 3, "relevance_rationale": "Enables effort-level routing on open models.", "found_via": "arXiv title search: reasoning effort", "cites_or_cited_by": [] }, { "id": "hou-2025-edge-cloud-expert-cascades", "title": "Reliable LLM-Based Edge-Cloud-Expert Cascades for Telecom Knowledge Systems", "authors": [ "Qiushuo Hou", "Sangwoo Park", "Matteo Zecchin", "Yunlong Cai", "Guanding Yu", "Osvaldo Simeone", "Tommaso Melodia" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.20012", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "knowledge and confidence tests", "multiple hypothesis testing" ], "cost_reduction_reported": null, "benchmarks": [ "TeleQnA" ], "models_studied": [], "production_evidence": false, "summary": "Edge model, cloud model, then human expert cascade for telecom QA, with thresholds chosen by multiple hypothesis testing to give finite-sample guarantees on misalignment with experts while minimising cost.", "key_idea": "MHT-based threshold selection with risk guarantees.", "relevance_score": 3, "relevance_rationale": "Guaranteed thresholds for multi-tier cascades.", "found_via": "arXiv title search: cascade LLM", "cites_or_cited_by": [] }, { "id": "hou-2025-thinkprune", "title": "ThinkPrune: Pruning Long Chain-of-Thought of LLMs via Reinforcement Learning", "authors": [ "Bairu Hou", "Yang Zhang", "Jiabao Ji", "Yujian Liu", "Kaizhi Qian", "Jacob Andreas", "Shiyu Chang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.01296", "code_url": "https://github.com/UCSB-NLP-Chang/ThinkPrune", "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "token limit with zero reward beyond" ], "cost_reduction_reported": "On AIME24, reasoning length of DeepSeek-R1-Distill-Qwen-1.5B halved with only 2% performance drop", "benchmarks": [ "AIME24" ], "models_studied": [ "DeepSeek-R1-Distill-Qwen-1.5B" ], "production_evidence": false, "summary": "RL with a hard token limit, iteratively tightened over rounds, prunes thinking length while keeping the core reasoning.", "key_idea": "Iterative RL under shrinking token caps.", "relevance_score": 3, "relevance_rationale": "Training-side length reduction.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "huang-2025-pluralistic-ope", "title": "Pluralistic Off-policy Evaluation and Alignment", "authors": [ "Chyi Yun Huang", "Junda Wu", "Zhouhang Xie", "Yu Xia", "Rui Wang", "Tong Yu", "Subrata Mitra", "Julian McAuley", "Lina Yao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.19333", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "upvotes", "relevance scores", "diversity" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Extends off-policy evaluation to pluralistic preference alignment with decomposable IPS estimators for relevance and diversity, and uses them for off-policy optimization of LLM responses.", "key_idea": "Decomposed IPS OPE for pluralistic LLM alignment.", "relevance_score": 3, "relevance_rationale": "OPE machinery for LLMs from logged preference data.", "found_via": "snowball: cited-by bhargava-2024-ope-logged-human-feedback", "cites_or_cited_by": [ "bhargava-2024-ope-logged-human-feedback" ] }, { "id": "huang-2025-thriftllm", "title": "ThriftLLM: On Cost-Effective Selection of Large Language Models for Classification Queries", "authors": [ "Keke Huang", "Yimin Shi", "Dujian Ding", "Yifei Li", "Yang Fei", "Laks Lakshmanan", "Xiaokui Xiao" ], "year": 2025, "venue_or_source": "VLDB 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2501.04901", "code_url": null, "topic_clusters": [ "C1", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "correctness probability", "budget" ], "cost_reduction_reported": null, "benchmarks": [ "classification queries" ], "models_studied": [], "production_evidence": false, "summary": "ThriftLLM selects an ensemble of LLMs for classification queries that maximizes correctness probability under a cost budget, with guarantees.", "key_idea": "Budgeted ensemble selection with guarantees.", "relevance_score": 3, "relevance_rationale": "Theory for budget selection.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "ivanovici-2025-intercom-cost-of-serving", "title": "Cost of Serving LLMs", "authors": [ "Stefan Ivanovici" ], "year": 2025, "venue_or_source": "Fin.ai research blog (Intercom)", "source_type": "blog", "url": "https://fin.ai/research/cost-of-serving-llms/", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "throughput per dollar" ], "cost_reduction_reported": "Company-reported: self-hosted Qwen3 14B ~0.05x GPT-4.1 cost; Gemma 3 4B ~0.04x; Llama-70B distill ~1.7x GPT-4.1", "benchmarks": [], "models_studied": [ "Gemma 3 4B/27B", "Qwen3 14B", "DeepSeek Llama 70B", "GPT-4.1", "Sonnet 3.7" ], "production_evidence": true, "summary": "Intercom compares self-hosting open models under production latency targets to commercial API pricing and finds sub-30B models far cheaper per token.", "key_idea": "Self-hosting small models is much cheaper under latency SLOs.", "relevance_score": 3, "relevance_rationale": "Production cost analysis for downscaling.", "found_via": "seed list: Intercom case study", "cites_or_cited_by": [ "joglekar-2025-intercom-small-llms" ] }, { "id": "jackson-2025-cursor-tab-online-rl", "title": "Improving Cursor Tab with online RL", "authors": [ "Jacob Jackson", "Phillip Kravtsov", "Shomil Jain" ], "year": 2025, "venue_or_source": "Cursor blog", "source_type": "blog", "url": "https://cursor.com/blog/tab-rl", "code_url": null, "topic_clusters": [ "C5", "C7", "C14" ], "routing_granularity": "query", "learning_regime": "RL-online", "signals_used": [ "suggestion accept/reject" ], "cost_reduction_reported": "New Tab model makes 21% fewer suggestions with a 28% higher accept rate (production, >400M requests/day; vendor-reported)", "benchmarks": [ "Cursor production traffic" ], "models_studied": [ "Cursor Tab model" ], "production_evidence": true, "summary": "Cursor trains its completion model with on-policy policy-gradient RL from live accept/reject feedback, rolling out new checkpoints to users frequently throughout the day (about 1.5-2 hours from checkpoint rollout to data for the next step) to keep data on-policy. The reward is shaped so the policy shows a suggestion only when its implied acceptance probability exceeds a target, replacing a separate filter model.", "key_idea": "Frequent on-policy deployment turns live user feedback into an RL signal that also learns when not to act.", "relevance_score": 3, "relevance_rationale": "Rare production example of online RL from implicit feedback; transferable to learning abstention/escalation.", "found_via": "direct: cursor.com/blog", "cites_or_cited_by": [] }, { "id": "jhandi-2025-slm-agentic-tool-calling", "title": "Small Language Models for Efficient Agentic Tool Calling: Outperforming Large Models with Targeted Fine-tuning", "authors": [ "Polaris Jhandi", "Owais Kazi", "Shreyas Subramanian", "Neel Sendas" ], "year": 2025, "venue_or_source": "AAAI 2026 Workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2512.15943", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "ToolBench" ], "models_studied": [ "OPT-350M", "ChatGPT-CoT (baseline)", "ToolLLaMA (baseline)" ], "production_evidence": false, "summary": "AWS authors fine-tune a 350M-parameter OPT model for one epoch on tool-calling data and report a 77.55% ToolBench pass rate, above ChatGPT-CoT and ToolLLaMA baselines. Argues targeted SLMs can replace LLMs in routine enterprise agent steps.", "key_idea": "A tiny fine-tuned SLM can handle tool-calling steps.", "relevance_score": 3, "relevance_rationale": "Evidence for downscaling tool-call steps.", "found_via": "web search: SLM tool calling", "cites_or_cited_by": [] }, { "id": "jiang-2025-flashthink", "title": "FlashThink: An Early Exit Method For Efficient Reasoning", "authors": [ "Guochao Jiang", "Guofeng Quan", "Zepeng Ding", "Ziqin Luo", "Dixuan Wang", "Zheng Hu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.13949", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "verification model" ], "cost_reduction_reported": "Reasoning length reduced by 77.04% (DeepSeek-R1) and 77.47% (QwQ-32B) without reducing accuracy", "benchmarks": [ "four benchmarks" ], "models_studied": [ "DeepSeek-R1", "QwQ-32B" ], "production_evidence": false, "summary": "A separate verification model watches the reasoning and signals when the model can already answer correctly, triggering early exit.", "key_idea": "External verifier decides when to stop thinking.", "relevance_score": 3, "relevance_rationale": "Verifier-driven early exit.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "jiang-2025-lhrm", "title": "Think Only When You Need with Large Hybrid-Reasoning Models", "authors": [ "Lingjie Jiang", "Xun Wu", "Shaohan Huang", "Qingxiu Dong", "Zewen Chi", "Li Dong", "Xingxing Zhang", "Tengchao Lv", "Lei Cui", "Furu Wei" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.14631", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-online", "signals_used": [ "query context" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Trains large hybrid-reasoning models with hybrid fine-tuning then RL (HGPO) to decide whether to think per query, and proposes a Hybrid Accuracy metric.", "key_idea": "Hybrid fine-tuning plus HGPO for adaptive thinking.", "relevance_score": 3, "relevance_rationale": "Model-level hybrid thinking.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "jiao-2025-weak-strong-collaboration", "title": "Synergistic Weak-Strong Collaboration by Aligning Preferences", "authors": [ "Yizhu Jiao", "Xuchao Zhang", "Zhaoyang Wang", "Yubo Ma", "Zhun Deng", "Rujia Wang", "Chetan Bansal", "Saravan Rajmohan", "Jiawei Han", "Huaxiu Yao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.15188", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "collaborative feedback influence" ], "cost_reduction_reported": null, "benchmarks": [ "three domains" ], "models_studied": [], "production_evidence": false, "summary": "Pairs a domain-specialized weak model that drafts with a general strong model that refines, and preference-tunes the weak model with feedback quantifying how much its drafts helped the collaboration.", "key_idea": "Weak drafter plus strong refiner, with the weak model tuned on collaboration feedback.", "relevance_score": 3, "relevance_rationale": "Big-small collaboration pattern.", "found_via": "arXiv query ti:\"weak-strong\" OR ti:\"weak-to-strong\" AND ti:agent", "cites_or_cited_by": [] }, { "id": "jin-2025-radialrouter", "title": "RadialRouter: Structured Representation for Efficient and Robust Large Language Models Routing", "authors": [ "Ruihan Jin", "Pengpeng Shao", "Zhengqi Wen", "Jinyang Wu", "Mingkuan Feng", "Shuai Zhang", "Jianhua Tao" ], "year": 2025, "venue_or_source": "EMNLP 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.03880", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "RadialFormer query-LLM representation", "KL + contrastive loss" ], "cost_reduction_reported": "Outperforms existing routers by 9.2% (Balance) and 5.8% (Cost First) on RouterBench (abstract)", "benchmarks": [ "RouterBench" ], "models_studied": [], "production_evidence": false, "summary": "RadialRouter uses a lightweight radial-structured Transformer to model query-LLM relationships and selects the model from its final states, trained with KL plus query-query contrastive loss.", "key_idea": "Radial Transformer backbone for robust routing.", "relevance_score": 3, "relevance_rationale": "Incremental learned router.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "jung-2025-futureweaver", "title": "FutureWeaver: Planning Test-Time Compute for Multi-Agent Systems with Modularized Collaboration", "authors": [ "Dongwon Jung", "Peng Shi", "Muhao Chen", "Yi Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.11213", "code_url": null, "topic_clusters": [ "C3", "C4", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "budget", "lookahead planning" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "FutureWeaver plans test-time compute allocation across a multi-agent system under a fixed budget, using collaboration modules induced by self-play reflection and dual-level planning over short-horizon actions and long-horizon abstractions.", "key_idea": "Budgeted planning of compute across MAS collaboration modules.", "relevance_score": 3, "relevance_rationale": "Budget allocation within MAS.", "found_via": "snowball: cited-by zhang-2025-maas-agentic-supernet | snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [ "zhang-2025-maas-agentic-supernet" ] }, { "id": "kadekodi-2025-dualtune", "title": "AgentFlux: Decoupled Fine-Tuning & Inference for On-Device Agentic Systems", "authors": [ "Rohan Kadekodi", "Zhan Jin", "Keisuke Kamahori", "Yile Gu", "Sean Khatiri", "Noah H. Bayindirli", "Sergey Gorbunov", "Baris Kasikci" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.00229", "code_url": null, "topic_clusters": [ "C8", "C11" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "decoupled tool-selection and argument LoRAs" ], "cost_reduction_reported": "Qwen-2.5-7B tool-calling accuracy improves 46% on MCP-Bench", "benchmarks": [ "MCP-Bench" ], "models_studied": [ "Qwen-2.5-7B" ], "production_evidence": false, "summary": "Decouples fine-tuning into tool selection and argument generation LoRA adapters for on-device agent orchestration.", "key_idea": "Decoupled LoRAs for local tool agents.", "relevance_score": 3, "relevance_rationale": "Small agent specialization.", "found_via": "hf-papers search: small model orchestrator", "cites_or_cited_by": [] }, { "id": "kadu-2025-reflexgrad", "title": "ReflexGrad: Within-Episode Failure Recovery in LLM Agents via Progress-Gated Dual-Process Routing", "authors": [ "Ankush Kadu", "Aswanth Krishnan" ], "year": 2025, "venue_or_source": "ICML 2026 Workshop (FoGen)", "source_type": "paper", "url": "https://arxiv.org/abs/2511.14584", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "progress scores" ], "cost_reduction_reported": null, "benchmarks": [ "ALFWorld" ], "models_studied": [ "Qwen-3-8B", "GPT-5" ], "production_evidence": false, "summary": "ReflexGrad routes within an episode between a fast refinement process and a slow causal-diagnosis process when consecutive low progress scores trigger a gate. It raises Qwen-3-8B from 35.1% to 75.4% on ALFWorld and beats compute-matched LATS and ToT.", "key_idea": "Progress-gated switching between fast and slow processes inside a trajectory.", "relevance_score": 3, "relevance_rationale": "Progress signal as a trigger for heavier processing within trajectories.", "found_via": "OpenAlex search: per-step model routing", "cites_or_cited_by": [] }, { "id": "kassem-2025-router-robustness", "title": "How Robust Are Router-LLMs? Analysis of the Fragility of LLM Routing Capabilities", "authors": [ "Aly M. Kassem", "Bernhard Schölkopf", "Zhijing Jin" ], "year": 2025, "venue_or_source": "EACL (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2504.07113", "code_url": null, "topic_clusters": [ "C1", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "preference-based routers", "commercial routers" ], "cost_reduction_reported": null, "benchmarks": [ "DSC benchmark" ], "models_studied": [], "production_evidence": false, "summary": "Introduces the DSC benchmark to test routers by query category, including privacy and safety. Finds preference-trained routers make category-driven choices, e.g. a BERT router sends all code and math to the strongest model and jailbreaks to weaker models.", "key_idea": "Routers are fragile and category-biased; safety risks.", "relevance_score": 3, "relevance_rationale": "Evaluation caveat.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "kilpatrick-2025-gemini-implicit-caching", "title": "Gemini 2.5 Models now support implicit caching", "authors": [ "Logan Kilpatrick" ], "year": 2025, "venue_or_source": "Google Developers Blog", "source_type": "blog", "url": "https://developers.googleblog.com/en/gemini-2-5-models-now-support-implicit-caching/", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "shared prompt prefix" ], "cost_reduction_reported": "(vendor-reported) 75% token discount on implicit cache hits (min 1,024 tokens Flash, 2,048 Pro)", "benchmarks": [], "models_studied": [ "Gemini 2.5 Flash", "Gemini 2.5 Pro" ], "production_evidence": false, "summary": "Announces automatic prefix-based caching on Gemini 2.5 with the same discount as explicit caching and no setup.", "key_idea": "Automatic prefix caching discount.", "relevance_score": 3, "relevance_rationale": "Provider cost lever.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [ "google-2024-gemini-context-caching" ] }, { "id": "kim-2025-cost-of-dynamic-reasoning", "title": "The Cost of Dynamic Reasoning: Demystifying AI Agents and Test-Time Scaling from an AI Infrastructure Perspective", "authors": [ "Jiin Kim", "Byeongjun Shin", "Jinha Chung", "Minsoo Rhu" ], "year": 2025, "venue_or_source": "HPCA 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2506.04301", "code_url": null, "topic_clusters": [ "C11", "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Agents show rapidly diminishing accuracy returns with increased compute (system-level resource, latency, energy and power analysis)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A system-level analysis of AI agents quantifying resource use, latency, energy and datacenter power across agent designs and test-time scaling strategies. Accuracy gains from more compute diminish quickly while infrastructure costs grow, raising sustainability concerns.", "key_idea": "Agents' compute returns diminish fast; system costs do not.", "relevance_score": 3, "relevance_rationale": "Infrastructure view on agent cost.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai | snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "kleinman-2025-e1-adaptive-effort", "title": "e1: Learning Adaptive Control of Reasoning Effort", "authors": [ "Michael Kleinman", "Matthew Trager", "Alessandro Achille", "Wei Xia", "Stefano Soatto" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.27042", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "relative effort fraction parameter" ], "cost_reduction_reported": "2-3x reduction in CoT length while maintaining or improving performance relative to the base model (1.5B-32B)", "benchmarks": [], "models_studied": [ "1.5B-32B models" ], "production_evidence": false, "summary": "Trains models to use a user-chosen fraction of their average CoT length for each query, giving a continuous effort knob that adapts to difficulty without absolute token budgets.", "key_idea": "Relative effort control instead of absolute token budgets.", "relevance_score": 3, "relevance_rationale": "Controllable effort knob for routers.", "found_via": "arXiv title search: reasoning effort", "cites_or_cited_by": [] }, { "id": "kolagani-2025-crm-multi-model-routing", "title": "Cost-Aware Agentic Architectures for Multi-Model Routing and Tool-Use Optimisation in CRM Workflows", "authors": [ "Sri Hari Deep Kolagani", "Dr Mamata Bhandar" ], "year": 2025, "venue_or_source": "International Journal of Leading Research Publication", "source_type": "paper", "url": "https://doi.org/10.70528/ijlrp.v6.i7.1866", "code_url": null, "topic_clusters": [ "C3", "C14" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "task tier" ], "cost_reduction_reported": "Dynamic task routing saved 33% API cost vs always-largest LLM with CSAT >90% and FCR >85%; up to 35% operating cost savings", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies dispatching CRM post-sale tasks such as case escalation and email summarization to small, medium or large LLM tiers on Salesforce Service Cloud, alongside vector-DB query optimization and API-call limits.", "key_idea": "Tiered model routing in CRM agent workflows.", "relevance_score": 3, "relevance_rationale": "Enterprise mixed-tier agent report; low-rigor venue.", "found_via": "Semantic Scholar query \"per-turn model routing coding agent\"", "cites_or_cited_by": [] }, { "id": "lan-2025-expert-failures-agent-tuning", "title": "Exploring Expert Failures Improves LLM Agent Tuning", "authors": [ "Li-Cheng Lan", "Andrew Bai", "Minhao Cheng", "Cho-Jui Hsieh", "Tianyi Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.13145", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "failed expert trajectories" ], "cost_reduction_reported": null, "benchmarks": [ "WebShop", "SciWorld", "ALFWorld" ], "models_studied": [ "GPT-4", "Llama" ], "production_evidence": false, "summary": "Shows failed GPT-4 expert trajectories contain useful plans and actions; mining them improves rejection-sampling fine-tuning on hard subtasks.", "key_idea": "Exploit failed teacher traces in agent SFT.", "relevance_score": 3, "relevance_rationale": "Relevant for training on noisy production logs.", "found_via": "hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [] }, { "id": "lee-2025-cais-opt-survey", "title": "Compound AI Systems Optimization: A Survey of Methods, Challenges, and Future Directions", "authors": [ "Yu-Ang Lee", "Guan-Ting Yi", "Mei-Yi Liu", "Jui-Chao Lu", "Guan-Bo Yang", "Yun-Nung Chen" ], "year": 2025, "venue_or_source": "EMNLP 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.08234", "code_url": null, "topic_clusters": [ "C4", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Systematic survey of compound AI system optimization covering numerical (SFT, RL) and language-feedback-based methods, with a formalization and taxonomy of existing approaches and open challenges.", "key_idea": "Taxonomy of compound AI optimization methods.", "relevance_score": 3, "relevance_rationale": "Survey that maps C4.", "found_via": "search: compound AI optimization survey arXiv", "cites_or_cited_by": [] }, { "id": "lee-2025-prints", "title": "PRInTS: Reward Modeling for Long-Horizon Information Seeking", "authors": [ "Jaewoo Lee", "Archiki Prasad", "Justin Chih-Yao Chen", "Zaid Khan", "Elias Stengel-Eskin", "Mohit Bansal" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.19314", "code_url": null, "topic_clusters": [ "C7", "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "generative PRM", "trajectory summarization" ], "cost_reduction_reported": null, "benchmarks": [ "FRAMES", "GAIA", "WebWalkerQA" ], "models_studied": [], "production_evidence": false, "summary": "Generative PRM for long-horizon information seeking that scores steps across quality dimensions and summarizes growing context for evaluation; improves best-of-n for multiple agents.", "key_idea": "PRM with built-in context compression for long trajectories.", "relevance_score": 3, "relevance_rationale": "Step-level signal for long agent runs.", "found_via": "hf-papers search: process reward model agents tool use", "cites_or_cited_by": [] }, { "id": "li-2025-agentswift", "title": "AgentSwift: Efficient LLM Agent Design via Value-guided Hierarchical Search", "authors": [ "Yu Li", "Lehui Li", "Zhihao Wu", "Qingmin Liao", "Jianye Hao", "Kun Shao", "Fengli Xu", "Yong Li" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2506.06017", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "value model predictions", "uncertainty" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AgentSwift searches a hierarchical space of agent workflows and functional components using uncertainty-guided hierarchical MCTS and a learned value model to cut evaluation cost, gaining 8.34% on average across seven benchmarks.", "key_idea": "Value-model-guided hierarchical MCTS for agent design.", "relevance_score": 3, "relevance_rationale": "Reduces cost of agent design search via learned predictors.", "found_via": "search: GPTSwarm Language Agents as Optimizable Graphs arXiv", "cites_or_cited_by": [ "shang-2024-agentsquare" ] }, { "id": "li-2025-budget-guidance", "title": "Steering LLM Thinking with Budget Guidance", "authors": [ "Junyan Li", "Wenshuo Zhao", "Yang Zhang", "Chuang Gan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.13752", "code_url": "https://github.com/UMass-Embodied-AGI/BudgetGuidance", "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "predicted remaining thinking length (Gamma model)" ], "cost_reduction_reported": "Up to 26% accuracy gain on MATH-500 under tight budgets vs baselines; competitive accuracy with 63% of full-thinking tokens", "benchmarks": [ "MATH-500" ], "models_studied": [], "production_evidence": false, "summary": "A small predictor models the remaining thinking length during decoding and softly steers generation toward a target budget, without fine-tuning the LLM.", "key_idea": "Soft token-level steering to a thinking budget.", "relevance_score": 3, "relevance_rationale": "Fine-tuning-free budget control.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "li-2025-self-moa", "title": "Rethinking Mixture-of-Agents: Is Mixing Different Large Language Models Beneficial?", "authors": [ "Wenzhe Li", "Yong Lin", "Mengzhou Xia", "Chi Jin" ], "year": 2025, "venue_or_source": "TMLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2502.00674", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "AlpacaEval 2.0", "MMLU", "CRUX", "MATH" ], "models_studied": [], "production_evidence": false, "summary": "Finds that Self-MoA, which aggregates multiple samples from only the best single model, beats standard MoA that mixes different LLMs in many settings (+6.6% on AlpacaEval 2.0). Mixing models often lowers average quality; the paper characterizes when mixing helps.", "key_idea": "Mixing heterogeneous models is often worse than sampling the best one.", "relevance_score": 3, "relevance_rationale": "Caution on heterogeneous composition; informs model-mix decisions.", "found_via": "web search: rethinking MoA", "cites_or_cited_by": [] }, { "id": "li-2025-selfbudgeter", "title": "SelfBudgeter: Adaptive Token Allocation for Efficient LLM Reasoning", "authors": [ "Zheng Li", "Qingxiu Dong", "Jingyuan Ma", "Di Zhang", "Kai Jia", "Zhifang Sui" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.11274", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "self-estimated reasoning budget" ], "cost_reduction_reported": "Average response length compression of 61% on math reasoning tasks while maintaining accuracy", "benchmarks": [ "math reasoning tasks" ], "models_studied": [], "production_evidence": false, "summary": "Trains the model to first predict how many tokens a query needs, then uses budget-guided GRPO to respect that budget. Users can see or override the budget up front.", "key_idea": "Model predicts its own token budget before reasoning.", "relevance_score": 3, "relevance_rationale": "Explicit per-query budget estimates useful for schedulers.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "li-2025-small-model-learnability-gap", "title": "Small Models Struggle to Learn from Strong Reasoners", "authors": [ "Yuetai Li", "Xiang Yue", "Zhangchen Xu", "Fengqing Jiang", "Luyao Niu", "Bill Yuchen Lin", "Bhaskar Ramasubramanian", "Radha Poovendran" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.12143", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "long vs short CoT" ], "cost_reduction_reported": null, "benchmarks": [ "MATH", "GSM8K", "AIME" ], "models_studied": [ "Qwen2.5 0.5B-3B", "Llama" ], "production_evidence": false, "summary": "Finds small models (<=3B) learn better from shorter CoT or smaller teachers than from strong long-CoT reasoners, proposing mix distillation.", "key_idea": "Small students need teachers matched to their capacity.", "relevance_score": 3, "relevance_rationale": "Guides teacher choice when distilling.", "found_via": "hf-papers search: agent distillation small models", "cites_or_cited_by": [] }, { "id": "liang-2025-thinkswitcher", "title": "ThinkSwitcher: When to Think Hard, When to Think Fast", "authors": [ "Guosheng Liang", "Longguang Zhong", "Ziyi Yang", "Xiaojun Quan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.14183", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted relative performance of short vs long CoT" ], "cost_reduction_reported": "Reduces computational cost by 20-30% while maintaining high accuracy on complex tasks", "benchmarks": [ "multiple reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "A small switching module, trained on the relative success of short and long CoT modes, decides per query which mode a single reasoning model should use.", "key_idea": "Lightweight switcher between short and long CoT.", "relevance_score": 3, "relevance_rationale": "Simple learned mode router.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "lin-2025-cuarewardbench", "title": "CUARewardBench: A Benchmark for Evaluating Reward Models on Computer-using Agent", "authors": [ "Haojia Lin", "Xiaoyu Tan", "Qin, Yulei", "Zihan Xu", "Yuchen Shi", "Zongyi Li", "Gang Li", "Shaofei Cai", "Siqi Cai", "Chaoyou Fu", "Ke Li", "Xing Wei Sun" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.18596", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "ORM", "PRM" ], "cost_reduction_reported": null, "benchmarks": [ "CUARewardBench" ], "models_studied": [], "production_evidence": false, "summary": "First benchmark for outcome and process reward models on computer-using agents, with expert-annotated trajectories from 10 software categories and 7 agent architectures.", "key_idea": "Benchmark for ORMs/PRMs on computer-use agents.", "relevance_score": 3, "relevance_rationale": "Tells whether step-level judges transfer to CUA workloads.", "found_via": "WebSearch: agent trajectory evaluation step-level reward benchmark", "cites_or_cited_by": [] }, { "id": "liu-2025-answer-convergence-early-stop", "title": "Answer Convergence as a Signal for Early Stopping in Reasoning", "authors": [ "Xin Liu", "Lu Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.02536", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "hybrid", "signals_used": [ "answer consistency", "end-of-reasoning probability", "internal activations" ], "cost_reduction_reported": "On NaturalQuestions, Answer Consistency reduces tokens by over 40% while improving accuracy", "benchmarks": [ "five benchmarks incl. NaturalQuestions" ], "models_studied": [ "five open-weight LLMs" ], "production_evidence": false, "summary": "Finds models usually converge to their final answer after about 60% of reasoning steps and proposes three stopping methods: answer consistency, boosting end-of-thinking probability and a learned probe.", "key_idea": "Answers converge early; stop when they stabilise.", "relevance_score": 3, "relevance_rationale": "Simple inference-time stopping rules.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "liu-2025-budget-aware-tool-use", "title": "Budget-Aware Tool Use Enables Effective Agent Scaling", "authors": [ "Tengxiao Liu", "Zifeng Wang", "Jin Miao", "I-Hung Hsu", "Jun Yan", "Jiefeng Chen", "Rujun Han", "Fangyuan Xu", "Yanfei Chen", "Ke Jiang", "Samira Daruki", "Yi Liang", "William Yang Wang", "Tomas Pfister", "Chen-Yu Lee" ], "year": 2025, "venue_or_source": "COLM 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2511.17006", "code_url": null, "topic_clusters": [ "C9", "C12", "C13" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "remaining budget" ], "cost_reduction_reported": "Budget-aware methods push the cost-performance Pareto frontier under a unified token+tool cost metric", "benchmarks": [ "web search agent benchmarks (e.g., BrowseComp)" ], "models_studied": [], "production_evidence": false, "summary": "Shows that simply raising tool-call budgets does not help web agents because they lack budget awareness, and introduces a Budget Tracker plug-in and BATS, which adapts planning and verification to the remaining budget. Proposes a unified cost metric over tokens and tool calls.", "key_idea": "Make agents budget-aware to scale tool use efficiently.", "relevance_score": 3, "relevance_rationale": "Tool-call cost control within agents.", "found_via": "web search: Efficient Agents", "cites_or_cited_by": [ "wang-2025-efficient-agents" ] }, { "id": "liu-2025-context-as-a-tool", "title": "Context as a Tool: Context Management for Long-Horizon SWE-Agents", "authors": [ "Shukai Liu", "Jian Yang", "Bo Jiang", "Yizhi Li", "Jinyang Guo", "Xianglong Liu", "Bryan Dai" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.22087", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "context-management actions" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-Bench-Verified" ], "models_studied": [ "SWE-Compressor" ], "production_evidence": false, "summary": "Treats context maintenance as a tool the agent can call; offline data injects context-management actions into trajectories to train SWE-Compressor, which reaches 57.6% on SWE-Bench-Verified under a bounded context budget.", "key_idea": "Make context management an explicit, trained agent action.", "relevance_score": 3, "relevance_rationale": "Learned context management for SWE agents under a budget.", "found_via": "SWE-agent context management trajectory reduction cost coding agents arXiv", "cites_or_cited_by": [ "sun-2025-context-folding" ] }, { "id": "liu-2025-off-cmab", "title": "Offline Learning for Combinatorial Multi-armed Bandits", "authors": [ "Xutong Liu", "Xiangxiang Dai", "Jinhang Zuo", "Siwei Wang", "Carlee-Joe Wong", "John C. S. Lui", "Wei Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.19300", "code_url": null, "topic_clusters": [ "C6", "C10" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "offline logged data", "pessimistic LCB" ], "cost_reduction_reported": null, "benchmarks": [ "learning to rank", "LLM caching", "influence maximization" ], "models_studied": [], "production_evidence": false, "summary": "Off-CMAB learns combinatorial bandit policies from offline logs with a pessimistic lower-confidence-bound algorithm, with applications including LLM caching.", "key_idea": "Offline pessimistic learning for combinatorial bandits.", "relevance_score": 3, "relevance_rationale": "Learning from logs instead of costly online exploration.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "dai-2024-c2mab-v" ] }, { "id": "liu-2025-rcr-router", "title": "RCR-Router: Efficient Role-Aware Context Routing for Multi-Agent LLM Systems with Structured Memory", "authors": [ "Jun Liu", "Zhenglun Kong", "Changdi Yang", "Fan Yang", "Tianqi Li", "Peiyan Dong", "Joannah Nanjekye", "Hao Tang", "Geng Yuan", "Wei Niu", "Wenbin Zhang", "Pu Zhao", "Xue Lin", "Dong Huang", "Yanzhi Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.04903", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "role", "task stage", "relevance score" ], "cost_reduction_reported": "Up to 30% token reduction while improving or maintaining answer quality", "benchmarks": [ "HotPotQA", "MuSiQue", "2WikiMultihop" ], "models_studied": [], "production_evidence": false, "summary": "RCR-Router routes only the relevant subset of shared memory to each agent based on its role and task stage under a strict token budget, and iteratively merges outputs back into structured memory. It cuts tokens while keeping answer quality on multi-hop QA.", "key_idea": "Role- and stage-aware context routing to each agent under a token budget.", "relevance_score": 3, "relevance_rationale": "Context-level cost lever for MAS.", "found_via": "arXiv title search: routing multi-agent system LLM", "cites_or_cited_by": [ "ye-2025-x-mas" ] }, { "id": "liu-2025-user-feedback-noisy-signal", "title": "User Feedback in Human-LLM Dialogues: A Lens to Understand Users But Noisy as a Learning Signal", "authors": [ "Yuhan Liu", "Michael J.Q. Zhang", "Eunsol Choi" ], "year": 2025, "venue_or_source": "Conference on Empirical Methods in Natural Language Processing", "source_type": "paper", "url": "https://arxiv.org/abs/2507.23158", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "implicit user feedback in conversation logs" ], "cost_reduction_reported": null, "benchmarks": [ "MT-Bench", "WildBench" ], "models_studied": [], "production_evidence": false, "summary": "Analyzes when and why users give implicit feedback in WildChat and LMSYS logs, then tests whether that feedback improves models. Gains appear on short MT-Bench questions but not on longer WildBench tasks, showing the signal is informative but noisy.", "key_idea": "Implicit feedback in chat logs explains users but is a noisy training signal.", "relevance_score": 3, "relevance_rationale": "Direct evidence on how reliable log-mined feedback is for learning quality predictors.", "found_via": "query: implicit user feedback LLM production", "cites_or_cited_by": [] }, { "id": "loghmani-2025-deconfoundlm", "title": "Aligning Language Models with Observational Data: Opportunities and Risks from a Causal Perspective", "authors": [ "Erfan Loghmani" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.00152", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "observational business outcomes" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies fine-tuning LLMs on historical observational outcomes such as conversion rates instead of A/B test data, showing that naive training learns spurious correlations. Proposes DeconfoundLM to remove known confounders from the reward signal, evaluated on real datasets.", "key_idea": "Observational outcome logs need deconfounding before use as LLM training signal.", "relevance_score": 3, "relevance_rationale": "The confounding issue applies equally to learning routers from production outcomes.", "found_via": "query: counterfactual learning to route observed outcome only", "cites_or_cited_by": [] }, { "id": "lu-2025-agentrewardbench", "title": "AgentRewardBench: Evaluating Automatic Evaluations of Web Agent Trajectories", "authors": [ "Xing Han Lù", "Amirhossein Kazemnejad", "Nicholas Meade", "Arkil Patel", "Dongchan Shin", "Alejandra Zambrano", "Karolina Stańczak", "Peter Shaw", "Christopher J. Pal", "Siva Reddy" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.08942", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "LLM judge", "trajectory evaluation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Benchmark of 1302 expert-reviewed web-agent trajectories across 5 benchmarks and 4 LLM agents for assessing LLM judges of success, side effects and repetitiveness. Finds no single judge excels and rule-based evaluation underreports success.", "key_idea": "Meta-evaluation of LLM judges for web-agent trajectories.", "relevance_score": 3, "relevance_rationale": "Tells how far trajectory judges can be trusted for routing/feedback.", "found_via": "WebSearch: agent trajectory evaluation step-level reward benchmark", "cites_or_cited_by": [] }, { "id": "lu-2025-autoannotator", "title": "From LLM-anation to LLM-orchestrator: Coordinating Small Models for Data Labeling", "authors": [ "Yao Lu", "Zhaiyuan Ji", "Jiawei Du", "Yu Shanqing", "Qi Xuan", "Tianyi Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.16393", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "subagent", "learning_regime": "hybrid", "signals_used": [ "multi-model voting", "difficult samples" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AutoAnnotator uses an LLM meta-controller to choose SLMs, generate annotation code and review hard samples, while a layer of task-specialist SLMs labels by voting; hard samples continually fine-tune the SLMs.", "key_idea": "LLM as orchestrator of cheap specialist SLMs.", "relevance_score": 3, "relevance_rationale": "Big-orchestrator/small-worker split with continual learning.", "found_via": "arXiv query ti:orchestrator AND abs:\"small\" AND abs:\"cost\" AND abs:LLMs", "cites_or_cited_by": [] }, { "id": "luo-2025-ada-r1", "title": "Ada-R1: Hybrid-CoT via Bi-Level Adaptive Reasoning Optimization", "authors": [ "Haotian Luo", "Haiying He", "Yibo Wang", "Jinluan Yang", "Rui Liu", "Naiqiang Tan", "Xiaochun Cao", "Dacheng Tao", "Li Shen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.21659", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "group- and instance-level preferences" ], "cost_reduction_reported": "Average reasoning length reduced by more than 50% on five math datasets while maintaining performance", "benchmarks": [ "five math datasets" ], "models_studied": [], "production_evidence": false, "summary": "Merges long- and short-CoT models into a hybrid, then applies bi-level preference training to choose a reasoning style per problem and prefer concise correct reasoning within it.", "key_idea": "Model merging plus bi-level preference optimisation.", "relevance_score": 3, "relevance_rationale": "Adaptive reasoning length.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "luo-2025-agent-lightning", "title": "Agent Lightning: Train ANY AI Agents with Reinforcement Learning", "authors": [ "Xufang Luo", "Yuge Zhang", "Zhiyuan He", "Zilong Wang", "Siyun Zhao", "Dongsheng Li", "Luna K. Qiu", "Yuqing Yang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.03680", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "transition-level rewards" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Microsoft framework decoupling agent execution from RL training so any existing agent (LangChain, AutoGen, etc.) can be RL-trained with minimal code change.", "key_idea": "Train any existing agent with RL via decoupled traces.", "relevance_score": 3, "relevance_rationale": "Practical bridge from production agents' traces to RL.", "found_via": "keyword: RL fine-tuning for agents", "cites_or_cited_by": [] }, { "id": "luo-2025-o1-pruner", "title": "O1-Pruner: Length-Harmonizing Fine-Tuning for O1-Like Reasoning Pruning", "authors": [ "Haotian Luo", "Li Shen", "Haiying He", "Yibo Wang", "Shiwei Liu", "Wei Li", "Naiqiang Tan", "Xiaochun Cao", "Dacheng Tao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.12570", "code_url": "https://github.com/StarDewXXX/O1-Pruner", "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "pre-sampled baseline accuracy and length" ], "cost_reduction_reported": null, "benchmarks": [ "mathematical reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Estimates baseline performance by pre-sampling and then fine-tunes with an RL-style objective that rewards shorter reasoning under accuracy constraints.", "key_idea": "Length-harmonising fine-tuning for long-thought models.", "relevance_score": 3, "relevance_rationale": "Training-side reasoning length reduction.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "ma-2025-automaas", "title": "AutoMaAS: Self-Evolving Multi-Agent Architecture Search for Large Language Models", "authors": [ "Bo Ma", "Hang Li", "ZeHua Hu", "XiaoFan Gui", "LuYao Liu", "Simon Liu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.02669", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "performance-cost analysis", "online feedback" ], "cost_reduction_reported": "1.0-7.1% performance improvement with 3-5% lower inference cost vs SOTA", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "AutoMaAS extends agentic supernet search with automatic operator generation, fusion and elimination based on performance-cost analysis, plus online feedback for continual refinement. Gains over MaAS are modest in both accuracy and cost.", "key_idea": "Self-evolving supernet with cost-aware operator lifecycle.", "relevance_score": 3, "relevance_rationale": "Follow-up to MaAS with cost-aware search.", "found_via": "web search: MaAS", "cites_or_cited_by": [] }, { "id": "ma-2025-cot-valve", "title": "CoT-Valve: Length-Compressible Chain-of-Thought Tuning", "authors": [ "Xinyin Ma", "Guangnian Wan", "Runpeng Yu", "Gongfan Fang", "Xinchao Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.09601", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "parameter-space length direction" ], "cost_reduction_reported": "QwQ-32B-Preview: GSM8K chains 741 to 225 tokens (95.07% to 94.92%); AIME 6827 to 4629 tokens with one additional error", "benchmarks": [ "GSM8K", "AIME" ], "models_studied": [ "QwQ-32B-Preview" ], "production_evidence": false, "summary": "Finds a direction in parameter space that controls CoT length and tunes models to produce chains of varying length from a single model, with progressive compression.", "key_idea": "A parameter-space valve for CoT length.", "relevance_score": 3, "relevance_rationale": "Single model with tunable reasoning length.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "ma-2025-nothinking", "title": "Reasoning Models Can Be Effective Without Thinking", "authors": [ "Wenjie Ma", "Jingxuan He", "Charlie Snell", "Tyler Griggs", "Sewon Min", "Matei Zaharia" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.09858", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "NoThinking prompt", "verifier or confidence best-of-N" ], "cost_reduction_reported": "Token-matched NoThinking beats Thinking, e.g. 51.3 vs 28.9 on ACM 23 at 700 tokens; parallel NoThinking comparable to Thinking with up to 9x lower latency", "benchmarks": [ "seven reasoning datasets (math, theorem proving, coding)" ], "models_studied": [ "DeepSeek-R1-Distill-Qwen" ], "production_evidence": false, "summary": "Bypassing the thinking block via prompting is competitive with thinking at matched token budgets, and parallel NoThinking samples with aggregation match Thinking at much lower latency.", "key_idea": "Skip thinking and scale in parallel instead.", "relevance_score": 3, "relevance_rationale": "Cheap alternative to long reasoning.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "mao-2025-multi-expert-deferral-thesis", "title": "Theory and Algorithms for Learning with Multi-Class Abstention and Multi-Expert Deferral", "authors": [ "Anqi Mao" ], "year": 2025, "venue_or_source": "Ph.D. dissertation, New York University", "source_type": "paper", "url": "https://arxiv.org/abs/2512.22886", "code_url": null, "topic_clusters": [ "C2", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "surrogate losses", "H-consistency" ], "cost_reduction_reported": null, "benchmarks": [ "CIFAR-10", "CIFAR-100", "SVHN" ], "models_studied": [], "production_evidence": false, "summary": "Thesis on learning with abstention and multi-expert deferral, providing surrogate losses with H-consistency guarantees for single-stage and two-stage deferral, motivated by routing LLM queries to experts.", "key_idea": "Consistency theory for multi-expert deferral.", "relevance_score": 3, "relevance_rationale": "Theoretical base for learned routing/deferral.", "found_via": "search: learning to defer large language models", "cites_or_cited_by": [] }, { "id": "mcnamara-2025-shopify-sidekick", "title": "Building production-ready agentic systems: Lessons from Shopify Sidekick", "authors": [ "Andrew McNamara" ], "year": 2025, "venue_or_source": "Shopify Engineering blog", "source_type": "blog", "url": "https://shopify.engineering/building-production-ready-agentic-systems", "code_url": null, "topic_clusters": [ "C7", "C13", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judges calibrated to humans", "user simulator" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Lessons from Shopify's merchant assistant: just-in-time instructions returned with tool results, evaluation on ground-truth sets built from production conversations, LLM judges calibrated to human labels (Cohen's kappa from 0.02 to 0.61 vs 0.69 human), a merchant simulator for pre-release testing and gated rewards against reward hacking.", "key_idea": "Production-grounded judges and simulators for agent evaluation.", "relevance_score": 3, "relevance_rationale": "Evaluation methodology from real traffic, needed to validate cheaper configs.", "found_via": "seed list: Shopify case study", "cites_or_cited_by": [] }, { "id": "molinari-2025-rp-react", "title": "Reason-Plan-ReAct: A Reasoner-Planner Supervising a ReAct Executor for Complex Enterprise Tasks", "authors": [ "Gianni Molinari", "Fabio Ciravegna" ], "year": 2025, "venue_or_source": "AAAI 2026 Workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2512.03560", "code_url": null, "topic_clusters": [ "C3", "C10" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "ToolQA" ], "models_studied": [ "six open-weight reasoning models" ], "production_evidence": false, "summary": "RP-ReAct splits an enterprise agent into a reasoner-planner (a large reasoning model) that plans and reviews each sub-step and proxy-execution agents that carry out tool calls ReAct-style, with external storage for large tool outputs. It improves reliability over single-agent baselines on ToolQA.", "key_idea": "Reasoning-model planner supervising ReAct executors.", "relevance_score": 3, "relevance_rationale": "Planner-executor role split for enterprise tasks.", "found_via": "OpenAlex search: planner executor", "cites_or_cited_by": [] }, { "id": "muennighoff-2025-s1", "title": "s1: Simple test-time scaling", "authors": [ "Niklas Muennighoff", "Zitong Yang", "Weijia Shi", "Xiang Lisa Li", "Li Fei-Fei", "Hannaneh Hajishirzi", "Luke Zettlemoyer", "Percy Liang", "Emmanuel Candès", "Tatsunori Hashimoto" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.19393", "code_url": "https://github.com/simplescaling/s1", "topic_clusters": [ "C8", "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "thinking-token count", "1K curated reasoning traces" ], "cost_reduction_reported": "s1-32B exceeds o1-preview on competition math by up to 27% (MATH, AIME24); budget forcing extrapolates AIME24 from 50% to 57%", "benchmarks": [ "MATH", "AIME24", "GPQA" ], "models_studied": [ "Qwen2.5-32B-Instruct", "o1-preview", "Qwen2.5-32B" ], "production_evidence": false, "summary": "Fine-tunes Qwen2.5-32B on 1,000 curated reasoning traces and introduces budget forcing: terminate thinking at a token limit or append \"Wait\" to extend it. This gives controllable test-time scaling.", "key_idea": "Budget forcing: hard-stop or extend thinking with \"Wait\".", "relevance_score": 3, "relevance_rationale": "Seed; simplest decode-time thinking-budget control.", "found_via": "seed: s1 budget forcing | keyword: distillation reasoning", "cites_or_cited_by": [] }, { "id": "munkhbat-2025-self-training-concise", "title": "Self-Training Elicits Concise Reasoning in Large Language Models", "authors": [ "Tergel Munkhbat", "Namgyu Ho", "Seo Hyun Kim", "Yongjin Yang", "Yujin Kim", "Se-Young Yun" ], "year": 2025, "venue_or_source": "Findings of ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.20122", "code_url": "https://github.com/TergelMunkhbat/concise-reasoning", "topic_clusters": [ "C8", "C9" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "best-of-N shortest correct samples" ], "cost_reduction_reported": "30% reduction in output tokens on average across five model families on GSM8K and MATH while maintaining average accuracy", "benchmarks": [ "GSM8K", "MATH" ], "models_studied": [ "five model families" ], "production_evidence": false, "summary": "Fine-tunes models on their own concise reasoning paths gathered by best-of-N sampling and few-shot conditioning.", "key_idea": "Self-distil concise reasoning.", "relevance_score": 3, "relevance_rationale": "Cheap SFT for shorter reasoning.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "nath-2025-toolcomp", "title": "ToolComp: A Multi-Tool Reasoning & Process Supervision Benchmark", "authors": [ "Vaskar Nath", "Pranav Raja", "Claire Yoon", "Sean Hendryx" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.01290", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process supervision labels" ], "cost_reduction_reported": null, "benchmarks": [ "ToolComp" ], "models_studied": [], "production_evidence": false, "summary": "Multi-tool reasoning benchmark with step-wise process supervision labels; PRMs generalize better than ORMs (19% and 11% rank@1 gains).", "key_idea": "Process labels for tool-use trajectories.", "relevance_score": 3, "relevance_rationale": "Evidence PRMs beat ORMs for tool agents.", "found_via": "hf-papers search: process reward model agents tool use", "cites_or_cited_by": [] }, { "id": "ni-2025-astraea", "title": "Astraea: A State-Aware Scheduling Engine for LLM-Powered Agents", "authors": [ "Hongqiu Ni", "Jiabao Zhang", "Guopeng Li", "Zilong Wang", "Ruiqi Wu", "Chi Zhang", "Haisheng Tan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.14142", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "request historical state", "future predictions", "I/O vs compute intensity" ], "cost_reduction_reported": "average job completion time reduced by up to 25.5% vs baselines", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "State-aware hierarchical scheduling for agents with alternating compute and external API calls, using an enhanced HRRN policy and an adaptive KV manager for agent state during I/O waits.", "key_idea": "Schedule over the whole agent request lifecycle, not single segments.", "relevance_score": 3, "relevance_rationale": "Agent lifecycle-aware scheduling.", "found_via": "serving engine LLM agents agentic workflows scheduling arXiv 2025", "cites_or_cited_by": [ "luo-2025-autellix" ] }, { "id": "notdiamond-2025-prompt-adaptation-weaker-models", "title": "Weaker models, stronger results: How prompt adaptation levels the field, part 2", "authors": [ "Not Diamond" ], "year": 2025, "venue_or_source": "Not Diamond blog", "source_type": "blog", "url": "https://www.notdiamond.ai/blog/weaker-models-stronger-results", "code_url": null, "topic_clusters": [ "C4", "C14" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "task eval accuracy" ], "cost_reduction_reported": "clinc150 intent classification: Gemini 2.5 Flash with adapted prompt 97.5% vs 86.75% with the original prompt and 93% for Gemini 2.5 Pro with the original prompt (~30 minutes background optimisation; vendor-reported)", "benchmarks": [ "clinc150" ], "models_studied": [ "Gemini 2.5 Pro", "Gemini 2.5 Flash" ], "production_evidence": false, "summary": "Shows that a prompt written for a stronger model loses accuracy on a cheaper model, and that automatic per-model prompt adaptation recovers and exceeds the stronger model's score on an intent-classification task. Part of a series on prompt adaptation for model migration.", "key_idea": "Re-optimising prompts per target model is a prerequisite for downgrading to cheaper models.", "relevance_score": 3, "relevance_rationale": "Supporting technique for model downscaling; single dataset.", "found_via": "direct: notdiamond.ai/blog index", "cites_or_cited_by": [ "google-vertex-data-driven-prompt-optimizer" ] }, { "id": "openai-model-optimization-distillation-track", "title": "Model optimization track: fine-tuning, distillation and evals (OpenAI developer docs)", "authors": [ "OpenAI" ], "year": 2025, "venue_or_source": "OpenAI developer documentation", "source_type": "docs", "url": "https://developers.openai.com/tracks/model-optimization", "code_url": null, "topic_clusters": [ "C6", "C8", "C14" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "stored production inputs/outputs", "graders/evals" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "OpenAI models" ], "production_evidence": false, "summary": "OpenAI's guidance on its built-in distillation workflow: store production inputs and outputs (default in the Responses API), build an eval with graders, tune prompts on the large model, fine-tune a smaller student on the larger model's outputs, and re-evaluate the student before switching. It sets distillation alongside SFT, DPO and RFT as ways to cut latency and cost.", "key_idea": "Platform-native loop from logged frontier-model traffic to an evaluated smaller student model.", "relevance_score": 3, "relevance_rationale": "Productised distillation from production logs; no numbers.", "found_via": "direct: developers.openai.com/llms.txt", "cites_or_cited_by": [ "gola-2026-langsmith-fine-tuning" ] }, { "id": "orbach-2025-rag-hpo", "title": "An Analysis of Hyper-Parameter Optimization Methods for Retrieval Augmented Generation", "authors": [ "Matan Orbach", "Ohad Eytan", "Benjamin Sznajder", "Ariel Gera", "Odellia Boni", "Yoav Kantor", "Gal Bloch", "Omri Levy", "Hadas Abraham", "Nitzan Barzilay", "Eyal Shnarch", "Michael E. Factor", "Shila Ofek-Koifman", "Paula Ta-Shma", "Assaf Toledo" ], "year": 2025, "venue_or_source": "AAAI 2026 Workshop (NewFrontiers in IR)", "source_type": "paper", "url": "https://arxiv.org/abs/2505.03452", "code_url": null, "topic_clusters": [ "C4", "C13" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "RAG evaluation metrics" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Benchmarks five hyperparameter-optimization algorithms for RAG over five datasets, including a real-world product documentation set, with full grid-search references. Finds that greedy or random search is efficient and that optimizing model choice first beats following pipeline order.", "key_idea": "Empirical benchmark of RAG HPO methods; tune model selection first.", "relevance_score": 3, "relevance_rationale": "Evidence on how to search RAG configs efficiently, relevant to config search design.", "found_via": "search: AutoRAG automated RAG pipeline optimization arXiv", "cites_or_cited_by": [] }, { "id": "patel-2025-proxrouter", "title": "ProxRouter: Proximity-Weighted LLM Query Routing for Improved Robustness to Outliers", "authors": [ "Shivam Patel", "Neharika Jali", "Ankur Mallick", "Gauri Joshi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.09852", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query embedding similarity", "exponentially tilted aggregation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ProxRouter improves nonparametric kNN-style routers on outlier queries via exponentially tilted aggregation that balances bias and variance.", "key_idea": "Robust nonparametric routing for outliers.", "relevance_score": 3, "relevance_rationale": "Robustness of simple routers.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "pu-2025-thoughtterminator", "title": "THOUGHTTERMINATOR: Benchmarking, Calibrating, and Mitigating Overthinking in Reasoning Models", "authors": [ "Xiao Pu", "Michael Saxon", "Wenyue Hua", "William Yang Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.13367", "code_url": null, "topic_clusters": [ "C9", "C13" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "problem difficulty estimates" ], "cost_reduction_reported": null, "benchmarks": [ "DUMB500", "frontier reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Shows a clear relation between difficulty and optimal token spend and that reasoning models are poorly calibrated on easy problems. Introduces DUMB500 of trivial problems and a training-free decoding method that improves token calibration.", "key_idea": "Benchmark and fix token-spend calibration on easy problems.", "relevance_score": 3, "relevance_rationale": "Evaluation and mitigation of overthinking.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "pulishetty-2025-cross-attention-router", "title": "One Head, Many Models: Cross-Attention Routing for Cost-Aware LLM Selection", "authors": [ "Roshini Pulishetty", "Mani Kishan Ghantasala", "Keerthy Kaushik Dasoju", "Niti Mangwani", "Vishal Garimella", "Aditya Mate", "Somya Chatterjee", "Yue Kang", "Ehi Nosakhare", "Sadid Hasan", "Soundar Srinivasan" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.09782", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query and model embeddings cross-attention", "predicted quality and cost" ], "cost_reduction_reported": "Up to 6.6% improvement in AIQ and 2.9% in max performance over existing routers on RouterBench (abstract)", "benchmarks": [ "RouterBench" ], "models_studied": [], "production_evidence": false, "summary": "Uses a single-head cross-attention between query and model embeddings to predict response quality and cost, with an exponential reward to stabilize trade-offs across preferences.", "key_idea": "Cross-attention query-model router.", "relevance_score": 3, "relevance_rationale": "Incremental.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "qi-2025-anytime-reasoner-brpo", "title": "Optimizing Anytime Reasoning via Budget Relative Policy Optimization", "authors": [ "Penghui Qi", "Zichen Liu", "Tianyu Pang", "Chao Du", "Wee Sun Lee", "Min Lin" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.13438", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "RL-offline", "signals_used": [ "sampled token budgets", "dense verifiable rewards" ], "cost_reduction_reported": null, "benchmarks": [ "mathematical reasoning tasks" ], "models_studied": [], "production_evidence": false, "summary": "Trains for anytime reasoning by truncating thinking at budgets sampled from a prior and rewarding the summary answer at each cut; introduces Budget Relative Policy Optimization for variance reduction.", "key_idea": "Optimise answer quality at every budget, not only the maximum.", "relevance_score": 3, "relevance_rationale": "Anytime property matters when budgets vary per step.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "qu-2025-efficient-reasoning-lrm-survey", "title": "A Survey of Efficient Reasoning for Large Reasoning Models: Language, Multimodality, and Beyond", "authors": [ "Xiaoye Qu", "Yafu Li", "Zhao-Chen Su", "Weigao Sun", "Jianhao Yan", "Dongrui Liu", "Ganqu Cui", "Daizong Liu", "Shuxian Liang", "Junxian He", "Peng Li", "Wei Wei", "Jing Shao", "Chaochao Lu", "Yue Zhang", "Xian-Sheng Hua", "Bowen Zhou", "Yu Cheng", "et al." ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.21614", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of reasoning-efficiency issues in large reasoning models across pretraining to inference, noting token economy matters in agent systems.", "key_idea": "Efficient reasoning in LRMs.", "relevance_score": 3, "relevance_rationale": "Seed-type survey.", "found_via": "snowball: cited-by ong-2024-routellm | GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "ong-2024-routellm", "sui-2025-stop-overthinking" ] }, { "id": "qu-2025-mrt", "title": "Optimizing Test-Time Compute via Meta Reinforcement Fine-Tuning", "authors": [ "Yuxiao Qu", "Matthew Y. R. Yang", "Amrith Setlur", "Lewis Tunstall", "Edward Emanuel Beeching", "Ruslan Salakhutdinov", "Aviral Kumar" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.07572", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "RL-offline", "signals_used": [ "progress bonus (change in success likelihood)" ], "cost_reduction_reported": "2-3x relative gain in performance and roughly 1.5x gain in token efficiency for math reasoning vs outcome-reward RL", "benchmarks": [ "math reasoning" ], "models_studied": [], "production_evidence": false, "summary": "Frames test-time compute use as meta-RL and measures it by cumulative regret over output tokens, adding a dense progress reward per output block.", "key_idea": "Reward per-block progress to spend tokens efficiently.", "relevance_score": 3, "relevance_rationale": "Principled token-efficiency objective.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "rabanser-2025-gatekeeper", "title": "Gatekeeper: Improving Model Cascades Through Confidence Tuning", "authors": [ "Stephan Rabanser", "Nathalie Rauschmayr", "Achin Kulshrestha", "Petra Poklukar", "Wittawat Jitkrittum", "Sean Augenstein", "Congchao Wang", "Federico Tombari" ], "year": 2025, "venue_or_source": "ICML 2025 workshop (TTODLer-FM)", "source_type": "paper", "url": "https://arxiv.org/abs/2502.19335", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "small-model confidence (tuned)" ], "cost_reduction_reported": null, "benchmarks": [ "image classification", "language modelling", "vision-language tasks" ], "models_studied": [ "encoder-only, decoder-only, encoder-decoder models" ], "production_evidence": false, "summary": "Proposes a loss that fine-tunes the small model in a cascade to be confident only on inputs it gets right, improving deferral quality. A knob controls the trade-off between the small model's accuracy and deferral accuracy.", "key_idea": "Confidence-tuning loss for the small model so deferral decisions are sharper.", "relevance_score": 3, "relevance_rationale": "Training technique for better cascade deferral.", "found_via": "arXiv search: model cascade language model", "cites_or_cited_by": [] }, { "id": "rajasekaran-2025-effective-context-engineering", "title": "Effective context engineering for AI agents", "authors": [ "Prithvi Rajasekaran", "Ethan Dixon", "Carly Ryan", "Jeremy Hadfield" ], "year": 2025, "venue_or_source": "Anthropic engineering blog", "source_type": "blog", "url": "https://www.anthropic.com/engineering/effective-context-engineering-for-ai-agents", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude" ], "production_evidence": false, "summary": "Anthropic's guidance on curating the smallest high-signal set of tokens for agents: just-in-time retrieval via tools instead of preloading, compaction of history near context limits, clearing old tool results, structured note-taking outside the context window, and sub-agent architectures that return condensed results to a coordinator.", "key_idea": "Treat context as a finite budget; compact, clear tool results, take notes, and isolate work in sub-agents.", "relevance_score": 3, "relevance_rationale": "Authoritative practitioner framing of context-management levers for long-horizon agents.", "found_via": "Effective context engineering for AI agents Anthropic", "cites_or_cited_by": [ "kang-2025-acon", "langchain-2025-context-engineering", "lindenbauer-2025-complexity-trap" ] }, { "id": "roig-2025-kami", "title": "Towards a Standard, Enterprise-Relevant Agentic AI Benchmark: Lessons from 5.5 billion tokens' worth of agentic AI evaluations", "authors": [ "JV Roig" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.08042", "code_url": null, "topic_clusters": [ "C13", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "KAMI v0.1" ], "models_studied": [ "35 model configurations incl. Llama 4, Qwen 3" ], "production_evidence": false, "summary": "KAMI is an enterprise-focused agentic benchmark run over 170,000 items and 5.5B tokens across 35 model configurations. Traditional benchmark rankings poorly predict agentic enterprise performance, and the paper reports cost-performance trade-offs.", "key_idea": "Enterprise agentic benchmark with cost-performance trade-offs.", "relevance_score": 3, "relevance_rationale": "Evaluation evidence for picking models per enterprise agent task.", "found_via": "snowball: cited-by ye-2025-x-mas", "cites_or_cited_by": [] }, { "id": "ruan-2025-cortex-knowledge-cache", "title": "Cortex: Achieving Low-Latency, Cost-Efficient Remote Data Access For LLM via Semantic-Aware Knowledge Caching", "authors": [ "Chaoyi Ruan", "Chao Bi", "Kaiwen Zheng", "Ziji Shi", "Xinyi Wan", "Jialin Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.17360", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "semantic similarity", "small-LLM judger", "prefetching" ], "cost_reduction_reported": "up to 3.6x throughput with >85% cache hit rate on search workloads at accuracy nearly identical to non-cached baselines; 20% throughput gain on coding tasks", "benchmarks": [ "agentic search workloads", "coding tasks" ], "models_studied": [], "production_evidence": false, "summary": "Semantic-aware caching of remote data accessed by agents (e.g., search results), with a small co-located LLM judging whether a cached item satisfies the request and proactive prefetching.", "key_idea": "Semantic cache for agent tool/data accesses validated by a small LLM judge.", "relevance_score": 3, "relevance_rationale": "Tool-output caching for agents with a correctness check.", "found_via": "semantic cache LLM agents tool calls caching arXiv", "cites_or_cited_by": [] }, { "id": "shen-2025-dast", "title": "DAST: Difficulty-Adaptive Slow-Thinking for Large Reasoning Models", "authors": [ "Yi Shen", "Jian Zhang", "Jieyun Huang", "Shuming Shi", "Wenjing Zhang", "Jiangze Yan", "Ning Wang", "Kai Wang", "Zhaoxiang Liu", "Shiguo Lian" ], "year": 2025, "venue_or_source": "EMNLP 2025 Industry Track", "source_type": "paper", "url": "https://arxiv.org/abs/2503.04472", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "Token Length Budget difficulty metric" ], "cost_reduction_reported": "Reduces token usage by over 30% on average while preserving accuracy on complex problems", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defines a token length budget metric for difficulty and uses budget-aware reward shaping and preference optimisation so easy problems get short CoT and hard ones enough reasoning.", "key_idea": "Difficulty-scaled length budgets via preference optimisation.", "relevance_score": 3, "relevance_rationale": "Adaptive reasoning length, industry track.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "shen-2025-halo", "title": "Batch Query Processing and Optimization for Agentic Workflows", "authors": [ "Junyi Shen", "Noppanat Wadlom", "Yao Lu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.02121", "code_url": null, "topic_clusters": [ "C4", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "query-plan optimization" ], "cost_reduction_reported": "up to 3.6x speedup for batch inference and 2.6x throughput under online serving without compromising output quality", "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Halo treats batches of agentic workflow queries like database queries, applying query optimization (plan reuse, batching, placement) across heterogeneous LLM serving.", "key_idea": "Database-style query optimization for batches of agentic workflows.", "relevance_score": 3, "relevance_rationale": "Workflow-level serving optimization.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "shen-2026-dynbranch", "wadlom-2026-helium" ] }, { "id": "shi-2025-inferencedynamics", "title": "INFERENCEDYNAMICS: Efficient Routing Across LLMs through Structured Capability and Knowledge Profiling", "authors": [ "Hao-Chen Shi", "Tianshi ZHENG", "Weiqi Wang", "Baixuan Xu", "Chunyang Li", "Chunkit Chan", "Tao Fan", "Yangqiu Song", "Qiang Yang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.16303", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "structured capability and knowledge profiles" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU-Pro", "GPQA", "BigGenBench", "LiveBench", "RouteMix" ], "models_studied": [], "production_evidence": false, "summary": "InferenceDynamics models each LLM's capabilities and knowledge along multiple dimensions to route queries across large pools of specialized models, evaluated on the RouteMix dataset with group-level routing.", "key_idea": "Multi-dimensional capability/knowledge profiling for scalable routing.", "relevance_score": 3, "relevance_rationale": "Scales to large model pools.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "song-2025-agent-data-protocol", "title": "Agent Data Protocol: Unifying Datasets for Diverse, Effective Fine-tuning of LLM Agents", "authors": [ "Yueqi Song", "Ketan Ramaneti", "Zaid Sheikh", "Ziru Chen", "Boyu Gou", "Tianbao Xie", "Yiheng Xu", "Danyang Zhang", "Apurva Gandhi", "Fan Yang", "Joseph Liu", "Tianyue Ou", "Zhihao Yuan", "Frank Xu", "Shuyan Zhou", "Xingyao Wang", "Xiang Yue", "Tao Yu", "Huan Sun", "Yu Su", "Graham Neubig" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.24702", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "unified agent datasets" ], "cost_reduction_reported": "~20% average gain over base models after SFT on unified data", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Agent Data Protocol standardizes 13 agent datasets into one representation convertible to multiple frameworks, enabling large-scale SFT.", "key_idea": "Interlingua for agent training data.", "relevance_score": 3, "relevance_rationale": "Useful for converting production traces to training data.", "found_via": "hf-papers search: SFT agent traces cost", "cites_or_cited_by": [] }, { "id": "sui-2025-stop-overthinking", "title": "Stop Overthinking: A Survey on Efficient Reasoning for Large Language Models", "authors": [ "Yang Sui", "Yu-Neng Chuang", "Guanchu Wang", "Jiamu Zhang", "Tianyi Zhang", "Jiayi Yuan", "Hongyi Liu", "Andrew Wen", "Shaochen Zhong", "Na Zou", "Hanjie Chen", "Xia Hu" ], "year": 2025, "venue_or_source": "TMLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2503.16419", "code_url": "https://github.com/Eclipsess/Awesome-Efficient-Reasoning-LLMs", "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "First structured survey of efficient reasoning, grouping methods into model-based, reasoning-output-based and input-prompt-based approaches to curb overthinking in reasoning models.", "key_idea": "Efficient reasoning survey.", "relevance_score": 3, "relevance_rationale": "Seed survey on efficient reasoning.", "found_via": "seed | seed: Stop Overthinking survey", "cites_or_cited_by": [ "aggarwal-2023-adaptive-consistency", "aggarwal-2025-l1-lcpo", "arora-2025-reason-efficiently", "chuang-2024-self-ref", "chuang-2025-confident-or-seek-stronger", "dai-2025-s-grpo", "ding-2025-best-route", "fan-2025-cothink", "fatemi-2025-concise-reasoning-rl", "feng-2025-efficient-reasoning-models-survey", "fu-2024-dynasor-certaindex", "gao-2025-reasoning-efficiency-gap", "hou-2025-thinkprune", "jiang-2025-flashthink", "jiang-2025-lhrm", "li-2024-early-stopping-self-consistency", "li-2025-budget-guidance", "li-2025-selfbudgeter", "liang-2025-thinkswitcher", "lin-2025-plan-and-budget", "lin-2025-sleep-time-compute", "liu-2025-answer-convergence-early-stop", "lou-2025-adacot", "lu-2025-car-certainty-routing", "luo-2025-ada-r1", "luo-2025-o1-pruner", "ma-2025-cot-valve", "ma-2025-nothinking", "munkhbat-2025-self-training-concise", "ong-2024-routellm", "pu-2025-thoughtterminator", "qi-2025-anytime-reasoner-brpo", "qu-2025-efficient-reasoning-lrm-survey", "qu-2025-mrt", "renze-2024-concise-cot", "she-2025-hawkeye", "shen-2025-dast", "su-2025-underthinking-overthinking-length", "team-2025-kimi-k15", "wang-2024-difficulty-adaptive-sc", "wang-2025-entropy-after-think", "wang-2025-reasoning-economy", "wu-2025-when-more-is-less-cot", "xia-2025-tokenskip", "xu-2025-chain-of-draft", "xu-2025-elastic-reasoning", "yang-2025-deer-dynamic-early-exit", "zhang-2025-alphaone", "zhang-2025-othink-r1" ] }, { "id": "tan-2025-zero-step-thinking", "title": "The Zero-Step Thinking: An Empirical Study of Mode Selection as Harder Early Exit in Reasoning Models", "authors": [ "Yuqiao Tan", "Shizhu He", "Kang Liu", "Jun Zhao" ], "year": 2025, "venue_or_source": "NeurIPS 2025 Efficient Reasoning Workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2510.19176", "code_url": "https://github.com/Trae1ounG/Zero_Step_Thinking", "topic_clusters": [ "C9", "C13" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "prompt features", "internal states" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Frames thinking-mode selection as a harder early-exit problem decided before any reasoning. Across nine baselines, prompt-based selectors fail and internal-state methods do better but are unstable.", "key_idea": "Mode selection is early exit at step zero, and current methods are weak.", "relevance_score": 3, "relevance_rationale": "Empirical limits of pre-generation reasoning routers.", "found_via": "web search: Do NOT Think That Much (related results)", "cites_or_cited_by": [] }, { "id": "tripathy-2025-swenergy", "title": "SWEnergy: An Empirical Study on Energy Efficiency in Agentic Issue Resolution Frameworks with SLMs", "authors": [ "Arihant Tripathy", "Ch Pavan Harshit", "Karthik Vaidhyanathan" ], "year": 2025, "venue_or_source": "ICSE 2026 Workshop (AGENT)", "source_type": "paper", "url": "https://arxiv.org/abs/2512.09543", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "energy", "tokens", "duration" ], "cost_reduction_reported": "Most energy-intensive framework consumed 9.4x more energy than the least; resolution rates near zero with SLMs", "benchmarks": [ "SWE-bench Verified Mini" ], "models_studied": [ "Gemma-3 4B", "Qwen-3 1.7B" ], "production_evidence": false, "summary": "Evaluates SWE-Agent, OpenHands, Mini SWE Agent and AutoCodeRover with 1.7-4B SLMs and finds near-zero resolution while energy use varies 9.4x by framework, largely wasted in unproductive loops. Frameworks built for LLMs do not transfer efficiently to SLMs.", "key_idea": "Dropping SLMs into LLM-designed agent frameworks wastes energy.", "relevance_score": 3, "relevance_rationale": "Caution on naive downscaling.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "trirat-2025-agentic-predictor", "title": "Multi-View Encoders for Performance Prediction in LLM-Based Agentic Workflows", "authors": [ "Patara Trirat", "Wonyong Jeong", "Sung Ju Hwang" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2505.19764", "code_url": null, "topic_clusters": [ "C4", "C6" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "code, prompt and graph embeddings" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Agentic Predictor is a lightweight predictor of agentic-workflow success from multi-view encodings (graph, code, prompts) with cross-domain pretraining, used to select workflow configurations without costly execution.", "key_idea": "Surrogate performance predictor for workflow configs.", "relevance_score": 3, "relevance_rationale": "Reduces cost of config search by predicting performance.", "found_via": "search: agentic workflow performance predictor surrogate cheap evaluation arXiv", "cites_or_cited_by": [] }, { "id": "tu-2025-autothink", "title": "Learning When to Think: Shaping Adaptive Reasoning in R1-Style Models via Multi-Stage RL", "authors": [ "Songjun Tu", "Jiahao Lin", "Qichao Zhang", "Xiangyu Tian", "Linjing Li", "Xiangyuan Lan", "Dongbin Zhao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.10832", "code_url": "https://github.com/ScienceOne-AI/AutoThink", "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "ellipsis prompt trigger", "stage-wise reward" ], "cost_reduction_reported": "+6.4% relative accuracy with 52% fewer tokens on DeepSeek-R1-Distill-Qwen-1.5B", "benchmarks": [ "five math benchmarks" ], "models_studied": [ "DeepSeek-R1-Distill-Qwen-1.5B" ], "production_evidence": false, "summary": "Finds an ellipsis in the prompt stochastically toggles thinking in R1-style models, then uses multi-stage RL with reward shaping to invoke explicit reasoning only when needed.", "key_idea": "Exploit latent think/no-think controllability with staged RL.", "relevance_score": 3, "relevance_rationale": "Adaptive thinking for R1-style models.", "found_via": "web search: AdaptThink related", "cites_or_cited_by": [] }, { "id": "vllm-2025-halugate", "title": "Token-Level Truth: Real-Time Hallucination Detection for Production LLMs (HaluGate)", "authors": [ "vLLM Semantic Router Team" ], "year": 2025, "venue_or_source": "vLLM blog", "source_type": "blog", "url": "https://blog.vllm.ai/blog/2025-12-14-halugate", "code_url": "https://github.com/vllm-project/semantic-router", "topic_clusters": [ "C7", "C11" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "fact-check need classifier", "token-level hallucination detector", "NLI" ], "cost_reduction_reported": "Pre-classification skips detection for non-factual queries, a 72.2% efficiency gain where ~35% of queries are non-factual; detection ~125ms for typical RAG context, ~365ms at 16K tokens (project-reported)", "benchmarks": [ "project evaluation" ], "models_studied": [ "ModernBERT-based classifiers" ], "production_evidence": false, "summary": "HaluGate is a conditional pipeline in the vLLM Semantic Router: a small classifier (96.4% validation accuracy, ~12ms) decides whether a response needs fact-checking, then a token-level detector plus an NLI explainer flag unsupported spans against tool/RAG context, with results surfaced in response headers. Token-level detection alone reached only 59% F1, which motivated the ensemble.", "key_idea": "Cheap gated, non-LLM verifiers can provide per-response quality signals at the gateway without an LLM judge.", "relevance_score": 3, "relevance_rationale": "Low-cost quality signal usable as an escalation trigger in routing.", "found_via": "direct: blog.vllm.ai index", "cites_or_cited_by": [] }, { "id": "vllm-2025-semantic-router", "title": "vLLM Semantic Router: Next Phase in LLM inference", "authors": [ "vLLM Semantic Router Team" ], "year": 2025, "venue_or_source": "vLLM blog", "source_type": "blog", "url": "https://blog.vllm.ai/2025/09/11/semantic-router.html", "code_url": "https://github.com/vllm-project/semantic-router", "topic_clusters": [ "C1", "C9", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "ModernBERT intent classification" ], "cost_reduction_reported": "(vendor-reported) ~10% higher accuracy, ~50% lower latency, ~50% fewer tokens in trials (no benchmark or baseline disclosed)", "benchmarks": [], "models_studied": [ "open models served on vLLM" ], "production_evidence": false, "summary": "Open-source router (Red Hat, IBM Research, Tencent contributors) that classifies intent with a ModernBERT classifier and chooses a fast path or chain-of-thought reasoning path and a model, deployed on Kubernetes. Claims lack disclosed methodology; an independent evaluation found no task-specific advantage over blind allocation.", "key_idea": "Classifier-based reasoning-mode and model selection in the serving layer.", "relevance_score": 3, "relevance_rationale": "Widely adopted OSS router; independent evaluation is skeptical.", "found_via": "seed list: model router product", "cites_or_cited_by": [ "kumar-2026-task-session-routing-eval" ] }, { "id": "wang-2025-augserve", "title": "AugServe: Adaptive Request Scheduling for Augmented Large Language Model Inference Serving", "authors": [ "Ying Wang", "Zhen Jin", "Jiexiong Xu", "Wenhai Lin", "Yiquan Chen", "Wenzhi Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.04013", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "request characteristics", "runtime load" ], "cost_reduction_reported": "4.7x and 3.3x higher effective throughput than vLLM and InferCept, and TTFT reduced up to 96.3% and 95.0% respectively", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Two-stage adaptive scheduling for augmented (tool-calling) LLM requests, refining initial decisions with runtime information and adapting token batching to load.", "key_idea": "Adaptive two-stage scheduling for tool-augmented requests.", "relevance_score": 3, "relevance_rationale": "Tool-aware serving.", "found_via": "InferCept intercept augmented LLM inference tool calls", "cites_or_cited_by": [ "abhyankar-2024-infercept", "shahout-2024-lamps" ] }, { "id": "wang-2025-cosmos", "title": "COSMOS: Predictable and Cost-Effective Adaptation of LLMs", "authors": [ "Jiayu Wang", "Aws Albarghouthi", "Frederic Sala" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.01449", "code_url": null, "topic_clusters": [ "C4", "C6" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "proxy models", "low-sample scaling laws" ], "cost_reduction_reported": "Reduces computational costs of strategy selection by 92.72% on average with high prediction accuracy", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "COSMOS predicts performance and cost of model-plus-adaptation strategies (fine-tuning, retrieval-augmented ICL) without full trials using proxy models and scaling laws.", "key_idea": "Predict cost and performance of LLM adaptation choices cheaply.", "relevance_score": 3, "relevance_rationale": "Cheap prediction for choosing model/strategy configurations.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "wang-2025-demystifying-hybrid-thinking", "title": "Demystifying Hybrid Thinking: Can LLMs Truly Switch Between Think and No-Think?", "authors": [ "Shouren Wang", "Wang Yang", "Xianxuan Long", "Qifan Wang", "Vipin Chaudhary", "Xiaotian Han" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.12680", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "mode control" ], "cost_reduction_reported": "No-think output length on MATH500 reduced from 1085 to 585 tokens and \"wait\" occurrences from 5917 to 522 while maintaining accuracy", "benchmarks": [ "MATH500" ], "models_studied": [], "production_evidence": false, "summary": "Finds hybrid-thinking models leak reasoning into no-think mode and identifies training factors that improve mode separation, proposing a practical training recipe.", "key_idea": "No-think mode is leaky; training recipe fixes it.", "relevance_score": 3, "relevance_rationale": "Relevant to relying on no-think mode for savings.", "found_via": "arXiv title search: hybrid thinking", "cites_or_cited_by": [] }, { "id": "wang-2025-drift-dissatisfaction", "title": "DRIFT: Learning from Abundant User Dissatisfaction in Real-World Preference Learning", "authors": [ "Yifan Wang", "Bolian Li", "Junlin Wu", "Zhaoxuan Tan", "Zheli Liu", "Ruqi Zhang", "Ananth Grama", "Qingkai Zeng" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.02341", "code_url": null, "topic_clusters": [ "C6", "C8" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "implicit dissatisfaction signals (refinements, corrections)" ], "cost_reduction_reported": null, "benchmarks": [ "WildBench", "AlpacaEval 2", "WildFeedback", "UltraFeedback" ], "models_studied": [ "7B", "14B" ], "production_evidence": false, "summary": "Observes that deployments produce abundant implicit dissatisfaction signals (users refining or correcting) but few explicit positives, and proposes DRIFT, which anchors preference training on dissatisfaction examples and samples positives from the evolving policy. Reports up to +7.61% WildBench task score and +12.29% AlpacaEval2 win rate at 14B over base models.", "key_idea": "Learn from plentiful dissatisfaction signals in real logs instead of scarce positive labels.", "relevance_score": 3, "relevance_rationale": "Shows how regenerations and corrections in production logs become a usable learning signal.", "found_via": "query: \"regenerate\" user dissatisfaction signal", "cites_or_cited_by": [] }, { "id": "wang-2025-efficient-moa-serving", "title": "Efficient Mixture-of-Agents Serving via Tree-Structured Routing, Adaptive Pruning, and Dependency-Aware Prefill-Decode Overlap", "authors": [ "Zijun Wang", "Yijiahao Qi", "Hanqiu Chen", "Zishen Wan", "Gongjin Sun", "Dongyang Li", "Shuyi Pei", "Cong Hao" ], "year": 2025, "venue_or_source": "arXiv (submitted to DAC 2026)", "source_type": "paper", "url": "https://arxiv.org/abs/2512.18126", "code_url": null, "topic_clusters": [ "C3", "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "semantic agreement", "confidence" ], "cost_reduction_reported": "Up to 90% lower end-to-end latency with accuracy within +/-1% of dense MoA", "benchmarks": [ "representative tasks" ], "models_studied": [], "production_evidence": false, "summary": "Co-designs MoA serving with a tree topology for sparse communication, runtime skipping of downstream agents based on agreement and confidence, and overlapping prefill with decode across dependent agents. Latency drops sharply at near-equal accuracy.", "key_idea": "Sparse tree MoA with confidence-based agent skipping and pipelined serving.", "relevance_score": 3, "relevance_rationale": "Serving-level MoA efficiency.", "found_via": "web search: MoA cost", "cites_or_cited_by": [ "zhang-2024-agentprune" ] }, { "id": "wang-2025-entropy-after-think", "title": "Entropy After for reasoning model early exiting", "authors": [ "Xi Wang", "James McInerney", "Lequn Wang", "Nathan Kallus" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.26522", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "entropy of token after appended " ], "cost_reduction_reported": "Reduces token usage by 12-22% without harming accuracy on MATH500 and AIME2025; works black-box via proxy models (Llama 70B with 1.5B proxy, Claude 3.7 with 4B proxy)", "benchmarks": [ "MATH500", "AIME2025" ], "models_studied": [ "Llama 70B", "Claude 3.7" ], "production_evidence": false, "summary": "Appends a stop-thinking token and tracks the entropy of the next token; when its moving variance stabilises, reasoning is stopped. Can run with a small proxy for black-box models.", "key_idea": "Cheap entropy signal for stopping reasoning, even for API models.", "relevance_score": 3, "relevance_rationale": "Black-box-compatible early exit.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "wang-2025-evoagentx", "title": "EvoAgentX: An Automated Framework for Evolving Agentic Workflows", "authors": [ "Yingxu Wang", "Siwei Liu", "Jinyuan Fang", "Zaiqiao Meng" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.03616", "code_url": "https://github.com/EvoAgentX/EvoAgentX", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task metric" ], "cost_reduction_reported": null, "benchmarks": [ "HotPotQA", "MBPP", "MATH", "GAIA" ], "models_studied": [], "production_evidence": false, "summary": "EvoAgentX is an open-source platform for generating, executing and evolving multi-agent workflows, integrating TextGrad, AFlow and MIPRO to refine prompts, tool configs and topologies. Reports gains such as +7.44% HotPotQA F1 and up to +20% on GAIA.", "key_idea": "Unified open-source platform for evolutionary optimization of agent workflows.", "relevance_score": 3, "relevance_rationale": "Practical toolkit combining several C4 optimizers.", "found_via": "search: FlowReasoner EvoAgent arXiv multi-agent workflow", "cites_or_cited_by": [ "opsahlong-2024-mipro", "yuksekgonul-2024-textgrad", "zhang-2024-aflow" ] }, { "id": "wang-2025-kvcache-in-the-wild", "title": "KVCache Cache in the Wild: Characterizing and Optimizing KVCache Cache at a Large Cloud Provider", "authors": [ "Jiahao Wang", "Jinbo Han", "Xingda Wei", "Sijie Shen", "Dingyan Zhang", "Chenguang Fang", "Rong Chen", "Wenyuan Yu", "Haibo Chen" ], "year": 2025, "venue_or_source": "USENIX ATC 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2506.02634", "code_url": null, "topic_clusters": [ "C10", "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "per-category reuse probability" ], "cost_reduction_reported": "ideal KV hit rates of 62% and 54% on two production traces; workload-aware eviction improves hit ratio by up to 23.9% and reduces latency by up to 42% vs LRU/FIFO", "benchmarks": [ "Alibaba Cloud (ALIYUN) to-C and to-B production traces" ], "models_studied": [], "production_evidence": true, "summary": "First characterization of KV cache reuse in production traces from a large cloud LLM provider: reuse is skewed, single-turn API requests with shared system prompts account for most hits in business workloads, and required cache capacity is moderate. Proposes a workload-aware eviction policy.", "key_idea": "Real traces show where KV reuse comes from and how to evict for it.", "relevance_score": 3, "relevance_rationale": "Production evidence on prefix/KV reuse patterns relevant to agent API traffic.", "found_via": "KVCache Cache in the Wild large cloud provider characterizing", "cites_or_cited_by": [] }, { "id": "wang-2025-mas2", "title": "MAS$^2$: Self-Generative, Self-Configuring, Self-Rectifying Multi-Agent Systems", "authors": [ "Kun Wang", "Guibin Zhang", "ManKit Ye", "Xinyu Deng", "Dongxia Wang", "Xiaobin Hu", "Jinyang Guo", "Yang Liu", "Yufei Guo" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.24323", "code_url": "https://github.com/yeyeyeah2/MAS2", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "task demands", "task reward" ], "cost_reduction_reported": "Up to 19.6% gains over SOTA MAS while staying on the cost-performance Pareto frontier", "benchmarks": [ "deep research", "code generation", "seven benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "MAS^2 uses a generator-implementer-rectifier meta-agent team, trained with Collaborative Tree Optimization, to compose and adaptively fix a target multi-agent system per task. It improves on prior automatic MAS and generalizes to unseen backbones without excessive token cost.", "key_idea": "A MAS that builds and rectifies other MAS at run time.", "relevance_score": 3, "relevance_rationale": "Automatic MAS construction on the cost-performance frontier.", "found_via": "web search: MAS generation | search: MaAS Multi-agent Architecture Search via Agentic Supernet arXiv", "cites_or_cited_by": [ "yue-2025-masrouter", "zhang-2024-agentprune", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "wang-2025-practitioners-guide-agentic-rl", "title": "A Practitioner's Guide to Multi-turn Agentic Reinforcement Learning", "authors": [ "Ruiyi Wang", "Prithviraj Ammanabrolu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.01132", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Systematic study of what works in multi-turn agentic RL across environment, reward and policy design choices.", "key_idea": "Empirical recipe for agentic RL.", "relevance_score": 3, "relevance_rationale": "Practical guidance for RL on small agents.", "found_via": "hf-papers search: multi-turn reinforcement learning agents", "cites_or_cited_by": [] }, { "id": "wang-2025-reasoning-economy", "title": "Harnessing the Reasoning Economy: A Survey of Efficient Reasoning for Large Language Models", "authors": [ "Rui Wang", "Hongru Wang", "Boyang Xue", "Yixia Li", "Jianhui Pang", "Shudong Liu", "Yi Chen", "Jiahao Qiu", "Derek Fai Wong", "Guanhua Chen", "Heng Ji", "Kam-Fai Wong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.24377", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of 'reasoning economy': balancing System 1 and System 2 costs across post-training and test-time, covering causes of inefficiency and solutions.", "key_idea": "Reasoning economy survey.", "relevance_score": 3, "relevance_rationale": "Budget control background.", "found_via": "snowball: cited-by ong-2024-routellm | GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm", "sui-2025-stop-overthinking" ] }, { "id": "wang-2025-slm-llm-collab-survey", "title": "A Survey on Collaborating Small and Large Language Models for Performance, Cost-effectiveness, Cloud-edge Privacy, and Trustworthiness", "authors": [ "Fali Wang", "Jihai Chen", "Shuhua Yang", "Ali Al-Lawati", "Linli Tang", "Hui Liu", "Suhang Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.13890", "code_url": null, "topic_clusters": [ "C1", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Systematic survey of SLM-LLM collaboration organized by goals: performance, cost-effectiveness, cloud-edge privacy and trustworthiness.", "key_idea": "Goal-based SLM-LLM survey.", "relevance_score": 3, "relevance_rationale": "Cost-effectiveness section relevant.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "wang-2025-slm-mux", "title": "Slm-mux: Orchestrating small language models for reasoning", "authors": [ "Chenyu Wang", "Zishen Wan", "Hao Kang", "Emma Chen", "Zhiqiang Xie", "Tushar Krishna", "Vijay Janapa Reddi", "Yilun Du" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.05077", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "model complementarity", "confidence/consistency" ], "cost_reduction_reported": "With two SLMs, outperforms Qwen 2.5 72B on GPQA and GSM8K and matches it on MATH; up to +13.4% on MATH vs prior orchestration", "benchmarks": [ "MATH", "GPQA", "GSM8K", "HumanEval" ], "models_studied": [ "Qwen 2.5 72B (baseline)", "SLMs" ], "production_evidence": false, "summary": "SLM-MUX orchestrates multiple small models, with a search for the most complementary subset and SLM-specific test-time scaling, showing that orchestration methods designed for frontier models underperform with SLMs. Two SLMs can beat a 72B model on some reasoning benchmarks.", "key_idea": "Orchestrate complementary SLMs instead of calling a large model.", "relevance_score": 3, "relevance_rationale": "SLM ensembles as a substitute for large-model calls.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "warren-2025-bidirectional-cascading", "title": "Bi-directional Model Cascading with Proxy Confidence", "authors": [ "David Warren", "Mark Dras" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.19391", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state confidence", "proxy for large-model confidence" ], "cost_reduction_reported": null, "benchmarks": [ "multiple-choice datasets" ], "models_studied": [], "production_evidence": false, "summary": "Uses both small-model confidence (from hidden-state analysis) and a tiny proxy estimate of the large model's confidence to make deferral decisions, reducing deferrals.", "key_idea": "Consider large-model confidence via a proxy before deferring.", "relevance_score": 3, "relevance_rationale": "Improved deferral signal.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "wei-2025-webagent-r1", "title": "WebAgent-R1: Training Web Agents via End-to-End Multi-Turn Reinforcement Learning", "authors": [ "Zhepei Wei", "Wenlin Yao", "Yao Liu", "Weizhi Zhang", "Qin Lu", "Liang Qiu", "Changlong Yu", "Puyang Xu", "Chao Zhang", "Bing Yin", "Hyokun Yun", "Lihong Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.16421", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "binary outcome reward" ], "cost_reduction_reported": "Qwen-2.5-3B success rises 6.1% to 33.9%, Llama-3.1-8B 8.5% to 44.8% on WebArena-Lite, outperforming proprietary models such as OpenAI o3 (per paper)", "benchmarks": [ "WebArena-Lite" ], "models_studied": [ "Qwen-2.5-3B", "Llama-3.1-8B" ], "production_evidence": false, "summary": "End-to-end multi-turn RL framework for web agents that lifts small open models above strong proprietary models on WebArena-Lite.", "key_idea": "Simple end-to-end RL for small web agents.", "relevance_score": 3, "relevance_rationale": "Small agents beating frontier via RL.", "found_via": "hf-papers search: multi-turn reinforcement learning agents", "cites_or_cited_by": [] }, { "id": "wen-2025-budgetthinker", "title": "BudgetThinker: Empowering Budget-aware LLM Reasoning with Control Tokens", "authors": [ "Hao Wen", "Xinrui Wu", "Yi Sun", "Feifei Zhang", "Liye Chen", "Jie Wang", "Yunxin Liu", "Yunhao Liu", "Ya-Qin Zhang", "Yuanchun Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.17196", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "hybrid", "signals_used": [ "periodic remaining-budget control tokens" ], "cost_reduction_reported": null, "benchmarks": [ "mathematical benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Inserts control tokens during inference that tell the model its remaining budget, with SFT and curriculum RL for budget adherence and accuracy.", "key_idea": "In-stream budget reminders via control tokens.", "relevance_score": 3, "relevance_rationale": "Precise thinking-length control.", "found_via": "arXiv title search: token budget reasoning", "cites_or_cited_by": [] }, { "id": "wood-2025-martian-up-and-to-the-left", "title": "Up and to the left! How Martian Uses Routing to Push the Pareto Frontier", "authors": [ "Adam Michael Wood" ], "year": 2025, "venue_or_source": "Martian blog", "source_type": "blog", "url": "https://withmartian.com/post/up-and-to-the-left", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted (request, model) score", "user feedback" ], "cost_reduction_reported": "Customer help chat: 52.4% lower error rate and 92% lower cost; RAG system on GPT-4: +20% quality at 1/80 the cost; user-facing system: router responses preferred 79.2% of the time at 1/300 of GPT-4 cost, 13->113 tokens/s (vendor-reported customer deployments, no methodology)", "benchmarks": [], "models_studied": [ "GPT-4" ], "production_evidence": true, "summary": "An explainer on routing as Pareto-frontier expansion that lists approaches Martian explores (learned (request, model)->score predictors, chain-of-thought inspection, self-assessment, response-similarity prediction) and their data problems. It reports aggregate results from customer-specific routers trained on customer data.", "key_idea": "Customer-specific routers trained on customer data can beat the single-model cost-quality frontier.", "relevance_score": 3, "relevance_rationale": "Production claims for customer-data-trained routing, but without methodology.", "found_via": "direct: withmartian.com/blog index", "cites_or_cited_by": [] }, { "id": "wu-2025-bayesianrouter-reward-models", "title": "Reward Model Routing in Alignment", "authors": [ "Xinle Wu", "Yao Lu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.02850", "code_url": null, "topic_clusters": [ "C5", "C6" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "preference data", "Thompson sampling posterior" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Selects one reward model per query from a pool during RLHF, combining an offline multi-task router trained on preference data with an online Bayesian Thompson-sampling selector that addresses cold start and exploration.", "key_idea": "Offline-warm-started, online Thompson-sampling routing among reward models.", "relevance_score": 3, "relevance_rationale": "Offline-to-online router template; applied to reward models rather than generators.", "found_via": "query: abs:\"preference data\" AND abs:\"routing\" AND abs:\"LLM\"", "cites_or_cited_by": [] }, { "id": "wu-2025-resum", "title": "ReSum: Unlocking Long-Horizon Search Intelligence via Context Summarization", "authors": [ "Xixi Wu", "Kuan Li", "Yida Zhao", "Liwen Zhang", "Litu Ou", "Huifeng Yin", "Zhongwang Zhang", "Xinmiao Yu", "Dingchu Zhang", "Yong Jiang", "Pengjun Xie", "Fei Huang", "Minhao Cheng", "Shuai Wang", "Hong Cheng", "Jingren Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.13313", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "periodic context summarization" ], "cost_reduction_reported": null, "benchmarks": [ "web search agent benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "ReSum periodically summarizes a search agent's growing interaction history into a compact state and continues from it; ReSum-GRPO trains agents to reason over such summaries, giving 4.5% (training-free) and a further 8.2% improvement over ReAct.", "key_idea": "Periodic summarization plus RL adaptation for unbounded search horizons.", "relevance_score": 3, "relevance_rationale": "Summarization memory for long-horizon agents.", "found_via": "snowball: related work of sun-2025-context-folding", "cites_or_cited_by": [ "sun-2025-context-folding" ] }, { "id": "wu-2025-rirs-iterative-routing-specialists", "title": "Talk to Right Specialists: Iterative Routing in Multi-agent Systems for Question Answering", "authors": [ "Feijie Wu", "Zitao Li", "Fei Wei", "Yaliang Li", "Bolin Ding", "Jing Gao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.07813", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "corpus embedding summaries" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RIRS routes questions only to the most relevant RAG agents using embedding summaries of their private corpora, and iteratively aggregates and refines for complex questions, avoiding broadcast-to-all costs.", "key_idea": "Route questions to relevant specialist agents instead of broadcasting.", "relevance_score": 3, "relevance_rationale": "Routing among agents to save calls.", "found_via": "snowball: cited-by yue-2025-masrouter", "cites_or_cited_by": [ "yue-2025-masrouter" ] }, { "id": "wu-2025-when-more-is-less-cot", "title": "When More is Less: Understanding Chain-of-Thought Length in LLMs", "authors": [ "Yuyang Wu", "Yifei Wang", "Ziyu Ye", "Tianqi Du", "Stefanie Jegelka", "Yisen Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.07266", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows accuracy follows an inverted U with CoT length, with the optimum rising with task difficulty and falling with model capability, and gives a theoretical model and practical length-aware training and filtering.", "key_idea": "Optimal CoT length scales with difficulty and inversely with capability.", "relevance_score": 3, "relevance_rationale": "Theory for choosing reasoning budgets.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "xi-2025-agentgym-rl", "title": "AgentGym-RL: Training LLM Agents for Long-Horizon Decision Making through Multi-Turn Reinforcement Learning", "authors": [ "Zhiheng Xi", "Jixuan Huang", "Chenyang Liao", "Baodai Huang", "Honglin Guo", "Jiaqi Liu", "Rui Zheng", "Junjie Ye", "Jiazheng Zhang", "Wenxiang Chen", "Wei He", "Yiwen Ding", "Guanyu Li", "Zehui Chen", "Zhengyin Du", "Xuesong Yao", "Yufei Xu", "Jiecao Chen", "Tao Gui", "Zuxuan Wu", "Qi Zhang", "Xuanjing Huang", "Yu-Gang Jiang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.08755", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "environment rewards", "ScalingInter-RL" ], "cost_reduction_reported": "Agents match or surpass commercial models on 27 tasks", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Modular multi-turn RL framework with ScalingInter-RL that gradually extends interaction horizon, training open agents to match commercial models.", "key_idea": "Horizon-scaling RL for long-horizon agents.", "relevance_score": 3, "relevance_rationale": "RL specialization infra.", "found_via": "hf-papers search: multi-turn reinforcement learning agents", "cites_or_cited_by": [] }, { "id": "xia-2025-agentrm", "title": "AgentRM: Enhancing Agent Generalization with Reward Modeling", "authors": [ "Yu Xia", "Jingru Fan", "Weize Chen", "Siyu Yan", "Xin Cong", "Zhong Zhang", "Yaxi Lu", "Yankai Lin", "Zhiyuan Liu", "Maosong Sun" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.18407", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "reward model", "best-of-N", "beam search" ], "cost_reduction_reported": null, "benchmarks": [ "nine agent tasks (e.g., WebShop, ALFWorld, SciWorld)" ], "models_studied": [ "LLaMA-3-70B" ], "production_evidence": false, "summary": "Finds fine-tuning a reward model to guide the policy is more robust than fine-tuning the policy; AgentRM guides best-of-N and step-level beam search, improving the base policy by 8.8 points on average and 12.6 on a LLaMA-3-70B policy.", "key_idea": "A generalizable agent reward model for test-time search.", "relevance_score": 3, "relevance_rationale": "Agent-level verifier that transfers weak-to-strong.", "found_via": "hf-papers search: process reward model agents tool use", "cites_or_cited_by": [] }, { "id": "xia-2025-hivemind", "title": "HiveMind: Contribution-Guided Online Prompt Optimization of LLM Multi-Agent Systems", "authors": [ "Yihan Xia", "Taotao Wang", "Shengli Zhang", "Zhangyuhua Weng", "Bin Cao", "Soung Chang Liew" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.06432", "code_url": null, "topic_clusters": [ "C3", "C4", "C7" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "DAG-Shapley contribution" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "HiveMind quantifies each agent's contribution in an LLM multi-agent system with an efficient DAG-Shapley approximation, then refines prompts of underperforming agents online.", "key_idea": "Shapley-based per-agent contribution scoring for MAS optimization.", "relevance_score": 3, "relevance_rationale": "Per-agent credit signals usable for deciding where cheaper models are safe.", "found_via": "arXiv query abs:\"credit assignment\" AND abs:\"multi-agent\" AND abs:\"LLM\" AND abs:cost", "cites_or_cited_by": [] }, { "id": "xia-2025-tokenskip", "title": "TokenSkip: Controllable Chain-of-Thought Compression in LLMs", "authors": [ "Heming Xia", "Chak Tou Leong", "Wenjie Wang", "Yongqi Li", "Wenjie Li" ], "year": 2025, "venue_or_source": "EMNLP 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.12067", "code_url": "https://github.com/hemingkx/TokenSkip", "topic_clusters": [ "C9", "C10" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "token importance" ], "cost_reduction_reported": "On Qwen2.5-14B-Instruct, reduces reasoning tokens by 40% (313 to 181) on GSM8K with less than 0.4% performance drop", "benchmarks": [ "GSM8K" ], "models_studied": [ "Qwen2.5-14B-Instruct" ], "production_evidence": false, "summary": "Scores the semantic importance of CoT tokens and fine-tunes models to skip unimportant ones at a controllable compression ratio.", "key_idea": "Learn to skip low-importance CoT tokens.", "relevance_score": 3, "relevance_rationale": "Controllable CoT compression.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "xiang-2025-spo", "title": "Self-Supervised Prompt Optimization", "authors": [ "Jinyu Xiang", "Jiayi Zhang", "Zhaoyang Yu", "Xinbing Liang", "Fengwei Teng", "Jinhao Tu", "Fashen Ren", "Xiangru Tang", "Sirui Hong", "Chenglin Wu", "Yuyu Luo" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.06855", "code_url": "https://github.com/FoundationAgents/SPO", "topic_clusters": [ "C4", "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "pairwise LLM-judge output comparison" ], "cost_reduction_reported": "Comparable or superior results to SOTA prompt optimizers at 1.1%-5.6% of their cost, using about three samples", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Self-Supervised Prompt Optimization selects prompts via pairwise output comparisons by an LLM evaluator, requiring no ground truth, and cuts optimization cost drastically.", "key_idea": "Reference-free prompt optimization via pairwise LLM judging.", "relevance_score": 3, "relevance_rationale": "Label-free, low-cost optimization relevant when production data lacks ground truth.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "xie-2025-gmtrouter", "title": "GMTRouter: Personalized LLM Router over Multi-turn User Interactions", "authors": [ "Yihang Sun", "Encheng Xie", "Tao Feng", "Jiaxuan You" ], "year": 2025, "venue_or_source": "EMNLP 2025 (Findings)", "source_type": "paper", "url": "https://arxiv.org/abs/2511.08590", "code_url": "https://github.com/ulab-uiuc/GMTRouter", "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "user-query-response-turn heterogeneous graph", "few-shot user preferences" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "GMTRouter personalizes routing by modeling multi-turn user-LLM interactions as a heterogeneous graph and learning user preferences from few-shot data.", "key_idea": "Personalized graph router over multi-turn interactions.", "relevance_score": 3, "relevance_rationale": "Personalization.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "xie-2025-rmoa", "title": "RMoA: Optimizing Mixture-of-Agents through Diversity Maximization and Residual Compensation", "authors": [ "Zhentao Xie", "Chengcheng Han", "Jinxin Shi", "Wenjun Cui", "Xin Zhao", "Xingjiao Wu", "Jiabao Zhao" ], "year": 2025, "venue_or_source": "ACL 2025 Findings", "source_type": "paper", "url": "https://arxiv.org/abs/2505.24442", "code_url": "https://github.com/mindhunter01/RMoA", "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "embedding diversity", "residual convergence" ], "cost_reduction_reported": "Significantly reduces computational overhead vs MoA", "benchmarks": [ "alignment, math reasoning, code generation, multitask understanding" ], "models_studied": [], "production_evidence": false, "summary": "Residual MoA selects diverse responses via embedding similarity, adds residual extraction/aggregation agents to preserve cross-layer information, and stops adaptively when residuals converge. It gets state-of-the-art MoA results with less compute.", "key_idea": "Diversity-based response selection and adaptive termination in MoA.", "relevance_score": 3, "relevance_rationale": "Cost-aware MoA variant.", "found_via": "web search: MoA cost", "cites_or_cited_by": [] }, { "id": "xu-2025-chain-of-draft", "title": "Chain of Draft: Thinking Faster by Writing Less", "authors": [ "Silei Xu", "Wenhao Xie", "Lingxiao Zhao", "Pengcheng He" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.18600", "code_url": "https://github.com/sileix/chain-of-draft", "topic_clusters": [ "C9", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "prompting for minimal drafts" ], "cost_reduction_reported": "Matches or surpasses CoT accuracy while using as little as 7.6% of the tokens", "benchmarks": [ "various reasoning tasks" ], "models_studied": [], "production_evidence": false, "summary": "Prompts LLMs to write terse, draft-like intermediate steps instead of verbose chain-of-thought.", "key_idea": "Terse drafts instead of verbose CoT.", "relevance_score": 3, "relevance_rationale": "Zero-cost prompt lever for token reduction.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "xu-2025-collaborative-expertise-delegation", "title": "Towards Multi-Agent Reasoning Systems for Collaborative Expertise Delegation: An Exploratory Design Study", "authors": [ "Baixuan Xu", "Chunyang Li", "Weiqi Wang", "Wei Fan", "Tianshi Zheng", "Haochen Shi", "Tao Fan", "Yangqiu Song", "Qiang Yang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.07313", "code_url": null, "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "An exploratory study of multi-agent reasoning design across expertise alignment, collaboration paradigm and scale. It finds domain-dependent benefits of expertise alignment, better results from diversity-driven integration than rigid decomposition, and diminishing returns versus computational cost as scale grows.", "key_idea": "Design-space study of when specialized multi-agent reasoning pays off.", "relevance_score": 3, "relevance_rationale": "Evidence on MAS design trade-offs and cost of scaling.", "found_via": "web search: scaling agents cost", "cites_or_cited_by": [] }, { "id": "xu-2025-elastic-reasoning", "title": "Scalable Chain of Thoughts via Elastic Reasoning", "authors": [ "Yuhui Xu", "Hanze Dong", "Lei Wang", "Doyen Sahoo", "Junnan Li", "Caiming Xiong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.05315", "code_url": "https://github.com/SalesforceAIResearch/Elastic-Reasoning", "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "RL-offline", "signals_used": [ "separate thinking and solution budgets" ], "cost_reduction_reported": null, "benchmarks": [ "AIME", "MATH500", "LiveCodeBench", "Codeforces" ], "models_studied": [], "production_evidence": false, "summary": "Splits generation into thinking and solution phases with separate budgets and trains with budget-constrained rollouts so the model still answers well when thinking is truncated.", "key_idea": "Budgeted thinking that degrades gracefully under truncation.", "relevance_score": 3, "relevance_rationale": "Hard token budgets for deployment.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "yan-2025-dont-build-multi-agents", "title": "Don't Build Multi-Agents", "authors": [ "Walden Yan" ], "year": 2025, "venue_or_source": "Cognition blog", "source_type": "blog", "url": "https://cognition.ai/blog/dont-build-multi-agents", "code_url": null, "topic_clusters": [ "C10", "C12", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Cognition (Devin) argues from production agent-building experience that parallel multi-agent decompositions are fragile because subagents lack shared context and make conflicting implicit decisions. It proposes two principles (share full traces; actions carry implicit decisions), recommends single-threaded linear agents, and for very long tasks a dedicated model that compresses history. Claude Code's read-only subagents and edit-apply models are discussed as examples.", "key_idea": "Context sharing, not agent count, determines reliability; prefer single-threaded agents plus history compression over parallel subagents.", "relevance_score": 3, "relevance_rationale": "Widely cited practitioner position on when multi-agent architectures are not worth it; qualitative, no numbers.", "found_via": "direct: cognition.ai blog", "cites_or_cited_by": [] }, { "id": "yang-2025-deer-dynamic-early-exit", "title": "Dynamic Early Exit in Reasoning Models", "authors": [ "Chenxu Yang", "Qingyi Si", "Yongjie Duan", "Zheliang Zhu", "Chenyu Zhu", "Qiaowei Li", "Minghui Chen", "Zheng Lin", "Weiping Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.15895", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "trial-answer confidence at reasoning transitions" ], "cost_reduction_reported": "Reduces CoT length by 19.1%-80.1% on average while improving accuracy by 0.3%-5.0% across 10 benchmarks and 11 reasoning LLMs", "benchmarks": [ "GSM8K", "MATH-500", "AMC", "GPQA", "AIME", "LiveCodeBench" ], "models_studied": [ "11 reasoning LLMs" ], "production_evidence": false, "summary": "At reasoning transition points, the model produces a trial answer; if its confidence is high, thinking is terminated. Training-free.", "key_idea": "Stop thinking when a trial answer is confident.", "relevance_score": 3, "relevance_rationale": "Training-free early exit.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "yang-2025-mashost", "title": "MasHost Builds It All: Autonomous Multi-Agent System Directed by Reinforcement Learning", "authors": [ "Kuo Yang", "Xingjie Yang", "Linhui Yu", "Qing Xu", "Yan Fang", "Xu Wang", "Zhengyang Zhou", "Yang Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.08507", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "query" ], "cost_reduction_reported": null, "benchmarks": [ "six benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "MasHost builds query-adaptive multi-agent graphs with RL, sampling roles and interactions jointly and optimizing accuracy, efficiency and component rationality via a Hierarchical Relative Policy Optimization method. It beats competitive MAS construction baselines on six benchmarks.", "key_idea": "RL-driven autonomous MAS graph construction with efficiency objective.", "relevance_score": 3, "relevance_rationale": "Query-adaptive MAS design with efficiency term.", "found_via": "web search: MAS generation", "cites_or_cited_by": [ "yue-2025-masrouter", "zhang-2024-g-designer", "zhang-2025-maas-agentic-supernet" ] }, { "id": "yang-2025-qwen3", "title": "Qwen3 Technical Report", "authors": [ "An Yang", "Anfeng Li", "Baosong Yang", "Beichen Zhang", "Binyuan Hui", "Bo Zheng", "Bowen Yu", "Chang Gao", "Chengen Huang", "Chenxu Lv", "Chujie Zheng", "Dayiheng Liu", "Fan Zhou", "Fei Huang", "Feng Hu", "et al." ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.09388", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "user-set thinking budget", "mode switch via chat template" ], "cost_reduction_reported": null, "benchmarks": [ "code, math and agent benchmarks" ], "models_studied": [ "Qwen3 0.6B-235B dense and MoE" ], "production_evidence": false, "summary": "Qwen3 technical report: a model family that unifies thinking and non-thinking modes in one model, with a user-controlled thinking budget, and distils small models from flagship models.", "key_idea": "Single model with switchable thinking and a thinking budget.", "relevance_score": 3, "relevance_rationale": "Widely used open hybrid-thinking model; substrate for mode routing.", "found_via": "arXiv search: hybrid thinking (related)", "cites_or_cited_by": [] }, { "id": "yang-2025-thinking-optimal-scaling", "title": "Towards Thinking-Optimal Scaling of Test-Time Compute for LLM Reasoning", "authors": [ "Wenkai Yang", "Shuming Ma", "Yankai Lin", "Furu Wei" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.18080", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "shortest correct response under varying effort" ], "cost_reduction_reported": null, "benchmarks": [ "math benchmarks" ], "models_studied": [ "Qwen2.5-32B-Instruct", "QwQ-32B-Preview" ], "production_evidence": false, "summary": "Shows longer CoT can hurt performance in some domains and that the optimal length distribution varies by domain. Teaches models several reasoning efforts and self-improves on the shortest correct responses.", "key_idea": "There is an optimal, domain-dependent CoT length.", "relevance_score": 3, "relevance_rationale": "Evidence against always-max thinking.", "found_via": "web search: test-time compute scaling (related)", "cites_or_cited_by": [] }, { "id": "yao-2025-super-agent-hybrid-routers", "title": "Toward Super Agent System with Hybrid AI Routers", "authors": [ "Yuhang Yao", "Haixin Wang", "Yibo Chen", "Jiawen Wang", "Min Chang Jordan Ren", "Bosheng Ding", "Salman Avestimehr", "Chaoyang He" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.10519", "code_url": null, "topic_clusters": [ "C1", "C3", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "intent", "task complexity" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A position paper describing a super-agent system that detects intent, routes requests to specialized task agents or generated workflows, and uses a hybrid router choosing between on-device and cloud models by task complexity.", "key_idea": "Blueprint for intent routing plus local/cloud model routing in a super agent.", "relevance_score": 3, "relevance_rationale": "Position paper; architectural framing only.", "found_via": "arXiv query ti:router AND ti:agent", "cites_or_cited_by": [] }, { "id": "ye-2025-agentfold", "title": "AgentFold: Long-Horizon Web Agents with Proactive Context Management", "authors": [ "Rui Ye", "Zhongwang Zhang", "Kuan Li", "Huifeng Yin", "Zhengwei Tao", "Yida Zhao", "Liangcai Su", "Liwen Zhang", "Zile Qiao", "Xinyu Wang", "Pengjun Xie", "Fei Huang", "Siheng Chen", "Jingren Zhou", "Yong Jiang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.24699", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "multi-scale folding actions" ], "cost_reduction_reported": null, "benchmarks": [ "BrowseComp", "BrowseComp-ZH" ], "models_studied": [ "AgentFold-30B-A3B" ], "production_evidence": false, "summary": "Web agent that proactively manages its trajectory, either condensing recent steps finely or consolidating whole sub-tasks; trained with SFT only, the 30B-A3B agent reaches 36.2% on BrowseComp and 47.3% on BrowseComp-ZH, competitive with much larger models.", "key_idea": "Proactive multi-scale folding of agent history.", "relevance_score": 3, "relevance_rationale": "Context management that lets smaller models compete on long-horizon tasks.", "found_via": "Scaling Long-Horizon LLM Agent via Context-Folding", "cites_or_cited_by": [ "sun-2025-context-folding" ] }, { "id": "zehle-2025-capo", "title": "CAPO: Cost-Aware Prompt Optimization", "authors": [ "Tom Zehle", "Moritz Schlager", "Timo Heiß", "Matthias Feurer" ], "year": 2025, "venue_or_source": "AutoML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2504.16005", "code_url": null, "topic_clusters": [ "C4", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "accuracy", "prompt length", "racing" ], "cost_reduction_reported": "Outperforms SOTA discrete prompt optimizers in 11/15 cases (up to 21%p); reduces average prompt length via length penalty and saves evaluations via racing", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CAPO is an evolutionary prompt optimizer with LLM operators that uses racing to save evaluations and a multi-objective length penalty to shorten prompts, jointly tuning instructions and few-shot examples.", "key_idea": "AutoML-style cost-aware prompt optimization (racing plus length penalty).", "relevance_score": 3, "relevance_rationale": "Cost-aware prompt optimization, both of optimization cost and inference prompt cost.", "found_via": "search: cost-aware prompt optimization LLM token cost Pareto arXiv", "cites_or_cited_by": [ "bussing-2026-mo-capo", "kumar-2026-craft" ] }, { "id": "zeng-2025-pac-efficient-routing", "title": "A note on conditional PAC-efficient reasoning in large language model routing", "authors": [ "Hao Zeng", "Bingyi Jing" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.03057", "code_url": null, "topic_clusters": [ "C1", "C2", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "conditioning sets", "PAC risk control" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows pointwise conditional PAC efficiency for model routing forces a near-impossible router, and proposes a restricted setwise conditional formulation with an explicit router achieving finite-sample validity and near-oracle expert usage.", "key_idea": "Limits of distribution-free conditional guarantees for routing.", "relevance_score": 3, "relevance_rationale": "Theory of safe routing.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "zhang-2025-ace", "title": "Agentic Context Engineering: Evolving Contexts for Self-Improving Language Models", "authors": [ "Qizheng Zhang", "Changran Hu", "Shubhangi Upasani", "Boyuan Ma", "Fenglu Hong", "Vamsidhar Kamanuru", "Jay Rainton", "Chen Wu", "Mengmeng Ji", "Hanchen Li", "Urmish Thakker", "James Zou", "Kunle Olukotun" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2510.04618", "code_url": null, "topic_clusters": [ "C4", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "execution feedback", "reflection and curation" ], "cost_reduction_reported": "+10.6% on agents and +8.6% on finance benchmarks over strong baselines, with lower adaptation latency and rollout cost", "benchmarks": [ "AppWorld", "finance benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "ACE treats contexts (system prompts, agent memory) as evolving playbooks that are incrementally generated, reflected on and curated, avoiding brevity bias and context collapse; on AppWorld it matches a top production agent using a smaller open model.", "key_idea": "Grow and curate a structured playbook instead of rewriting prompts wholesale.", "relevance_score": 3, "relevance_rationale": "Context adaptation that lets smaller models match larger ones; related to config optimization.", "found_via": "Cost-Efficient Serving of LLM Agents via Test-Time Plan Caching", "cites_or_cited_by": [ "gu-2026-peek", "zhang-2025-agentic-plan-caching" ] }, { "id": "zhang-2025-adaptthink", "title": "AdaptThink: Reasoning Models Can Learn When to Think", "authors": [ "Jiajie Zhang", "Nianyi Lin", "Lei Hou", "Ling Feng", "Juanzi Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.13417", "code_url": "https://github.com/THU-KEG/AdaptThink", "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "problem difficulty (implicit)" ], "cost_reduction_reported": "Reduces average response length of DeepSeek-R1-Distill-Qwen-1.5B by 53% and improves accuracy by 2.4% on three math datasets", "benchmarks": [ "three math datasets" ], "models_studied": [ "DeepSeek-R1-Distill-Qwen-1.5B" ], "production_evidence": false, "summary": "Shows NoThinking is better for simple problems and trains the model with constrained RL to choose Thinking or NoThinking per problem, using importance sampling to balance both modes.", "key_idea": "Constrained RL teaches when to skip thinking.", "relevance_score": 3, "relevance_rationale": "Learned thinking-mode selection.", "found_via": "web search: AdaptThink", "cites_or_cited_by": [] }, { "id": "zhang-2025-agentrouter-kg", "title": "AgentRouter: A Knowledge-Graph-Guided LLM Router for Collaborative Multi-Agent Question Answering", "authors": [ "Zheyuan Zhang", "Kaiwen Shi", "Zhengqing Yuan", "Zehong Wang", "Tianyi Ma", "Keerthiram Murugesan", "Vincent Galassi", "Chuxu Zhang", "Yanfang Ye" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.05445", "code_url": null, "topic_clusters": [ "C1", "C3" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "knowledge graph of query/entities/agents", "empirical performance" ], "cost_reduction_reported": null, "benchmarks": [ "QA benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Formulates multi-agent QA as routing over agents with a heterogeneous GNN on a graph linking queries, entities and agents, trained with soft supervision from empirical agent performance, then aggregates weighted agent outputs. Beats single-agent and ensemble baselines.", "key_idea": "GNN-based routing among agents from a query knowledge graph.", "relevance_score": 3, "relevance_rationale": "Routing among agents; not primarily cost-driven.", "found_via": "web search: routing among agents", "cites_or_cited_by": [] }, { "id": "zhang-2025-alphaone", "title": "AlphaOne: Reasoning Models Thinking Slow and Fast at Test Time", "authors": [ "Junyu Zhang", "Runpei Dong", "Han Wang", "Xuying Ning", "Haoran Geng", "Peihao Li", "Xialin He", "Yutong Bai", "Jitendra Malik", "Saurabh Gupta", "Huan Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.24863", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "alpha moment schedule", "reasoning transition tokens" ], "cost_reduction_reported": null, "benchmarks": [ "math, coding and science benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Test-time framework that schedules slow-thinking transitions stochastically up to a scaled \"alpha moment\" and then forces end of thinking, generalising monotonic budget-scaling methods such as budget forcing.", "key_idea": "Slow-then-fast reasoning schedule controlled by one parameter.", "relevance_score": 3, "relevance_rationale": "Decode-time reasoning budget control.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "zhang-2025-asrr", "title": "When to Continue Thinking: Adaptive Thinking Mode Switching for Efficient Reasoning", "authors": [ "Xiaoyun Zhang", "Jingqing Ruan", "Xing Ma", "Yawen Zhu", "Haodong Zhao", "Hao Li", "Jiansong Chen", "Ke Zeng", "Xunliang Cai" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.15400", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "accuracy-aware length reward" ], "cost_reduction_reported": "Reduces reasoning budget by up to 32.5% (1.5B) and 25.7% (7B) vs GRPO with 1.2% and 0.6% pass@1 loss; up to +21.7% harmless rate on safety benchmarks", "benchmarks": [ "multiple reasoning and safety benchmarks" ], "models_studied": [ "1.5B and 7B LRMs" ], "production_evidence": false, "summary": "Identifies that models implicitly supplement reasoning during answer generation in no-thinking mode, and uses accuracy-aware length rewards to suppress unnecessary reasoning while allowing recovery.", "key_idea": "Suppress thinking but allow implicit self-recovery.", "relevance_score": 3, "relevance_rationale": "Adaptive reasoning length via RL.", "found_via": "web search: AdaptThink related", "cites_or_cited_by": [] }, { "id": "zhang-2025-avengers", "title": "The Avengers: A Simple Recipe for Uniting Smaller Language Models to Challenge Proprietary Giants", "authors": [ "Yiqun Zhang", "Hao Li", "Chenxu Wang", "Linyao Chen", "Qiaosheng Zhang", "Peng Ye", "Shi Feng", "Daling Wang", "Zhen Wang", "Xinrun Wang", "Jia Xu", "Lei Bai", "Wanli Ouyang", "Shuyue Hu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.19797", "code_url": null, "topic_clusters": [ "C1", "C12" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query embedding clusters", "per-cluster model scores", "repeated sampling voting" ], "cost_reduction_reported": null, "benchmarks": [ "15 datasets" ], "models_studied": [ "10 ~7B open models", "GPT-4o", "GPT-4.1", "GPT-4.5" ], "production_evidence": false, "summary": "The Avengers embeds and clusters queries, scores each model per cluster, and routes to top models with repeated sampling and voting; ten ~7B open models surpass GPT-4.1 on average across 15 datasets.", "key_idea": "Cluster-based routing plus voting over small models.", "relevance_score": 3, "relevance_rationale": "Shows small-model pools can replace frontier on many tasks.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "zhang-2025-benchmark-prediction-misses", "title": "How Benchmark Prediction from Fewer Data Misses the Mark", "authors": [ "Guanhua Zhang", "Florian E. Dorner", "Moritz Hardt" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.07673", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "subset scores", "AIPW" ], "cost_reduction_reported": null, "benchmarks": [ "19 benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Evaluates 11 benchmark-prediction methods on 19 benchmarks and finds they mostly interpolate among similar models and fail to beat random sampling when extrapolating to stronger new models; proposes an AIPW-based method with modest gains.", "key_idea": "Benchmark prediction fails at the frontier.", "relevance_score": 3, "relevance_rationale": "Important caution for predicting new-model performance offline.", "found_via": "snowball: cited-by pacchiardi-2025-predictaboard", "cites_or_cited_by": [ "polo-2024-tinybenchmarks" ] }, { "id": "zhang-2025-othink-r1", "title": "OThink-R1: Intrinsic Fast/Slow Thinking Mode Switching for Over-Reasoning Mitigation", "authors": [ "Shengjia Zhang", "Junjie Wu", "Jiawei Chen", "Changwang Zhang", "Zhe Li", "Xingyu Lou", "Wangchunshu Zhou", "Sheng Zhou", "Can Wang", "Jun Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.02397", "code_url": "https://github.com/AgenticIR-Lab/OThink-R1", "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "LLM judge labels of redundant reasoning" ], "cost_reduction_reported": null, "benchmarks": [ "math and QA benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "An LLM judge labels reasoning trajectories as essential or redundant; pruned fast-thinking and full slow-thinking samples are used to fine-tune a model that picks its own mode.", "key_idea": "Judge-built hybrid dataset for intrinsic fast/slow switching.", "relevance_score": 3, "relevance_rationale": "Adaptive reasoning via SFT.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "zheng-2025-disrouter", "title": "DiSRouter: Distributed Self-Routing for LLM Selections", "authors": [ "Hang Zheng", "Hongshen Xu", "Yongkai Lin", "Shuai Fan", "Lu Chen", "Kai Yu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.19208", "code_url": null, "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "LLM self-awareness" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "DiSRouter replaces a centralized router with a network of LLM agents that each decide to answer or forward a query based on trained self-awareness of their competence.", "key_idea": "Distributed self-routing via self-awareness training.", "relevance_score": 3, "relevance_rationale": "Alternative to external routers.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "zhou-2025-mem1", "title": "MEM1: Learning to Synergize Memory and Reasoning for Efficient Long-Horizon Agents", "authors": [ "Zijian Zhou", "Ao Qu", "Zhaoxuan Wu", "Sunghwan Kim", "Alok Prakash", "Daniela Rus", "Jinhua Zhao", "Bryan Kian Hsiang Low", "Paul Pu Liang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.15841", "code_url": "https://github.com/MIT-MI/MEM1", "topic_clusters": [ "C8", "C10" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "internal state consolidation" ], "cost_reduction_reported": "MEM1-7B improves performance 3.5x while reducing memory usage 3.7x vs Qwen2.5-14B-Instruct on a 16-objective multi-hop QA task", "benchmarks": [ "16-objective multi-hop QA" ], "models_studied": [ "MEM1-7B", "Qwen2.5-14B-Instruct" ], "production_evidence": false, "summary": "MEM1 trains agents with RL to maintain a compact internal state that merges memory and reasoning at each turn and discards the rest, giving near-constant memory across long multi-turn tasks.", "key_idea": "Learn to keep constant-size memory by consolidating state each turn.", "relevance_score": 3, "relevance_rationale": "Small model with bounded context outperforming a larger one: memory as a cost lever.", "found_via": "MEM1 constant memory long-horizon agents reasoning consolidation", "cites_or_cited_by": [ "kang-2025-acon" ] }, { "id": "zhou-2025-sweet-rl", "title": "SWEET-RL: Training Multi-Turn LLM Agents on Collaborative Reasoning Tasks", "authors": [ "Yifei Zhou", "Song Jiang", "Yuandong Tian", "Jason Weston", "Sergey Levine", "Sainbayar Sukhbaatar", "Xian Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.15478", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "privileged critic" ], "cost_reduction_reported": "Llama-3.1-8B matches or exceeds GPT-4o on collaborative content creation; +6% success on ColBench", "benchmarks": [ "ColBench" ], "models_studied": [ "Llama-3.1-8B", "GPT-4o" ], "production_evidence": false, "summary": "Multi-turn RL with a critic that sees training-time information to give step-level rewards, enabling an 8B model to match GPT-4o in collaborative tasks.", "key_idea": "Asymmetric critic for multi-turn credit assignment.", "relevance_score": 3, "relevance_rationale": "Small agent reaches frontier level via RL.", "found_via": "snowball: related work of archer", "cites_or_cited_by": [ "zhou-2024-archer" ] }, { "id": "zhu-2025-agentdebug", "title": "Where LLM Agents Fail and How They can Learn From Failures", "authors": [ "Kunlun Zhu", "Zijia Liu", "Bingxuan Li", "Muxin Tian", "Yingxuan Yang", "Jiaxun Zhang", "Pengrui Han", "Qipeng Xie", "Fuyang Cui", "Weijia Zhang", "Xiaoteng Ma", "Xiaodong Yu", "Gowtham Ramesh", "Jialian Wu", "Zicheng Liu", "Pan Lu", "James Zou", "Jiaxuan You" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.25370", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "error taxonomy", "root-cause localization" ], "cost_reduction_reported": null, "benchmarks": [ "AgentErrorBench", "ALFWorld", "GAIA", "WebShop" ], "models_studied": [], "production_evidence": false, "summary": "Introduces AgentErrorTaxonomy, AgentErrorBench and AgentDebug, which isolates root-cause errors in agent trajectories and gives targeted feedback; improves step accuracy by 17% and task success by up to 26% relative.", "key_idea": "Root-cause error localization and targeted recovery for agents.", "relevance_score": 3, "relevance_rationale": "Failure detection plus recovery signal.", "found_via": "known-item lookup: where LLM agents fail", "cites_or_cited_by": [] }, { "id": "zhu-2025-agentic-benchmark-checklist", "title": "Establishing Best Practices for Building Rigorous Agentic Benchmarks", "authors": [ "Yuxuan Zhu", "Tengjun Jin", "Yada Pruksachatkun", "Andy Zhang", "Shu Liu", "Sasha Cui", "Sayash Kapoor", "Shayne Longpre", "Kevin Meng", "Rebecca Weiss", "Fazl Barez", "Rahul Gupta", "Jwala Dhamala", "Jacob Merizian", "Mario Giulianelli", "Harry Coppock", "Cozmin Ududec", "Jasjeet Sekhon", "Jacob Steinhardt", "Antony Kellermann", "Sarah Schwettmann", "Matei Zaharia", "Ion Stoica", "Percy Liang", "Daniel Kang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.02825", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Issues can misestimate agent performance by up to 100% relative; ABC reduces overestimation on CVE-Bench by 33%", "benchmarks": [ "SWE-bench Verified", "TAU-bench", "CVE-Bench", "others" ], "models_studied": [ "n/a" ], "production_evidence": false, "summary": "Documents task-setup and reward-design flaws in agentic benchmarks (e.g., insufficient tests in SWE-bench Verified, empty responses counted as success in TAU-bench) and proposes the Agentic Benchmark Checklist.", "key_idea": "Checklist for rigorous agentic benchmarks.", "relevance_score": 3, "relevance_rationale": "Benchmark validity matters when measuring small cost-quality deltas.", "found_via": "snowball: related work of kapoor-2025-hal", "cites_or_cited_by": [ "jimenez-2023-swe-bench", "yao-2024-tau-bench" ] }, { "id": "zhu-2025-multiagentbench", "title": "MultiAgentBench: Evaluating the Collaboration and Competition of LLM agents", "authors": [ "Kunlun Zhu", "Hongyi Du", "Zhaochen Hong", "Xiaocheng Yang", "Shuyi Guo", "Zhe Wang", "Zhenhailong Wang", "Cheng Qian", "Xiangru Tang", "Heng Ji", "Jiaxuan You" ], "year": 2025, "venue_or_source": "ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2503.01935", "code_url": "https://github.com/MultiagentBench/MARBLE", "topic_clusters": [ "C12", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "milestone KPIs" ], "cost_reduction_reported": null, "benchmarks": [ "MultiAgentBench (MARBLE)" ], "models_studied": [ "gpt-4o-mini", "other LLMs" ], "production_evidence": false, "summary": "Benchmark for LLM multi-agent systems that measures task completion and the quality of collaboration and competition via milestone-based KPIs, across coordination topologies (star, chain, tree, graph) and strategies like group discussion and cognitive planning. gpt-4o-mini achieves the best average task score, and graph topology works best in research scenarios.", "key_idea": "Evaluate multi-agent topologies and coordination, not just outcomes.", "relevance_score": 3, "relevance_rationale": "Useful for judging when multi-agent structure pays off.", "found_via": "web search: MultiAgentBench", "cites_or_cited_by": [] }, { "id": "anthropic-2024-prompt-caching", "title": "Prompt caching (Claude API documentation) and \"Prompt caching with Claude\" announcement", "authors": [ "Anthropic" ], "year": 2024, "venue_or_source": "Anthropic documentation / Anthropic news", "source_type": "docs", "url": "https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "cache_control breakpoints", "exact prefix match" ], "cost_reduction_reported": "announcement (anthropic.com/news/prompt-caching): up to 90% lower cost and up to 85% lower latency for long prompts; docs: cache reads billed at 0.1x base input price, 5-minute writes at 1.25x and 1-hour writes at 2x", "benchmarks": [], "models_studied": [ "Claude models" ], "production_evidence": true, "summary": "Official documentation for Claude prompt caching: developers mark cacheable prefixes (tools, system prompt, documents, conversation) with cache_control, either automatically or with explicit breakpoints, with 5-minute (default) or 1-hour TTLs and a model-dependent minimum cacheable length. The GA announcement (Dec 2024) claims up to 90% cost and 85% latency reduction for long prompts.", "key_idea": "Explicit, prefix-exact KV caching priced at a fraction of input tokens.", "relevance_score": 3, "relevance_rationale": "First-order production cost lever for multi-turn agents on a frontier API; pricing mechanics shape agent context design.", "found_via": "Anthropic prompt caching documentation", "cites_or_cited_by": [ "ji-2025-manus-context-engineering", "khailo-2026-keepalive-economics", "lumer-2026-dont-break-the-cache" ] }, { "id": "anthropic-2024-prompt-caching-b", "title": "Prompt caching with Claude", "authors": [ "Anthropic" ], "year": 2024, "venue_or_source": "Anthropic news", "source_type": "blog", "url": "https://www.anthropic.com/news/prompt-caching", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) Up to 90% cost and 79% latency reduction (100K-token book chat); 53% cost / 75% latency for 10-turn conversation", "benchmarks": [], "models_studied": [ "Claude 3.5 Sonnet", "Claude 3 Opus", "Claude 3 Haiku" ], "production_evidence": false, "summary": "Launch post for explicit prompt caching, with cache writes priced above and cache reads far below base input price, and use-case latency and cost reductions; Notion quoted as an early user. Docs at docs.anthropic.com/en/docs/build-with-claude/prompt-caching (5-minute or 1-hour TTL).", "key_idea": "Explicit prefix caching with priced writes and cheap reads.", "relevance_score": 3, "relevance_rationale": "Seed; major provider cost lever for agents.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [] }, { "id": "bai-2024-digirl", "title": "DigiRL: Training In-The-Wild Device-Control Agents with Autonomous Reinforcement Learning", "authors": [ "Hao Bai", "Yifei Zhou", "Mert Cemri", "Jiayi Pan", "Alane Suhr", "Sergey Levine", "Aviral Kumar" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.11896", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "VLM evaluator reward", "autonomous RL" ], "cost_reduction_reported": "1.3B VLM success rises from 17.7% to 67.2% on AitW, far above GPT-4V AppAgent (8.3%)", "benchmarks": [ "Android-in-the-Wild" ], "models_studied": [ "1.3B VLM", "GPT-4V AppAgent", "CogAgent" ], "production_evidence": false, "summary": "Offline-to-online RL for in-the-wild Android device control using an automatic evaluator as reward, lifting a 1.3B VLM far above prompted GPT-4V agents.", "key_idea": "Autonomous RL lets a tiny VLM beat frontier prompting.", "relevance_score": 3, "relevance_rationale": "Small-agent-beats-frontier evidence.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "boyeau-2024-autoeval-done-right", "title": "AutoEval Done Right: Using Synthetic Data for Model Evaluation", "authors": [ "Pierre Boyeau", "Anastasios Nikolas Angelopoulos", "Tian-Le Li", "N. Yosef", "Jitendra Malik", "Michael I. Jordan" ], "year": 2024, "venue_or_source": "International Conference on Machine Learning", "source_type": "paper", "url": "https://arxiv.org/abs/2403.07008", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "AI-labeled synthetic data", "human labels" ], "cost_reduction_reported": "Effective human-labeled sample size increased by up to 50% in GPT-4 experiments", "benchmarks": [], "models_studied": [ "GPT-4" ], "production_evidence": false, "summary": "Uses prediction-powered inference to combine AI-labeled data with a small set of human labels for model evaluation, giving unbiased estimates with better sample efficiency.", "key_idea": "Unbiased model evaluation mixing AI and human labels.", "relevance_score": 3, "relevance_rationale": "Supporting technique for evaluating configs with few ground-truth labels.", "found_via": "snowball: related work of xiang-2026-otrope", "cites_or_cited_by": [ "xiang-2026-otrope" ] }, { "id": "brown-2024-large-language-monkeys", "title": "Large Language Monkeys: Scaling Inference Compute with Repeated Sampling", "authors": [ "Bradley Brown", "Jordan Juravsky", "Ryan Ehrlich", "Ronald Clark", "Quoc V. Le", "Christopher Ré", "Azalia Mirhoseini" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.21787", "code_url": null, "topic_clusters": [ "C7", "C9", "C13", "C15" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "repeated sampling", "automatic verifiers", "automatic verifier" ], "cost_reduction_reported": "SWE-bench Lite: DeepSeek-V2-Coder solves 56% with 250 samples vs 15.9% with one; five cheap samples solve more issues at lower cost than a premium frontier single attempt", "benchmarks": [ "SWE-bench Lite", "coding", "formal proofs", "GSM8K", "MATH", "MiniF2F", "CodeContests" ], "models_studied": [ "DeepSeek-Coder-V2-Instruct", "DeepSeek-V2-Coder-Instruct" ], "production_evidence": false, "summary": "Shows coverage from repeated sampling scales log-linearly over four orders of magnitude, so cheap models with many samples can win where verifiers exist; voting and reward models plateau without verifiers.", "key_idea": "Inference-time scaling laws for repeated sampling.", "relevance_score": 3, "relevance_rationale": "Cheap-model-plus-sampling vs expensive model trade-off.", "found_via": "snowball: related work of snell-2024-test-time-compute-optimal | keyword: verifier", "cites_or_cited_by": [ "snell-2024-test-time-compute-optimal" ] }, { "id": "chatzi-2024-prediction-powered-ranking", "title": "Prediction-Powered Ranking of Large Language Models", "authors": [ "Ivi Chatzi", "Eleni Straitouri", "Suhas Thejaswi", "Manuel Gomez-Rodriguez" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.17826", "code_url": null, "topic_clusters": [ "C6", "C7", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "human pairwise comparisons", "LLM judge comparisons" ], "cost_reduction_reported": null, "benchmarks": [ "LMSYS Chatbot Arena" ], "models_studied": [ "three strong LLM judges" ], "production_evidence": false, "summary": "Combines a small set of human pairwise comparisons with many LLM-judge comparisons via prediction-powered inference to give rank-sets with coverage guarantees; finds judge-only rankings often inconsistent with human preference.", "key_idea": "Statistically valid ranking from few human plus many judge labels.", "relevance_score": 3, "relevance_rationale": "Method to combine cheap judge labels with scarce human feedback, applicable to router evaluation.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "chiang-2024-chatbot-arena" ] }, { "id": "chen-2024-agent-flan", "title": "Agent-FLAN: Designing Data and Methods of Effective Agent Tuning for Large Language Models", "authors": [ "Zehui Chen", "Kuikun Liu", "Qiuchen Wang", "Wenwei Zhang", "Jiangning Liu", "Dahua Lin", "Kai Chen", "Feng Zhao" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2403.12881", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "decomposed agent corpus", "negative samples" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Llama2-7B" ], "production_evidence": false, "summary": "Finds agent corpora entangle format following and reasoning, capabilities are learned at different speeds, and agent tuning can induce hallucination; Agent-FLAN redesigns data and uses negative samples, beating prior agent tuning on Llama2-7B.", "key_idea": "Data design principles for agent tuning.", "relevance_score": 3, "relevance_rationale": "Recipe details for SFT on agent traces.", "found_via": "seed", "cites_or_cited_by": [ "zeng-2023-agenttuning" ] }, { "id": "chen-2024-octopus-v2", "title": "Octopus v2: On-device language model for super agent", "authors": [ "Wei Chen", "Zhiyuan Li" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.01744", "code_url": null, "topic_clusters": [ "C8", "C11" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "functional tokens" ], "cost_reduction_reported": "2B on-device model surpasses GPT-4 in accuracy and latency and cuts context length by 95%", "benchmarks": [ "Android API function calling" ], "models_studied": [ "2B on-device model", "GPT-4" ], "production_evidence": false, "summary": "Fine-tunes a 2B model with functional tokens for on-device function calling, beating GPT-4 on accuracy and latency with much shorter context.", "key_idea": "Functional tokens for tiny on-device agents.", "relevance_score": 3, "relevance_rationale": "Specialization with latency/cost wins.", "found_via": "keyword: function calling small language model", "cites_or_cited_by": [] }, { "id": "chen-2024-overthinking-o1", "title": "Do NOT Think That Much for 2+3=? On the Overthinking of o1-Like LLMs", "authors": [ "Xingyu Chen", "Jiahao Xu", "Tian Liang", "Zhiwei He", "Jianhui Pang", "Dian Yu", "Linfeng Song", "Qiuzhi Liu", "Mengfei Zhou", "Zhuosheng Zhang", "Rui Wang", "Zhaopeng Tu", "Haitao Mi", "Dong Yu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.21187", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "outcome and process efficiency metrics" ], "cost_reduction_reported": "Reduces computational overhead while preserving performance on GSM8K, MATH500, GPQA, AIME (no single number in abstract)", "benchmarks": [ "GSM8K", "MATH500", "GPQA", "AIME" ], "models_studied": [ "o1-like models", "DeepSeek-R1" ], "production_evidence": false, "summary": "First systematic study of overthinking in o1-style models, which spend many tokens on easy problems for little benefit. Introduces outcome and process efficiency metrics and uses self-training to shorten reasoning without losing accuracy.", "key_idea": "Measure and self-train away overthinking on easy problems.", "relevance_score": 3, "relevance_rationale": "Seed; defines the overthinking cost problem for reasoning models.", "found_via": "seed: Do NOT Think That Much for 2+3=?", "cites_or_cited_by": [] }, { "id": "chen-2024-small-models-survey", "title": "What is the Role of Small Models in the LLM Era: A Survey", "authors": [ "Lihu Chen", "Gaël Varoquaux" ], "year": 2024, "venue_or_source": "Computational Linguistics", "source_type": "paper", "url": "https://arxiv.org/abs/2409.06857", "code_url": null, "topic_clusters": [ "C1", "C8", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Surveys the role of small models alongside LLMs from collaboration and competition perspectives, including using small models for routing, cascades and cheap substitutes.", "key_idea": "Small models in the LLM era.", "relevance_score": 3, "relevance_rationale": "Supports downscaling arguments.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "cheng-2024-trace", "title": "Trace is the Next AutoDiff: Generative Optimization with Rich Feedback, Execution Traces, and LLMs", "authors": [ "Ching-An Cheng", "Allen Nie", "Adith Swaminathan" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.16218", "code_url": "https://github.com/microsoft/Trace", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "execution traces", "rich feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Frames workflow optimization as Optimization with Trace Oracle (OPTO), where an LLM-based optimizer sees the execution trace plus feedback and updates heterogeneous parameters (prompts, code, hyperparameters). Provides the Trace library and the OptoPrime optimizer, competitive with specialized optimizers across several domains.", "key_idea": "Execution traces act like back-propagated gradients for LLM-driven generative optimization of workflows.", "relevance_score": 3, "relevance_rationale": "General generative-optimization framework applicable to agent configs including hyperparameters; no explicit cost objective.", "found_via": "seed: Trace", "cites_or_cited_by": [ "nie-2026-generative-opt-challenges" ] }, { "id": "chezelles-2024-browsergym", "title": "The BrowserGym Ecosystem for Web Agent Research", "authors": [ "Thibault Le Sellier De Chezelles", "Maxime Gasse", "Alexandre Drouin", "Massimo Caccia", "Léo Boisvert", "Megh Thakkar", "Tom Marty", "Rim Assouel", "Sahar Omidi Shayegan", "Lawrence Keunho Jang", "Xing Han Lù", "Ori Yoran", "Dehan Kong", "Frank F. Xu", "Siva Reddy", "Quentin Cappart", "Graham Neubig", "Ruslan Salakhutdinov", "Nicolas Chapados", "Alexandre Lacoste" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.05467", "code_url": "https://github.com/ServiceNow/BrowserGym", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "6 web-agent benchmarks unified in BrowserGym (incl. WorkArena)" ], "models_studied": [ "Claude-3.5-Sonnet", "GPT-4o", "4 other LLMs" ], "production_evidence": false, "summary": "Unifies web-agent benchmarks under a gym-like interface and adds AgentLab for building, running and analyzing agents with reproducibility and a leaderboard. Runs a large multi-benchmark comparison of six LLMs across six benchmarks.", "key_idea": "Common harness for comparing web agents across benchmarks.", "relevance_score": 3, "relevance_rationale": "Evaluation infrastructure for enterprise web agents; enables controlled model-swap cost studies.", "found_via": "web search: BrowserGym ecosystem", "cites_or_cited_by": [ "drouin-2024-workarena", "zhou-2023-webarena" ] }, { "id": "deshpande-2024-glider", "title": "GLIDER: Grading LLM Interactions and Decisions using Explainable Ranking", "authors": [ "Darshan Deshpande", "Selvan Sunitha Ravi", "Sky CH-Wang", "Bartosz Mielczarek", "Anand Kannappan", "Rebecca Qian" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.14140", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "small judge score" ], "cost_reduction_reported": "3B judge matches LLMs 17x its size", "benchmarks": [ "FLASK" ], "models_studied": [ "GLIDER 3B", "GPT-4o" ], "production_evidence": false, "summary": "3B evaluator trained on 685 domains and 183 criteria, with higher Pearson correlation than GPT-4o on FLASK and explainable span highlighting.", "key_idea": "Tiny explainable judge.", "relevance_score": 3, "relevance_rationale": "Cheap judge.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "duan-2024-muxserve", "title": "MuxServe: Flexible Spatial-Temporal Multiplexing for Multiple LLM Serving", "authors": [ "Jiangfei Duan", "Runyu Lu", "Haojie Duanmu", "Xiuhong Li", "Xingcheng Zhang", "Dahua Lin", "Ion Stoica", "Hao Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.02015", "code_url": "https://github.com/hao-ai-lab/MuxServe", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "model popularity", "prefill/decode characteristics" ], "cost_reduction_reported": "up to 1.8x higher throughput or 2.9x more requests within 99% SLO attainment", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Colocates multiple LLMs on shared GPUs according to popularity and separates prefill and decode to multiplex memory and compute spatially and temporally.", "key_idea": "Spatial-temporal multiplexing of multiple LLMs.", "relevance_score": 3, "relevance_rationale": "Serving several model sizes (as routing requires) on shared hardware.", "found_via": "MuxServe multiple LLM serving spatial temporal multiplexing", "cites_or_cited_by": [ "li-2023-alpaserve" ] }, { "id": "erdogan-2024-tinyagent", "title": "TinyAgent: Function Calling at the Edge", "authors": [ "Lutfi Eren Erdogan", "Nicholas Lee", "Siddharth Jha", "Sehoon Kim", "Ryan Tabrizi", "Suhong Moon", "Coleman Hooper", "Gopala Anumanchipalli", "Kurt Keutzer", "Amir Gholami" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2409.00608", "code_url": null, "topic_clusters": [ "C8", "C11" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "curated function-calling data", "tool retrieval" ], "cost_reduction_reported": "1.1B and 7B models match or surpass GPT-4-Turbo function-calling on the target task", "benchmarks": [ "TinyAgent macOS assistant tasks" ], "models_studied": [ "TinyLlama-1.1B", "Wizard-7B", "GPT-4-Turbo" ], "production_evidence": false, "summary": "End-to-end framework for training and deploying small function-calling agents at the edge using curated data, tool retrieval to shrink prompts, and quantization.", "key_idea": "Edge-deployable small agents with tool retrieval.", "relevance_score": 3, "relevance_rationale": "Specialization with deployment cost focus.", "found_via": "hf-papers search: small language model function calling", "cites_or_cited_by": [] }, { "id": "farquhar-2024-semantic-entropy-nature", "title": "Detecting hallucinations in large language models using semantic entropy", "authors": [ "Sebastian Farquhar", "Jannik Kossen", "Lorenz Kuhn", "Yarin Gal" ], "year": 2024, "venue_or_source": "Nature", "source_type": "paper", "url": "https://www.nature.com/articles/s41586-024-07421-0", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "semantic entropy", "sampled generations" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Journal version of semantic entropy for detecting confabulations: sample several answers, cluster them by meaning, and use entropy over meanings to flag likely hallucinations across QA and longer generations. Serves as the reference method that later cheap probes (SEPs) approximate.", "key_idea": "Entropy over semantic clusters of samples detects confabulation.", "relevance_score": 3, "relevance_rationale": "Seed; canonical ground-truth-free hallucination signal, though sampling-heavy.", "found_via": "seed (DOI verified via OpenAlex)", "cites_or_cited_by": [ "kossen-2024-semantic-entropy-probes" ] }, { "id": "fisch-2024-stratified-ppi", "title": "Stratified Prediction-Powered Inference for Hybrid Language Model Evaluation", "authors": [ "Adam Fisch", "Joshua Maynez", "R. Alex Hofer", "Bhuwan Dhingra", "Amir Globerson", "William W. Cohen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.04291", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "autorater labels", "human labels" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Improves prediction-powered inference for language-model evaluation by stratifying data, yielding provably valid and tighter confidence intervals when combining a few human labels with many autorater labels.", "key_idea": "Stratified PPI for hybrid human/autorater evaluation.", "relevance_score": 3, "relevance_rationale": "Label-efficient, valid evaluation of candidate routes or configs.", "found_via": "snowball: related work of xiang-2026-otrope", "cites_or_cited_by": [ "xiang-2026-otrope" ] }, { "id": "fourney-2024-magentic-one", "title": "Magentic-One: A Generalist Multi-Agent System for Solving Complex Tasks", "authors": [ "Adam Fourney", "Gagan Bansal", "Hussein Mozannar", "Cheng Tan", "Eduardo Salinas", "Erkang", "Zhu", "Friederike Niedtner", "Grace Proebsting", "Griffin Bassman", "Jack Gerrits", "Jacob Alber", "Peter Chang", "Ricky Loynd", "Robert West", "Victor Dibia", "Ahmed Awadallah", "Ece Kamar", "Rafah Hosn", "Saleema Amershi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.04468", "code_url": "https://aka.ms/magentic-one", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "GAIA", "AssistantBench", "WebArena" ], "models_studied": [], "production_evidence": false, "summary": "Microsoft's Magentic-One uses an Orchestrator that plans, tracks progress and re-plans while directing specialist agents (web, file, coder, terminal). Its modular design allows different models per agent, and it reaches competitive results on GAIA, AssistantBench and WebArena.", "key_idea": "Orchestrator-plus-specialists architecture with swappable per-agent models.", "relevance_score": 3, "relevance_rationale": "Production-grade reference orchestration pattern supporting heterogeneous model assignment.", "found_via": "arXiv title search", "cites_or_cited_by": [] }, { "id": "fu-2024-tiny-titans", "title": "Tiny Titans: Can Smaller Large Language Models Punch Above Their Weight in the Real World for Meeting Summarization?", "authors": [ "Xue-Yong Fu", "Md Tahmid Rahman Laskar", "Elena Khasanova", "Cheng Chen", "Shashi Bhushan TN" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.00841", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "task fine-tuning" ], "cost_reduction_reported": "FLAN-T5 (780M) performs on par or better than many zero-shot 7B to 70B+ LLMs on meeting summarization", "benchmarks": [ "real-world meeting summarization" ], "models_studied": [ "FLAN-T5", "TinyLLaMA", "LiteLLaMA", "GPT-3.5", "PaLM-2" ], "production_evidence": true, "summary": "Industrial study comparing fine-tuned compact LLMs (FLAN-T5, TinyLLaMA, LiteLLaMA) with larger zero-shot LLMs on real-world meeting summarization. Most compact models still trail, but FLAN-T5 (780M) matches or beats many 7B-70B zero-shot LLMs, making it a cost-efficient deployment option.", "key_idea": "Compact fine-tuned models as cost-efficient production alternatives.", "relevance_score": 3, "relevance_rationale": "Real industrial deployment study.", "found_via": "hf-papers search: RL fine-tuning small LLM tool use", "cites_or_cited_by": [] }, { "id": "gao-2024-cachedattention", "title": "Cost-Efficient Large Language Model Serving for Multi-turn Conversations with CachedAttention", "authors": [ "Bin Gao", "Zhuomin He", "Puru Sharma", "Qingxuan Kang", "Djordje Jevdjic", "Junbo Deng", "Xingkun Yang", "Zhou Yu", "Pengfei Zuo" ], "year": 2024, "venue_or_source": "USENIX ATC 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2403.19708", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "scheduler hints for KV prefetch/eviction" ], "cost_reduction_reported": "TTFT reduced by up to 87%, prefill throughput up to 7.8x higher for multi-turn conversations, and end-to-end inference cost reduced by up to 70%", "benchmarks": [ "multi-turn conversation workloads" ], "models_studied": [], "production_evidence": false, "summary": "Keeps KV caches of past conversation turns in a hierarchical store (GPU, host memory, disk) so later turns avoid recomputing history, with scheduler-aware prefetch/eviction and positional-encoding decoupling to survive context truncation.", "key_idea": "Persist multi-turn KV caches in a memory hierarchy.", "relevance_score": 3, "relevance_rationale": "Multi-turn KV reuse is exactly the agent-loop pattern.", "found_via": "snowball: related work of li-2025-continuum", "cites_or_cited_by": [] }, { "id": "gill-2024-meancache", "title": "MeanCache: User-Centric Semantic Caching for LLM Web Services", "authors": [ "Waris Gill", "Mohamed Elidrisi", "Pallavi Kalapatapu", "Ammar Ahmed", "Ali Anwar", "Muhammad Ali Gulzar" ], "year": 2024, "venue_or_source": "IPDPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2403.02694", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "query embedding similarity", "federated-learned similarity model", "conversation context chain" ], "cost_reduction_reported": "~17% higher F-score and 20% higher precision on semantic cache hit/miss decisions; 83% less storage; 11% faster hit/miss decisions", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "User-side semantic cache in which each device keeps a local cache and a query-similarity model trained with federated learning, so user queries never leave the device. It also stores context chains so that follow-up queries that look similar but depend on different conversation context are not falsely matched.", "key_idea": "Federated, context-aware semantic caching to cut repeated LLM calls while reducing false hits.", "relevance_score": 3, "relevance_rationale": "Addresses semantic-cache false hits on contextual (multi-turn) queries, a key risk for agent use.", "found_via": "MeanCache user-centric semantic cache LLM", "cites_or_cited_by": [ "bang-2023-gptcache" ] }, { "id": "google-2024-gemini-context-caching", "title": "Context caching (Gemini API documentation)", "authors": [ "Google" ], "year": 2024, "venue_or_source": "Google AI for Developers documentation", "source_type": "docs", "url": "https://ai.google.dev/gemini-api/docs/caching", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "implicit prefix caching", "explicit cache objects with TTL" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Gemini models" ], "production_evidence": true, "summary": "Gemini offers implicit caching (on by default for Gemini 2.5 and newer, savings passed on when requests hit, no guarantee) and explicit caching where developers create cache objects with a TTL for repeatedly used content such as long documents or media; minimum cacheable token counts vary by model.", "key_idea": "Implicit plus explicit, TTL-managed context caches.", "relevance_score": 3, "relevance_rationale": "Third major provider caching mechanism; explicit caches suit shared agent context.", "found_via": "Gemini API context caching documentation | seed: provider cost-optimization guides", "cites_or_cited_by": [ "kilpatrick-2025-gemini-implicit-caching", "lumer-2026-dont-break-the-cache" ] }, { "id": "guha-2024-smoothie", "title": "Smoothie: Label Free Language Model Routing", "authors": [ "Neel Guha", "Mayee F. Chen", "Trevor Chow", "Ishan S. Khare", "Christopher Ré" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "paper", "url": "https://arxiv.org/abs/2412.04692", "code_url": null, "topic_clusters": [ "C1", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "embedding agreement across model outputs", "latent-variable graphical model" ], "cost_reduction_reported": null, "benchmarks": [ "14 tasks" ], "models_studied": [], "production_evidence": false, "summary": "Smoothie performs label-free routing by fitting a weak-supervision latent variable model over embeddings of multiple LLM outputs, estimating per-sample model quality. Outperforms baselines by up to 10 points.", "key_idea": "Unsupervised routing from output agreement.", "relevance_score": 3, "relevance_rationale": "No ground truth needed, but requires generating from all models.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [] }, { "id": "hu-2024-pak-ucb", "title": "PAK-UCB Contextual Bandit: An Online Learning Approach to Prompt-Aware Selection of Generative Models and LLMs", "authors": [ "Xiaoyan Hu", "Ho-fung Leung", "Farzan Farnia" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.13287", "code_url": "https://github.com/yannxiaoyanhu/dgm-online-select", "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "prompt embedding", "generation score" ], "cost_reduction_reported": null, "benchmarks": [ "text-to-image and image-to-text model selection" ], "models_studied": [ "prompt-based generative models", "LLMs" ], "production_evidence": false, "summary": "PAK-UCB is a kernel contextual bandit that learns which generative model is best for each prompt, sharing context across arms and using random Fourier features for speed. Experiments mostly cover text-to-image and image-to-text generators.", "key_idea": "Prompt-aware kernel UCB for per-prompt generative model selection.", "relevance_score": 3, "relevance_rationale": "Online per-prompt model selection; evaluation mostly non-LLM.", "found_via": "OpenAlex title: online model selection LLM", "cites_or_cited_by": [ "hu-2024-mab-generative-eval" ] }, { "id": "huang-2024-crmarena", "title": "CRMArena: Understanding the Capacity of LLM Agents to Perform Professional CRM Tasks in Realistic Environments", "authors": [ "Kung-Hsiang Huang", "Akshara Prabhakar", "Sidharth Dhawan", "Yixin Mao", "Huan Wang", "Silvio Savarese", "Caiming Xiong", "Philippe Laban", "Chien-Sheng Wu" ], "year": 2024, "venue_or_source": "NAACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2411.02305", "code_url": "https://github.com/SalesforceAIResearch/CRMArena", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "CRMArena" ], "models_studied": [ "LLM agents (function calling, ReAct)" ], "production_evidence": false, "summary": "Benchmark of professional CRM tasks in a realistic Salesforce-like environment designed with CRM experts. Agents struggle to complete tasks reliably even with function calling.", "key_idea": "Realistic CRM tasks for agent evaluation.", "relevance_score": 3, "relevance_rationale": "Enterprise workflow benchmark.", "found_via": "seed: CRMArena", "cites_or_cited_by": [ "huang-2025-crmarena-pro" ] }, { "id": "ibm-2024-llm-routers-blog", "title": "LLM routing for quality, low-cost responses", "authors": [ "Tal Shnitzer", "Mikhail Yurochkin" ], "year": 2024, "venue_or_source": "IBM Research blog", "source_type": "blog", "url": "https://research.ibm.com/blog/LLM-routers", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "benchmark-derived correctness predictors" ], "cost_reduction_reported": "Vendor-reported: router saves about 5 cents per query vs GPT-4 at comparable performance on RouterBench", "benchmarks": [ "RouterBench", "HELM" ], "models_studied": [ "GPT-4", "Llama-2 family", "13B models" ], "production_evidence": false, "summary": "IBM Research blog on routing learned from benchmark data (companion to the COLM paper \"Large Language Model Routing with Benchmark Datasets\"), used in IBM products.", "key_idea": "Benchmark-trained correctness predictors for routing.", "relevance_score": 3, "relevance_rationale": "Industrial research router blog.", "found_via": "seed list: model router product", "cites_or_cited_by": [ "hu-2024-routerbench" ] }, { "id": "jha-2024-prompt-compression-characterization", "title": "Characterizing Prompt Compression Methods for Long Context Inference", "authors": [ "Siddharth Jha", "Lutfi Eren Erdogan", "Sehoon Kim", "Kurt Keutzer", "Amir Gholami" ], "year": 2024, "venue_or_source": "ES-FoMo workshop @ ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2407.08892", "code_url": null, "topic_clusters": [ "C10", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "extractive compression enables up to 10x compression with minimal accuracy degradation; token pruning often lags extractive compression", "benchmarks": [ "long-context QA and summarization" ], "models_studied": [], "production_evidence": false, "summary": "Controlled comparison of extractive, abstractive (summarization) and token-pruning compression for long-context inference. Finds that simple extractive compression is often best, contradicting several claims for token-pruning methods.", "key_idea": "Under standardized evaluation, extractive compression beats token pruning.", "relevance_score": 3, "relevance_rationale": "Useful evidence for choosing a compression method for agent contexts.", "found_via": "prompt compression survey LLM 2024 2025", "cites_or_cited_by": [ "jiang-2023-llmlingua", "pan-2024-llmlingua-2" ] }, { "id": "kapoor-2024-taught-to-know-uncertainty", "title": "Large Language Models Must Be Taught to Know What They Don't Know", "authors": [ "Sanyam Kapoor", "Nate Gruver", "Manley Roberts", "Katherine Collins", "Arka Pal", "Umang Bhatt", "Adrian Weller", "Samuel Dooley", "Micah Goldblum", "Andrew Gordon Wilson" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.08391", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "fine-tuned uncertainty head", "LoRA" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU", "open-ended QA" ], "models_studied": [], "production_evidence": false, "summary": "Shows prompting alone gives poor calibration and that fine-tuning on about a thousand graded examples yields generalizable, cheap uncertainty estimates; such estimators can also judge other models.", "key_idea": "A small labeled set suffices to train reliable uncertainty estimators.", "relevance_score": 3, "relevance_rationale": "Low-cost path to a routing confidence signal.", "found_via": "keyword: confidence estimation", "cites_or_cited_by": [] }, { "id": "kim-2024-autorag", "title": "AutoRAG: Automated Framework for optimization of Retrieval Augmented Generation Pipeline", "authors": [ "Dongkyu Kim", "Byoungwook Kim", "Donggeon Han", "Matouš Eibich" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.20878", "code_url": "https://github.com/Marker-Inc-Korea/AutoRAG", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "retrieval and generation metrics" ], "cost_reduction_reported": null, "benchmarks": [ "ARAGOG" ], "models_studied": [], "production_evidence": false, "summary": "AutoRAG automatically selects RAG modules for a given dataset by greedily optimizing one pipeline stage at a time in pipeline order, and releases its experimental results.", "key_idea": "Greedy stage-by-stage module selection for RAG pipelines.", "relevance_score": 3, "relevance_rationale": "RAG config search tool; quality-focused.", "found_via": "search: AutoRAG automated RAG pipeline optimization arXiv", "cites_or_cited_by": [] }, { "id": "kim-2024-prometheus-2", "title": "Prometheus 2: An Open Source Language Model Specialized in Evaluating Other Language Models", "authors": [ "Seungone Kim", "Juyoung Suk", "Shayne Longpre", "Bill Yuchen Lin", "Jamin Shin", "Sean Welleck", "Graham Neubig", "Moontae Lee", "Kyungjae Lee", "Minjoon Seo" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.01535", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "judge score", "rubric" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Mistral-7B", "Mixtral-8x7B" ], "production_evidence": false, "summary": "Open evaluator LM supporting both direct assessment and pairwise ranking under user-defined criteria, closely matching human and GPT-4 judgments on four direct and four pairwise benchmarks.", "key_idea": "Open small judge model mirrors GPT-4 grading.", "relevance_score": 3, "relevance_rationale": "Cheap in-house judge for quality monitoring.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "li-2024-agent-oriented-planning", "title": "Agent-Oriented Planning in Multi-Agent Systems", "authors": [ "Ao Li", "Yuexiang Xie", "Songze Li", "Fugee Tsung", "Bolin Ding", "Yaliang Li" ], "year": 2024, "venue_or_source": "ICLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2410.02189", "code_url": "https://github.com/lalaliat/Agent-Oriented-Planning", "topic_clusters": [ "C3" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "reward model on sub-task solvability" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AOP has a meta-agent decompose queries into sub-tasks allocated to suitable agents, guided by solvability, completeness and non-redundancy principles and a reward model that evaluates allocations, with a feedback loop for adjustment.", "key_idea": "Principled sub-task allocation to agents with a reward model.", "relevance_score": 3, "relevance_rationale": "Routing sub-tasks among agents.", "found_via": "arXiv API search", "cites_or_cited_by": [] }, { "id": "li-2024-arena-hard", "title": "From Crowdsourced Data to High-Quality Benchmarks: Arena-Hard and BenchBuilder Pipeline", "authors": [ "Tianle Li", "Wei-Lin Chiang", "Evan Frick", "Lisa Dunlap", "Tianhao Wu", "Banghua Zhu", "Joseph E. Gonzalez", "Ion Gabriel Stoica" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.11939", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "LLM judge" ], "cost_reduction_reported": "Arena-Hard-Auto: 3x higher separation than MT-Bench, 98.6% correlation with human preference rankings, at a cost of $20", "benchmarks": [ "Arena-Hard-Auto", "Chatbot Arena", "WildChat-1M" ], "models_studied": [], "production_evidence": false, "summary": "BenchBuilder curates hard prompts from crowdsourced logs (Chatbot Arena, WildChat) and scores them with LLM judges, producing Arena-Hard-Auto.", "key_idea": "Turning logged traffic into a cheap, human-aligned eval set.", "relevance_score": 3, "relevance_rationale": "Method for building offline evals from production-like logs.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "chiang-2024-chatbot-arena", "zhao-2024-wildchat" ] }, { "id": "li-2024-early-stopping-self-consistency", "title": "Escape Sky-high Cost: Early-stopping Self-Consistency for Multi-step Reasoning", "authors": [ "Yiwei Li", "Peiwen Yuan", "Shaoxiong Feng", "Boyuan Pan", "Xinglin Wang", "Bin Sun", "Heda Wang", "Kan Li" ], "year": 2024, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2401.10480", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "vote agreement in sampling windows" ], "cost_reduction_reported": "Reduces average number of CoT samples: MATH -33.8%, GSM8K -80.1%, StrategyQA -76.8%, CommonsenseQA -78.5%, Coin Flip -84.2%, Last Letters -67.4% at comparable performance", "benchmarks": [ "MATH", "GSM8K", "StrategyQA", "CommonsenseQA", "Coin Flip", "Last Letters" ], "models_studied": [], "production_evidence": false, "summary": "Samples self-consistency in small windows and stops once a window is unanimous, with a control scheme to set the cost-performance balance.", "key_idea": "Stop self-consistency sampling once answers agree.", "relevance_score": 3, "relevance_rationale": "Sampling-budget control.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "li-2024-smoa", "title": "SMoA: Improving Multi-agent Large Language Models with Sparse Mixture-of-Agents", "authors": [ "Dawei Li", "Zhen Tan", "Peijia Qian", "Yifan Li", "Kumar Satvik Chaudhary", "Lijie Hu", "Jiayi Shen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.03284", "code_url": "https://github.com/David-Li0406/SMoA", "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "judge selection", "early stopping" ], "cost_reduction_reported": "Comparable to MoA with significantly lower computational cost", "benchmarks": [ "reasoning, alignment and fairness benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Sparse Mixture-of-Agents adds response selection by a judge and early stopping so only a subset of agent outputs flows forward between layers, and gives agents distinct roles for diversity. It matches dense MoA at substantially lower compute.", "key_idea": "Sparsify MoA information flow with judge-based selection and early stopping.", "relevance_score": 3, "relevance_rationale": "Cost-aware MoA variant.", "found_via": "web search: Mixture-of-Agents cost", "cites_or_cited_by": [] }, { "id": "liu-2024-apigen", "title": "APIGen: Automated Pipeline for Generating Verifiable and Diverse Function-Calling Datasets", "authors": [ "Zuxin Liu", "Thai Hoang", "Jianguo Zhang", "Ming Zhu", "Tian Lan", "Shirley Kokane", "Juntao Tan", "Weiran Yao", "Zhiwei Liu", "Yihao Feng", "Rithesh Murthy", "Liangwei Yang", "Silvio Savarese", "Juan Carlos Niebles", "Huan Wang", "Shelby Heinecke", "Caiming Xiong" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.18518", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "verified synthetic function calls" ], "cost_reduction_reported": "7B models trained on APIGen data reach SOTA on BFCL, outperforming multiple GPT-4 models", "benchmarks": [ "BFCL" ], "models_studied": [ "xLAM-1B", "xLAM-7B", "GPT-4" ], "production_evidence": false, "summary": "Automated pipeline that generates function-calling data from 3,673 executable APIs with format, execution and semantic verification stages; small models trained on it beat GPT-4 variants on BFCL.", "key_idea": "Multi-stage verified synthetic function-calling data.", "relevance_score": 3, "relevance_rationale": "Recipe for synthetic tool data to specialize small models.", "found_via": "seed", "cites_or_cited_by": [ "prabhakar-2025-apigen-mt", "zhang-2024-xlam" ] }, { "id": "liu-2024-meta-ranking", "title": "Enabling Weak LLMs to Judge Response Reliability via Meta Ranking", "authors": [ "Zijun Liu", "Boqun Kou", "Peng Li", "Ming Yan", "Ji Zhang", "Fei Huang", "Yang Liu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.12146", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "cross-query pairwise reliability ranking" ], "cost_reduction_reported": "Model cascade of open and closed LLMs reaches performance comparable to GPT-4-turbo at lower cost (no single number in abstract)", "benchmarks": [], "models_studied": [ "Phi-2", "GPT-3.5-turbo", "GPT-4-turbo", "LLaMA-7B" ], "production_evidence": false, "summary": "Meta Ranking lets weak LLMs judge response reliability by comparing a target query-response pair against a few reference pairs. Used to decide escalation in a cascade and to filter instruction data.", "key_idea": "Weak-model reliability judging by comparison to references.", "relevance_score": 3, "relevance_rationale": "Cheap reliability signal for cascades.", "found_via": "arXiv search: model cascade language model", "cites_or_cited_by": [] }, { "id": "liu-2024-toolace", "title": "ToolACE: Winning the Points of LLM Function Calling", "authors": [ "Weiwen Liu", "Xu Huang", "Xingshan Zeng", "Xinlong Hao", "Shuai Yu", "Dexun Li", "Shuai Wang", "Weinan Gan", "Zhengying Liu", "Yuanqing Yu", "Zezhong Wang", "Yuxian Wang", "Wu Ning", "Yutai Hou", "Bin Wang", "Chuhan Wu", "Xinzhi Wang", "Yong Liu", "Yasheng Wang", "Duyu Tang", "Dandan Tu", "Lifeng Shang", "Xin Jiang", "Ruiming Tang", "Defu Lian", "Qun Liu", "Enhong Chen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2409.00920", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "self-evolved API pool", "dual verification" ], "cost_reduction_reported": "8B models rival latest GPT-4 models on BFCL", "benchmarks": [ "BFCL" ], "models_studied": [], "production_evidence": false, "summary": "Synthesizes function-calling data over 26,507 APIs with self-evolving synthesis and dual-layer verification; 8B models reach SOTA on BFCL.", "key_idea": "High-quality synthetic function-calling data.", "relevance_score": 3, "relevance_rationale": "Specialization recipe for tool use.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "maurya-2024-selectllm", "title": "SelectLLM: Query-Aware Efficient Selection Algorithm for Large Language Models", "authors": [ "Kaushal Kumar Maurya", "KV Aditya Srivatsa", "Ekaterina Kochmar" ], "year": 2024, "venue_or_source": "ACL (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2408.08545", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "multi-label classifier", "confidence scores" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "SelectLLM uses a multi-label classifier and confidence-based policy to pick a small subset of LLMs per query whose combined answers are accurate, reducing latency relative to large ensembles.", "key_idea": "Query-aware subset selection of LLMs.", "relevance_score": 3, "relevance_rationale": "Subset routing.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "miranda-2024-hybrid-preferences", "title": "Hybrid Preferences: Learning to Route Instances for Human vs. AI Feedback", "authors": [ "Lester James V. Miranda", "Yizhong Wang", "Yanai Elazar", "Sachin Kumar", "Valentina Pyatkin", "Faeze Brahman", "Noah A. Smith", "Hannaneh Hajishirzi", "Pradeep Dasigi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.19133", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "instance features", "performance prediction" ], "cost_reduction_reported": "Hybrid human/LM mixture yields better reward-model performance than either source alone while reducing human annotation", "benchmarks": [ "MultiPref" ], "models_studied": [], "production_evidence": false, "summary": "Routes each preference instance to human or LM annotation by training a performance-prediction model that estimates downstream reward-model quality for a given mix. Releases MultiPref (10K instances with both label types) and shows generalization to three other datasets.", "key_idea": "Predict where human labels add value and route only those instances to humans.", "relevance_score": 3, "relevance_rationale": "Performance-prediction-driven routing of costly feedback; relevant to budgeting labels for router training.", "found_via": "query: Chatbot Arena preference data train router", "cites_or_cited_by": [] }, { "id": "montreuil-2024-l2d-extractive-qa", "title": "Optimal Query Allocation in Extractive QA with LLMs: A Learning-to-Defer Framework with Theoretical Guarantees", "authors": [ "Yannis Montreuil", "Shu Heng Yeo", "Axel Carlier", "Lai Xing Ng", "Wei Tsang Ooi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.15761", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "deferral scores" ], "cost_reduction_reported": null, "benchmarks": [ "SQuADv1", "SQuADv2", "TriviaQA" ], "models_studied": [], "production_evidence": false, "summary": "Applies learning-to-defer with theoretical guarantees to allocate extractive QA queries among specialised experts, balancing performance against compute.", "key_idea": "Learning-to-defer to specialised experts for QA cost.", "relevance_score": 3, "relevance_rationale": "L2D theory applied to LLM query allocation.", "found_via": "arXiv search: learning to defer language model", "cites_or_cited_by": [] }, { "id": "notdiamond-2024-rorf", "title": "RoRF - Routing on Random Forests", "authors": [ "Not Diamond" ], "year": 2024, "venue_or_source": "GitHub (Not Diamond)", "source_type": "repo", "url": "https://github.com/Not-Diamond/RoRF", "code_url": "https://github.com/Not-Diamond/RoRF", "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding (jina-embeddings-v3, voyage-large-2-instruct)", "random forest classifier" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "llama-3.1-405b-instruct", "llama-3.1-70b-instruct", "claude-3.5-sonnet" ], "production_evidence": false, "summary": "Not Diamond's open-source framework for training and serving random-forest routers over embedding features for pairs of models, with 12 pretrained routers across 6 model pairs. The README states strong/weak routing cuts cost while keeping the strong model's performance and strong/strong routing can beat both; it reuses RouteLLM's Controller interface (companion blog at notdiamond.ai/blog/rorf).", "key_idea": "Random forests on text embeddings as cheap pairwise routers.", "relevance_score": 3, "relevance_rationale": "Seed; practitioner router from a routing vendor, but no quantitative claims verified here.", "found_via": "seed", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "openai-2024-prompt-caching", "title": "Prompt caching (OpenAI API guide) and \"Prompt Caching in the API\"", "authors": [ "OpenAI" ], "year": 2024, "venue_or_source": "OpenAI API documentation", "source_type": "docs", "url": "https://developers.openai.com/api/docs/guides/prompt-caching", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "automatic longest-prefix match", "prompt_cache_key routing hint" ], "cost_reduction_reported": "cached input tokens discounted up to 95% (docs); automatic for prompts of at least 1,024 tokens", "benchmarks": [], "models_studied": [ "GPT models" ], "production_evidence": true, "summary": "OpenAI applies prompt caching automatically to prompts above a minimum length, routing requests with a shared prefix to servers holding the computed prefix; prompt_cache_key helps partition high-volume traffic for better hit rates, and retention policies control how long prefixes stay warm. Guidance: put static content first and keep context append-only.", "key_idea": "Automatic prefix caching with cache-key-aware routing.", "relevance_score": 3, "relevance_rationale": "Provider-side cost lever that interacts with how agents structure prompts.", "found_via": "OpenAI prompt caching guide docs", "cites_or_cited_by": [ "lumer-2026-dont-break-the-cache" ] }, { "id": "pan-2024-autonomous-eval-digital-agents", "title": "Autonomous Evaluation and Refinement of Digital Agents", "authors": [ "Jiayi Pan", "Yichi Zhang", "Nicholas Tomlin", "Yifei Zhou", "Sergey Levine", "Alane Suhr" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.06474", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "evaluator model", "trajectory success" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Builds domain-general automatic evaluators for web/device agents trading off cost, modularity and accuracy (74.4-92.9% agreement with oracle metrics) and uses them for Reflexion-style refinement and filtered fine-tuning. Improves WebArena state of the art by 29% without extra supervision.", "key_idea": "Cheap automatic evaluators both judge and improve digital agents.", "relevance_score": 3, "relevance_rationale": "Ground-truth-free signal reused for self-training of agents.", "found_via": "keyword: agent trajectory evaluation", "cites_or_cited_by": [ "pan-2024-swe-gym" ] }, { "id": "pan-2024-llmlingua-2", "title": "LLMLingua-2: Data Distillation for Efficient and Faithful Task-Agnostic Prompt Compression", "authors": [ "Zhuoshi Pan", "Qianhui Wu", "Huiqiang Jiang", "Menglin Xia", "Xufang Luo", "Jue Zhang", "Qingwei Lin", "Victor Rühle", "Yuqing Yang", "Chin-Yew Lin", "H. Vicky Zhao", "Lili Qiu", "Dongmei Zhang" ], "year": 2024, "venue_or_source": "Findings of ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2403.12968", "code_url": "https://github.com/microsoft/LLMLingua", "topic_clusters": [ "C8", "C10" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "token keep/drop classifier (bidirectional encoder)" ], "cost_reduction_reported": "3x-6x faster than prior prompt-compression methods; end-to-end latency accelerated 1.6x-2.9x at 2x-5x compression ratios", "benchmarks": [ "MeetingBank", "LongBench", "ZeroScrolls", "GSM8K", "BBH" ], "models_studied": [ "GPT-4 (data distillation)" ], "production_evidence": false, "summary": "Recasts prompt compression as token classification: GPT-4 is used to produce extractive compressed versions of MeetingBank transcripts, and a small bidirectional encoder is trained to predict which tokens to keep. The result is task-agnostic, faithful (extractive) and much faster than perplexity-based compressors.", "key_idea": "Distil a frontier model's compression judgments into a small token classifier.", "relevance_score": 3, "relevance_rationale": "Cheap, task-agnostic compressor usable in front of any frontier-model agent call; also an example of distilling frontier behavior.", "found_via": "LLMLingua-2 data distillation task-agnostic prompt compression", "cites_or_cited_by": [ "jha-2024-prompt-compression-characterization", "jiang-2023-llmlingua", "li-2024-prompt-compression-survey" ] }, { "id": "pan-2024-swe-gym", "title": "Training Software Engineering Agents and Verifiers with SWE-Gym", "authors": [ "Jiayi Pan", "Xingyao Wang", "Graham Neubig", "Navdeep Jaitly", "Heng Ji", "Alane Suhr", "Yizhe Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.21139", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "rejection-sampled trajectories", "verifier" ], "cost_reduction_reported": "Up to 19% absolute gains in resolve rate; 32.0% on SWE-Bench Verified with verifier", "benchmarks": [ "SWE-Bench Verified", "SWE-Bench Lite" ], "models_studied": [], "production_evidence": false, "summary": "Environment of 2,438 real Python tasks used to train SWE agents and outcome verifiers; fine-tuning on sampled trajectories and inference-time verification set open-weight SOTA.", "key_idea": "Train agents and verifiers from executable environments.", "relevance_score": 3, "relevance_rationale": "Combines C8 training with C7 verifiers.", "found_via": "keyword: SFT on agent traces", "cites_or_cited_by": [ "pan-2024-autonomous-eval-digital-agents" ] }, { "id": "polo-2024-prompteval", "title": "Efficient multi-prompt evaluation of LLMs", "authors": [ "Felipe Maia Polo", "Ronald Xu", "Lucas Weber", "Mírian Silva", "Onkar Bhardwaj", "Leshem Choshen", "Allysson Flavio Melo de Oliveira", "Yuekai Sun", "Mikhail Yurochkin" ], "year": 2024, "venue_or_source": "NeurIPS 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2405.17202", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "IRT across prompts and examples" ], "cost_reduction_reported": "estimates performance quantiles across 100 prompt templates on MMLU with a budget of two single-prompt evaluations", "benchmarks": [ "MMLU", "BIG-bench Hard", "LMentry" ], "models_studied": [], "production_evidence": false, "summary": "PromptEval estimates an LLM's performance distribution across many prompt templates under small evaluation budgets by sharing strength across prompts and examples.", "key_idea": "Budgeted estimation of performance over prompt variants.", "relevance_score": 3, "relevance_rationale": "Efficient offline evaluation for configuration choice.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "polo-2024-tinybenchmarks" ] }, { "id": "polo-2024-tinybenchmarks", "title": "tinyBenchmarks: evaluating LLMs with fewer examples", "authors": [ "Felipe Maia Polo", "Lucas Weber", "Leshem Choshen", "Yuekai Sun", "Gongjun Xu", "Mikhail Yurochkin" ], "year": 2024, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2402.14992", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "IRT", "anchor examples" ], "cost_reduction_reported": "estimate MMLU (14K examples) performance from 100 curated examples", "benchmarks": [ "Open LLM Leaderboard", "MMLU", "HELM", "AlpacaEval 2.0" ], "models_studied": [], "production_evidence": false, "summary": "Shows small curated subsets (e.g. 100 MMLU items) plus IRT-based estimators can reproduce full-benchmark LLM results; releases tiny benchmark versions.", "key_idea": "Estimate full-benchmark performance from ~100 items.", "relevance_score": 3, "relevance_rationale": "Cheap evaluation of candidate models/configs before switching.", "found_via": "OpenAlex title: tinyBenchmarks", "cites_or_cited_by": [ "kipnis-2024-metabench", "polo-2024-prompteval", "somerstep-2025-carrot", "song-2025-irt-router", "vivek-2023-anchor-points", "zhang-2025-benchmark-prediction-misses" ] }, { "id": "putta-2024-agent-q", "title": "Agent Q: Advanced Reasoning and Learning for Autonomous AI Agents", "authors": [ "Pranav Putta", "Edmund Mills", "Naman Garg", "Sumeet Motwani", "Chelsea Finn", "Divyansh Garg", "Rafael Rafailov" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.07199", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "MCTS", "self-critique", "DPO" ], "cost_reduction_reported": "Llama-3 70B booking success rises from 18.6% to 81.7% zero-shot after training in real-world booking", "benchmarks": [ "WebShop", "real-world booking" ], "models_studied": [ "Llama-3-70B" ], "production_evidence": true, "summary": "Combines guided MCTS with self-critique and off-policy DPO to improve web agents, including a real-world booking deployment study.", "key_idea": "MCTS plus DPO self-improvement of web agents.", "relevance_score": 3, "relevance_rationale": "Includes real-website results.", "found_via": "keyword: RL fine-tuning for agents", "cites_or_cited_by": [] }, { "id": "qi-2024-webrl", "title": "WebRL: Training LLM Web Agents via Self-Evolving Online Curriculum Reinforcement Learning", "authors": [ "Zehan Qi", "Xiao Liu", "Iat Long Iong", "Hanyu Lai", "Xueqiao Sun", "Xinyue Yang", "Jiadai Sun", "Yu Yang", "Shuntian Yao", "Tianjie Zhang", "Wei Xu", "Jie Tang", "Yuxiao Dong" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.02337", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "outcome reward model", "self-evolving curriculum" ], "cost_reduction_reported": "WebArena-Lite success: Llama-3.1-8B 4.8% to 42.4%, GLM-4-9B 6.1% to 43%, vs GPT-4-Turbo 17.6% and GPT-4o 13.9%", "benchmarks": [ "WebArena-Lite" ], "models_studied": [ "Llama-3.1-8B", "GLM-4-9B", "GPT-4-Turbo" ], "production_evidence": false, "summary": "Online curriculum RL for open web agents with tasks generated from failures, an outcome reward model and adaptive RL, turning Llama-3.1 and GLM-4 into strong web agents.", "key_idea": "Self-evolving curriculum RL with ORM for small web agents.", "relevance_score": 3, "relevance_rationale": "RL route to replace proprietary web agents.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "qin-2024-mooncake", "title": "Mooncake: A KVCache-centric Disaggregated Architecture for LLM Serving", "authors": [ "Ruoyu Qin", "Zheming Li", "Weiran He", "Mingxing Zhang", "Yongwei Wu", "Weimin Zheng", "Xinran Xu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.00079", "code_url": null, "topic_clusters": [ "C10", "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "KVCache-centric scheduling", "early rejection prediction" ], "cost_reduction_reported": "up to 525% throughput increase in simulated scenarios under SLOs; under real workloads Kimi handles 75% more requests", "benchmarks": [ "Kimi production workloads" ], "models_studied": [], "production_evidence": true, "summary": "Serving platform of Moonshot AI's Kimi: disaggregated prefill/decode clusters plus a distributed KV cache pool built from spare CPU, DRAM, SSD and NIC resources, with a KV-centric scheduler and prediction-based early rejection under overload.", "key_idea": "Trade storage for compute with a cluster-wide KV cache pool.", "relevance_score": 3, "relevance_rationale": "Production-deployed KV-centric serving; relevant to long agent contexts.", "found_via": "Mooncake KVCache-centric disaggregated architecture Kimi", "cites_or_cited_by": [ "dhanani-2026-dynamo-agentic-inference", "zhong-2024-distserve" ] }, { "id": "raghavan-2024-alto", "title": "Alto: Orchestrating Distributed Compound AI Systems with Nested Ancestry", "authors": [ "Deepti Raghavan", "Keshav Santhanam", "Muhammad Shahir Rahman", "Nayani Modugula", "Luis Gaspar Schroeder", "Maximilien Cura", "Houjun Liu", "Pratiksha Thaker", "Philip Levis", "Matei Zaharia" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2403.04311", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "nested ancestry metadata" ], "cost_reduction_reported": "latency matched or improved by 10-30% vs LangGraph implementations", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Framework that streams partial outputs and exploits parallelism across components of compound AI queries, using nested ancestry metadata to correctly route and aggregate partial results.", "key_idea": "Stream and parallelize compound AI pipelines.", "relevance_score": 3, "relevance_rationale": "Compound AI system execution efficiency.", "found_via": "compound AI systems serving optimization arXiv", "cites_or_cited_by": [ "tan-2024-teola" ] }, { "id": "ruan-2024-observational-scaling", "title": "Observational Scaling Laws and the Predictability of Language Model Performance", "authors": [ "Yangjun Ruan", "Chris J. Maddison", "Tatsunori Hashimoto" ], "year": 2024, "venue_or_source": "NeurIPS 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2405.10938", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "benchmark scores of ~100 public models" ], "cost_reduction_reported": null, "benchmarks": [ "standard benchmarks", "agent benchmarks" ], "models_studied": [ "~100 public LLMs" ], "production_evidence": false, "summary": "Builds scaling laws from about 100 public models via a low-dimensional capability space, predicting emergent behavior and agent performance from simpler benchmarks.", "key_idea": "Observational scaling laws for predicting LLM performance.", "relevance_score": 3, "relevance_rationale": "Predicting agentic performance from cheap benchmarks helps model choice.", "found_via": "OpenAlex title: observational scaling laws", "cites_or_cited_by": [ "snell-2024-predicting-emergent" ] }, { "id": "schnabel-2024-sammo", "title": "Symbolic Prompt Program Search: A Structure-Aware Approach to Efficient Compile-Time Prompt Optimization", "authors": [ "Tobias Schnabel", "Jennifer Neville" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.02319", "code_url": "https://github.com/microsoft/sammo", "topic_clusters": [ "C4", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task metric", "prompt length" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "SAMMO represents prompt programs symbolically and searches structural transformations at compile time, improving instruction tuning, RAG pipeline tuning and prompt compression across LLMs; supports multiple objectives such as accuracy and efficiency.", "key_idea": "Symbolic structure-aware search over prompt programs.", "relevance_score": 3, "relevance_rationale": "Multi-objective compile-time prompt optimization including compression.", "found_via": "search: SAMMO structure-aware multi-objective metaprompt optimization", "cites_or_cited_by": [] }, { "id": "setlur-2024-rewarding-progress-pav", "title": "Rewarding Progress: Scaling Automated Process Verifiers for LLM Reasoning", "authors": [ "Amrith Setlur", "Chirag Nagpal", "Adam Fisch", "Xinyang Geng", "Jacob Eisenstein", "Rishabh Agarwal", "Alekh Agarwal", "Jonathan Berant", "Aviral Kumar" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.08146", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process advantage", "prover policy" ], "cost_reduction_reported": null, "benchmarks": [ "MATH" ], "models_studied": [ "Gemma" ], "production_evidence": false, "summary": "Argues process rewards should measure progress (step-level advantage under a separate prover policy) and shows such process advantage verifiers improve search and RL efficiency over outcome reward models.", "key_idea": "Process reward = change in success probability under a prover.", "relevance_score": 3, "relevance_rationale": "Theory behind progress-style agent PRMs.", "found_via": "keyword: process reward model", "cites_or_cited_by": [] }, { "id": "shahout-2024-lamps", "title": "Fast Inference for Augmented Large Language Models", "authors": [ "Rana Shahout", "Cong Liang", "Shiji Xin", "Qianru Lao", "Yong Cui", "Minlan Yu", "Michael Mitzenmacher" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.18248", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "predicted output length", "predicted API-call duration", "memory footprint" ], "cost_reduction_reported": "end-to-end latency improved 27%-85% and TTFT reduced 4%-96% vs InferCept, with larger gains over vLLM", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "LAMPS predicts output length and tool-call duration, picks a KV handling strategy for each API call, and schedules requests by their predicted total memory footprint over their lifetime.", "key_idea": "Memory-footprint-aware scheduling for tool-augmented requests.", "relevance_score": 3, "relevance_rationale": "Tool-call-aware scheduling.", "found_via": "LLM serving augmented LLM tool call memory-aware scheduling LAMPS", "cites_or_cited_by": [ "abhyankar-2024-infercept", "wang-2025-augserve" ] }, { "id": "shang-2024-agentsquare", "title": "AgentSquare: Automatic LLM Agent Search in Modular Design Space", "authors": [ "Yu Shang", "Yu Li", "Keyu Zhao", "Likai Ma", "Jiahe Liu", "Fengli Xu", "Yong Li" ], "year": 2024, "venue_or_source": "ICLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2410.06153", "code_url": "https://github.com/tsinghua-fib-lab/AgentSquare", "topic_clusters": [ "C3", "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "in-context performance predictor", "task score", "in-context surrogate predictor" ], "cost_reduction_reported": null, "benchmarks": [ "web, embodied, tool-use and game benchmarks (six)" ], "models_studied": [], "production_evidence": false, "summary": "AgentSquare defines a modular agent design space (planning, reasoning, tool use, memory) and searches it through module evolution and recombination, using an in-context surrogate to skip weak designs. Found agents beat hand-crafted ones by 17.2% on average across six benchmarks.", "key_idea": "Search over modular agent designs with a surrogate performance predictor.", "relevance_score": 3, "relevance_rationale": "Agent architecture search; cost is not the main objective but the predictor reduces search cost.", "found_via": "web search: seed | search: AgentSquare modular design space agent search arXiv", "cites_or_cited_by": [ "li-2025-agentswift" ] }, { "id": "shi-2024-triple-bai-prompt", "title": "Efficient Prompt Optimization Through the Lens of Best Arm Identification", "authors": [ "Chengshuai Shi", "Kun Yang", "Zihan Chen", "Jundong Li", "Jing Yang", "Cong Shen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.09723", "code_url": null, "topic_clusters": [ "C4", "C5" ], "routing_granularity": "none", "learning_regime": "bandit-online", "signals_used": [ "LLM response score" ], "cost_reduction_reported": null, "benchmarks": [ "instruction induction tasks", "BBH-style tasks" ], "models_studied": [ "GPT-3.5", "Llama-2" ], "production_evidence": false, "summary": "TRIPLE links prompt selection from a candidate pool to fixed-budget best-arm identification, so the number of LLM evaluations spent picking a prompt is explicitly bounded. Variants also select few-shot examples.", "key_idea": "Prompt selection as fixed-budget best arm identification.", "relevance_score": 3, "relevance_rationale": "Budgeted exploration for configuration choice; transferable to model/prompt config selection.", "found_via": "ti:\"best arm identification\" AND abs:prompt", "cites_or_cited_by": [ "lin-2023-instinct" ] }, { "id": "shi-2024-wildfeedback", "title": "WildFeedback: Aligning LLMs With In-situ User Interactions And Feedback", "authors": [ "Taiwei Shi", "Zhuoer Wang", "Longqi Yang", "Ying-Chun Lin", "Zexue He", "Mengting Wan", "Pei Zhou", "Sujay Kumar Jauhar", "Sihao Chen", "Shan Xia", "Hongfei Zhang", "Jieyu Zhao", "Xiaofeng Xu", "Xia Song", "Jennifer Neville" ], "year": 2024, "venue_or_source": "ACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2408.15549", "code_url": null, "topic_clusters": [ "C5", "C6" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "in-situ user satisfaction signals" ], "cost_reduction_reported": null, "benchmarks": [ "WildChat-derived preference data" ], "models_studied": [], "production_evidence": true, "summary": "Mines in-situ user satisfaction and dissatisfaction signals from real conversations to build preference data for aligning LLMs.", "key_idea": "Implicit feedback in real chat logs as preference labels.", "relevance_score": 3, "relevance_rationale": "Shows how to extract labels from production conversations.", "found_via": "OpenAlex title: WildFeedback", "cites_or_cited_by": [ "zhao-2024-wildchat" ] }, { "id": "sikeridis-2024-pickllm", "title": "PickLLM: Context-Aware RL-Assisted Large Language Model Routing", "authors": [ "Dimitrios Sikeridis", "Dennis Ramdass", "Pranay Pareek" ], "year": 2024, "venue_or_source": "Workshop on Scalable and Efficient AI (per arXiv comments)", "source_type": "paper", "url": "https://arxiv.org/abs/2412.12170", "code_url": null, "topic_clusters": [ "C1", "C5" ], "routing_granularity": "query", "learning_regime": "RL-online", "signals_used": [ "weighted reward (cost, latency, accuracy)", "learning automaton / Q-learning" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "four LLMs" ], "production_evidence": false, "summary": "PickLLM routes queries with lightweight RL (a learning automaton or stateless epsilon-greedy Q-learning) using a customizable reward over cost, latency and accuracy. It converges to a single LLM for a session and improves cost and latency.", "key_idea": "Session-level RL router with customizable reward.", "relevance_score": 3, "relevance_rationale": "Simple online RL routing.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "song-2024-agentbank", "title": "AgentBank: Towards Generalized LLM Agents via Fine-Tuning on 50000+ Interaction Trajectories", "authors": [ "Yifan Song", "Weimin Xiong", "Xiutian Zhao", "Dawei Zhu", "Wenhao Wu", "Ke Wang", "Cheng Li", "Wei Peng", "Sujian Li" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.07706", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "50k+ interaction trajectories" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Collects over 50,000 trajectories across 16 tasks and five skill dimensions and fine-tunes Samoyed models, finding scaling trajectory data improves generalization.", "key_idea": "Scale trajectory tuning data for generalization.", "relevance_score": 3, "relevance_rationale": "Evidence on data scaling for agent SFT.", "found_via": "seed", "cites_or_cited_by": [ "zeng-2023-agenttuning" ] }, { "id": "song-2024-eto-trial-and-error", "title": "Trial and Error: Exploration-Based Trajectory Optimization for LLM Agents", "authors": [ "Yifan Song", "Da Yin", "Xiang Yue", "Jie Huang", "Sujian Li", "Bill Yuchen Lin" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2403.02502", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "failure trajectories", "contrastive DPO" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Exploration-based trajectory optimization: after SFT, the agent explores, collects failed trajectories and learns from success/failure pairs with DPO.", "key_idea": "Learn from agent's own failures via contrastive pairs.", "relevance_score": 3, "relevance_rationale": "Improves small agents with self-collected data.", "found_via": "keyword: fine-tuning small agents", "cites_or_cited_by": [] }, { "id": "soylu-2024-bettertogether", "title": "Fine-Tuning and Prompt Optimization: Two Great Steps that Work Better Together", "authors": [ "Dilara Soylu", "Christopher Potts", "Omar Khattab" ], "year": 2024, "venue_or_source": "EMNLP 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2407.10930", "code_url": null, "topic_clusters": [ "C4", "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "bootstrapped traces", "task metric" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "mistral-7b", "llama-2-7b", "llama-3-8b" ], "production_evidence": false, "summary": "BetterTogether alternates prompt optimization and fine-tuning of LM weights in modular pipelines, outperforming weight-only and prompt-only optimization by up to 60% and 6% on average.", "key_idea": "Alternate prompt optimization and weight fine-tuning for LM programs.", "relevance_score": 3, "relevance_rationale": "Joint prompt/weight optimization supports small-model specialization.", "found_via": "search: BetterTogether fine-tuning and prompt optimization LM programs arXiv", "cites_or_cited_by": [ "khattab-2023-dspy", "ziems-2025-mmgrpo" ] }, { "id": "srivatsa-2024-lessons-routing", "title": "Harnessing the Power of Multiple Minds: Lessons Learned from LLM Routing", "authors": [ "KV Aditya Srivatsa", "Kaushal Kumar Maurya", "Ekaterina Kochmar" ], "year": 2024, "venue_or_source": "Insights from Negative Results in NLP Workshop 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2405.00467", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "query embedding classifier" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies whether each reasoning query can be directed to a single most suitable LLM. Finds that routing shows promise but is not feasible in all scenarios, arguing for more robust approaches.", "key_idea": "Negative-results study on classifier routing for reasoning tasks.", "relevance_score": 3, "relevance_rationale": "Useful caution about router generalization.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "srivatsa-2024-preble", "title": "Preble: Efficient Distributed Prompt Scheduling for LLM Serving", "authors": [ "Vikranth Srivatsa", "Zijian He", "Reyna Abhyankar", "Dongming Li", "Yiying Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.00023", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "prompt prefix sharing", "GPU load" ], "cost_reduction_reported": "1.5x-14.5x average and 2x-10x p99 latency improvement over SGLang (and similarly vs vLLM)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Distributed prompt scheduling that co-optimizes KV prefix reuse and load balancing across GPUs, replicating hot prefixes when needed.", "key_idea": "Prefix-aware distributed scheduling.", "relevance_score": 3, "relevance_rationale": "Prefix reuse at cluster scale for agent/system-prompt-heavy traffic.", "found_via": "Preble prefix-aware scheduling distributed LLM serving", "cites_or_cited_by": [ "zheng-2023-sglang" ] }, { "id": "styles-2024-workbench", "title": "WorkBench: a Benchmark Dataset for Agents in a Realistic Workplace Setting", "authors": [ "Olly Styles", "Sam Miller", "Patricio Cerda-Mardini", "Tanaya Guha", "Victor Sanchez", "Bertie Vidgen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.00823", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "outcome-centric evaluation" ], "cost_reduction_reported": null, "benchmarks": [ "WorkBench (690 tasks, 26 tools, 5 databases)" ], "models_studied": [ "GPT-4", "Llama2-70B", "others" ], "production_evidence": false, "summary": "Workplace sandbox with five databases, 26 tools and 690 tasks such as emailing and scheduling, graded by unique database outcomes. ReAct agents completed between 3% and 43% of tasks and sometimes took harmful wrong actions.", "key_idea": "Outcome-based workplace task evaluation.", "relevance_score": 3, "relevance_rationale": "Enterprise-style benchmark.", "found_via": "web search: WorkBench", "cites_or_cited_by": [] }, { "id": "tan-2024-teola", "title": "Teola: Towards End-to-End Optimization of LLM-based Applications", "authors": [ "Xin Tan", "Yimin Jiang", "Yitao Yang", "Hong Xu" ], "year": 2024, "venue_or_source": "ASPLOS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2407.00326", "code_url": "https://github.com/NetX-lab/Ayo", "topic_clusters": [ "C4", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "primitive-level dataflow graph" ], "cost_reduction_reported": "up to 2.09x speedup over existing systems across popular LLM applications", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Teola represents each query's workflow as a fine-grained dataflow graph of task primitives (spanning LLM, retrieval and other modules) and optimizes parallelism, pipelining and scheduling end to end. Its code lives in the Ayo repository.", "key_idea": "Fine-grained primitive-level orchestration of LLM applications.", "relevance_score": 3, "relevance_rationale": "End-to-end optimization of compound pipelines.", "found_via": "task seed (Ayo/Teola)", "cites_or_cited_by": [ "lin-2024-parrot", "raghavan-2024-alto" ] }, { "id": "trivedi-2024-appworld", "title": "AppWorld: A Controllable World of Apps and People for Benchmarking Interactive Coding Agents", "authors": [ "Harsh Trivedi", "Tushar Khot", "Mareike Hartmann", "Ruskin Manku", "Vinty Dong", "Edward Li", "Shashank Gupta", "Ashish Sabharwal", "Niranjan Balasubramanian" ], "year": 2024, "venue_or_source": "ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2407.18901", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "state-based unit tests" ], "cost_reduction_reported": null, "benchmarks": [ "AppWorld (750 tasks, 9 apps, 457 APIs)" ], "models_studied": [ "GPT-4o", "others" ], "production_evidence": false, "summary": "A controllable world of 9 everyday apps with 457 APIs and about 100 simulated users, and a benchmark of 750 tasks requiring interactive code generation, graded by state-based tests that also detect collateral damage. GPT-4o solves about 49% of normal and 30% of challenge tasks.", "key_idea": "API-rich simulated app world with collateral-damage checks.", "relevance_score": 3, "relevance_rationale": "Tool-use agent benchmark used in agent-efficiency studies.", "found_via": "web search: AppWorld", "cites_or_cited_by": [] }, { "id": "wang-2024-cascade-aware-training", "title": "Cascade-Aware Training of Language Models", "authors": [ "Congchao Wang", "Sean Augenstein", "Keith Rush", "Wittawat Jitkrittum", "Harikrishna Narasimhan", "Ankit Singh Rawat", "Aditya Krishna Menon", "Alec Go" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.00060", "code_url": null, "topic_clusters": [ "C2", "C8" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "small-model confidence" ], "cost_reduction_reported": null, "benchmarks": [ "SuperGLUE", "WMT22", "FLAN2021" ], "models_studied": [], "production_evidence": false, "summary": "Trains the small model of a cascade with awareness of its position and of the large model downstream, rather than training models independently. Improves the overall quality-cost trade-off over 60+ tasks.", "key_idea": "Train the small model for its role in the cascade.", "relevance_score": 3, "relevance_rationale": "Training-side lever for cascades; relevant to specialising cheap agents.", "found_via": "arXiv title search: cascading language models", "cites_or_cited_by": [] }, { "id": "wang-2024-difficulty-adaptive-sc", "title": "Make Every Penny Count: Difficulty-Adaptive Self-Consistency for Cost-Efficient Reasoning", "authors": [ "Xinglin Wang", "Shaoxiong Feng", "Yiwei Li", "Peiwen Yuan", "Yueqi Zhang", "Chuyi Tan", "Boyuan Pan", "Yao Hu", "Kan Li", "Heda Wang" ], "year": 2024, "venue_or_source": "NAACL 2025 Findings", "source_type": "paper", "url": "https://arxiv.org/abs/2408.13457", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "prior and posterior difficulty", "difficulty estimate", "consistency" ], "cost_reduction_reported": "Consistently surpasses ASC and ESC in cost by a significant margin at comparable performance (six benchmarks)", "benchmarks": [ "six reasoning benchmarks" ], "models_studied": [], "production_evidence": false, "summary": "Uses difficulty estimates for batches of queries to allocate self-consistency samples, avoiding repeated sampling on easy questions.", "key_idea": "Difficulty-aware sample allocation.", "relevance_score": 3, "relevance_rationale": "Budget allocation across queries.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking) | hf-papers search: self-consistency cost efficient adaptive", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "wang-2024-learning-from-failure-nat", "title": "Learning From Failure: Integrating Negative Examples when Fine-tuning Large Language Models as Agents", "authors": [ "Renxi Wang", "Haonan Li", "Xudong Han", "Yixuan Zhang", "Timothy Baldwin" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.11651", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "negative trajectories" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows including failed GPT-4 trajectories (with negative-aware prompts) when fine-tuning smaller agents improves results over discarding them.", "key_idea": "Use failed teacher trajectories in agent SFT.", "relevance_score": 3, "relevance_rationale": "Production logs contain many failures; this shows how to use them.", "found_via": "keyword: SFT on agent traces", "cites_or_cited_by": [] }, { "id": "wang-2024-mixture-of-agents", "title": "Mixture-of-Agents Enhances Large Language Model Capabilities", "authors": [ "Junlin Wang", "Jue Wang", "Ben Athiwaratkun", "Ce Zhang", "James Zou" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.04692", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "MoA-Lite reported as more cost-effective than GPT-4o with a 1.8% AlpacaEval 2.0 quality gain (per search summary of the paper)", "benchmarks": [ "AlpacaEval 2.0", "MT-Bench", "FLASK" ], "models_studied": [ "open-source LLMs", "GPT-4 Omni (baseline)" ], "production_evidence": false, "summary": "Layers of LLM agents each consume the previous layer's outputs and an aggregator synthesizes the final response; an all-open-source MoA reaches 65.1% on AlpacaEval 2.0 versus 57.5% for GPT-4 Omni. A lighter MoA-Lite configuration trades some quality for lower cost.", "key_idea": "Stack proposer LLMs in layers with an aggregator to exceed a single frontier model.", "relevance_score": 3, "relevance_rationale": "Heterogeneous multi-model composition; baseline for cost-aware MoA variants.", "found_via": "web search: Mixture-of-Agents cost", "cites_or_cited_by": [] }, { "id": "wang-2024-uber-genai-gateway", "title": "Navigating the LLM Landscape: Uber's Innovation with GenAI Gateway", "authors": [ "Tse-Chi Wang", "Roopansh Bansal" ], "year": 2024, "venue_or_source": "Uber engineering blog", "source_type": "blog", "url": "https://www.uber.com/blog/genai-gateway/", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "OpenAI", "Vertex AI", "Uber-hosted models" ], "production_evidence": true, "summary": "Uber's internal OpenAI-compatible gateway unifies external and self-hosted models with PII redaction, auth, audit logs for cost attribution and quality evaluation; about 30 teams and 16M queries per month at the time.", "key_idea": "Central gateway for multi-vendor access and cost attribution.", "relevance_score": 3, "relevance_rationale": "Production gateway at scale; foundation for routing.", "found_via": "seed list: Uber case study", "cites_or_cited_by": [] }, { "id": "wu-2024-inference-scaling-laws", "title": "Inference Scaling Laws: An Empirical Analysis of Compute-Optimal Inference for Problem-Solving with Language Models", "authors": [ "Yangzhen Wu", "Zhiqing Sun", "Shanda Li", "Sean Welleck", "Yiming Yang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.00724", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "inference strategy", "model size", "compute budget" ], "cost_reduction_reported": "Llemma-7B with the proposed tree search consistently outperforms Llemma-34B across tested inference strategies on MATH", "benchmarks": [ "MATH" ], "models_studied": [ "Llemma-7B", "Llemma-34B" ], "production_evidence": false, "summary": "Empirical study of compute-optimal inference across model sizes and strategies (greedy, voting, best-of-n, tree search), finding smaller models with better inference algorithms are often Pareto-optimal.", "key_idea": "Small model plus smart inference can be compute-optimal.", "relevance_score": 3, "relevance_rationale": "Guides size vs test-time compute choices.", "found_via": "snowball: related work of snell-2024-test-time-compute-optimal", "cites_or_cited_by": [ "snell-2024-test-time-compute-optimal" ] }, { "id": "wu-2024-ocean", "title": "OCEAN: Offline Chain-of-thought Evaluation and Alignment in Large Language Models", "authors": [ "Junda Wu", "Xintong Li", "Ruoyu Wang", "Yu Xia", "Yuxin Xiong", "Jianing Wang", "Tong Yu", "Xiang Chen", "Branislav Kveton", "Lina Yao", "Jingbo Shang", "Julian McAuley" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.23703", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "knowledge-graph feedback", "inverse propensity scores" ], "cost_reduction_reported": null, "benchmarks": [ "Wikidata5m-based evaluation" ], "models_studied": [], "production_evidence": false, "summary": "OCEAN performs offline evaluation of chain-of-thought using knowledge-graph feedback and a KG-IPS estimator with unbiasedness guarantees, then optimizes the LLM off-policy.", "key_idea": "IPS-based offline evaluation of LLM reasoning paths.", "relevance_score": 3, "relevance_rationale": "Shows IPS/OPE adapted to LLM outputs.", "found_via": "snowball: cited-by bhargava-2024-ope-logged-human-feedback", "cites_or_cited_by": [ "bhargava-2024-ope-logged-human-feedback" ] }, { "id": "xi-2024-agentgym", "title": "AgentGym: Evolving Large Language Model-based Agents across Diverse Environments", "authors": [ "Zhiheng Xi", "Yiwen Ding", "Wenxiang Chen", "Boyang Hong", "Honglin Guo", "Junzhe Wang", "Dingwen Yang", "Chenyang Liao", "Xin Guo", "Wei He", "Songyang Gao", "Lu Chen", "Rui Zheng", "Yicheng Zou", "Tao Gui", "Qi Zhang", "Xipeng Qiu", "Xuanjing Huang", "Zuxuan Wu", "Yu-Gang Jiang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.04151", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "environment feedback", "trajectory set" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Provides a framework of diverse environments, a trajectory set (AgentTraj) and AgentEvol, a self-evolution method letting agents improve beyond imitation across environments.", "key_idea": "Environments plus self-evolution for generalist agents.", "relevance_score": 3, "relevance_rationale": "Infrastructure for training small agents.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "xiong-2024-ipr-step-level-refinement", "title": "Watch Every Step! LLM Agent Learning via Iterative Step-Level Process Refinement", "authors": [ "Weimin Xiong", "Yifan Song", "Xiutian Zhao", "Wenhao Wu", "Xun Wang", "Ke Wang", "Cheng Li", "Wei Peng", "Sujian Li" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.11176", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "step", "learning_regime": "hybrid", "signals_used": [ "step-level Monte Carlo rewards" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Iterative step-level process refinement: step rewards estimated via Monte Carlo guide contrastive action pairs for agent training.", "key_idea": "Step-level rewards to refine agent training.", "relevance_score": 3, "relevance_rationale": "Links step signals to specialization.", "found_via": "keyword: step-level reward", "cites_or_cited_by": [] }, { "id": "xu-2024-conveyor", "title": "Conveyor: Efficient Tool-aware LLM Serving with Tool Partial Execution", "authors": [ "Yechen Xu", "Xinhao Kong", "Tingjun Chen", "Danyang Zhuo" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.00059", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "partial tool execution hints" ], "cost_reduction_reported": "tool partial execution improves request completion latency by up to 38.8%", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Lets tools start executing on partial LLM output during decoding via an interface for tool developers and a scheduler that overlaps tool work with generation.", "key_idea": "Start tool execution before the LLM finishes emitting the call.", "relevance_score": 3, "relevance_rationale": "Tool-call-aware serving reduces agent latency.", "found_via": "task seed", "cites_or_cited_by": [ "abhyankar-2024-infercept", "biswas-2026-sutradhara", "sui-2026-paste" ] }, { "id": "yao-2024-cacheblend", "title": "CacheBlend: Fast Large Language Model Serving for RAG with Cached Knowledge Fusion", "authors": [ "Jiayi Yao", "Hanchen Li", "Yuhan Liu", "Siddhant Ray", "Yihua Cheng", "Qizheng Zhang", "Kuntai Du", "Shan Lu", "Junchen Jiang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.16444", "code_url": "https://github.com/LMCache/LMCache", "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "KV deviation of tokens" ], "cost_reduction_reported": "time-to-first-token reduced 2.2-3.3x and throughput increased 2.8-5x vs full KV recompute without compromising quality", "benchmarks": [ "four RAG benchmark datasets" ], "models_studied": [ "three open-source LLMs" ], "production_evidence": false, "summary": "For RAG inputs made of several precomputed chunks, CacheBlend reuses each chunk's KV cache and recomputes only a small subset of tokens whose KV values deviate most, restoring cross-chunk attention cheaply and overlapping recompute with KV loading.", "key_idea": "Selective KV recompute makes non-prefix KV cache reuse accurate.", "relevance_score": 3, "relevance_rationale": "KV reuse for retrieval/tool-context chunks that recur across agent calls.", "found_via": "CacheBlend fast LLM serving RAG cached knowledge fusion", "cites_or_cited_by": [ "gim-2023-prompt-cache", "jin-2024-ragcache", "liu-2023-cachegen" ] }, { "id": "yuan-2024-evoagent", "title": "EvoAgent: Towards Automatic Multi-Agent Generation via Evolutionary Algorithms", "authors": [ "Siyu Yuan", "Kaitao Song", "Jiangjie Chen", "Xu Tan", "Dongsheng Li", "Deqing Yang" ], "year": 2024, "venue_or_source": "NAACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2406.14228", "code_url": "https://github.com/siyuyuan/evoagent", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "EvoAgent treats an existing agent framework as an initial individual and applies evolutionary operators (mutation, crossover, selection) to generate diverse expert agents, extending single agents into multi-agent systems without manual design.", "key_idea": "Evolutionary expansion of a single agent into a multi-agent system.", "relevance_score": 3, "relevance_rationale": "Automated multi-agent generation; quality-focused.", "found_via": "seed list: EvoAgent", "cites_or_cited_by": [] }, { "id": "yuksekgonul-2024-textgrad", "title": "TextGrad: Automatic \"Differentiation\" via Text", "authors": [ "Mert Yuksekgonul", "Federico Bianchi", "Joseph Boen", "Sheng Liu", "Zhi Huang", "Carlos Guestrin", "James Zou" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.07496", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "LLM textual feedback (textual gradients)" ], "cost_reduction_reported": null, "benchmarks": [ "GPQA", "LeetCode-Hard" ], "models_studied": [ "GPT-4o" ], "production_evidence": false, "summary": "TextGrad backpropagates natural-language feedback from LLMs through a computation graph to improve components of a compound AI system, with a PyTorch-like API. Demonstrated on QA, code, prompt optimization, molecule design and treatment planning, e.g. raising GPT-4o zero-shot GPQA from 51% to 55%.", "key_idea": "Automatic 'differentiation' via textual feedback for optimizing variables in compound AI graphs.", "relevance_score": 3, "relevance_rationale": "Widely used generic optimizer for compound systems; quality-focused, no cost objective.", "found_via": "seed: TextGrad", "cites_or_cited_by": [ "wang-2025-evoagentx", "zhang-2026-po-coin-flip" ] }, { "id": "zaharia-2024-compound-ai-systems", "title": "The Shift from Models to Compound AI Systems", "authors": [ "Matei Zaharia", "Omar Khattab", "Lingjiao Chen", "Jared Quincy Davis", "Heather Miller", "Chris Potts", "James Zou", "Michael Carbin", "Jonathan Frankle", "Naveen Rao", "Ali Ghodsi" ], "year": 2024, "venue_or_source": "BAIR blog", "source_type": "blog", "url": "https://bair.berkeley.edu/blog/2024/02/18/compound-ai-systems/", "code_url": null, "topic_clusters": [ "C4", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Position post arguing that state-of-the-art results increasingly come from systems of multiple interacting components rather than single models. Identifies co-optimizing non-differentiable components (e.g., via DSPy) and cost-quality management (e.g., FrugalGPT-style routing) as core open problems.", "key_idea": "Defines compound AI systems and frames their end-to-end optimization and cost control as a research agenda.", "relevance_score": 3, "relevance_rationale": "Framing document for the C4 cluster; motivates per-component optimization and cost-aware design.", "found_via": "seed: The Shift from Models to Compound AI Systems (BAIR blog)", "cites_or_cited_by": [ "khattab-2023-dspy" ] }, { "id": "zhang-2024-collaborative-performance-prediction", "title": "Collaborative Performance Prediction for Large Language Models", "authors": [ "Qiyuan Zhang", "Fuyuan Lyu", "Xue Liu", "Chen Ma" ], "year": 2024, "venue_or_source": "EMNLP 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2407.01300", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "historical model-task scores", "design factors" ], "cost_reduction_reported": null, "benchmarks": [ "collected collaborative performance data" ], "models_studied": [], "production_evidence": false, "summary": "Collaborative Performance Prediction treats model-by-task scores like a recommender matrix, using historical results and design factors to predict LLM downstream performance better than scaling laws.", "key_idea": "Collaborative filtering for LLM performance prediction.", "relevance_score": 3, "relevance_rationale": "Recsys-style prediction of which model will do well.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "ye-2023-predictable-bigbench", "zhao-2024-predict-lvlm" ] }, { "id": "zhang-2024-genrm-generative-verifiers", "title": "Generative Verifiers: Reward Modeling as Next-Token Prediction", "authors": [ "Lunjun Zhang", "Arian Hosseini", "Hritik Bansal", "Mehran Kazemi", "Aviral Kumar", "Rishabh Agarwal" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.15240", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "generative verifier", "CoT verification" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K", "algorithmic tasks" ], "models_studied": [ "Gemma" ], "production_evidence": false, "summary": "Trains verifiers with next-token prediction jointly on verification and generation, allowing chain-of-thought verification and majority voting over verifications; outperforms discriminative verifiers and LLM-as-a-Judge with 16-64% gains in best-of-N problems solved.", "key_idea": "Reward modeling as next-token prediction.", "relevance_score": 3, "relevance_rationale": "Verifier design choice with a tunable compute budget.", "found_via": "keyword: verifier", "cites_or_cited_by": [] }, { "id": "zhang-2024-p3defer", "title": "Privacy-preserved LLM Cascade via CoT-enhanced Policy Learning", "authors": [ "Kai Zhang", "Congchao Wang", "Liqian Peng", "Alec Go", "Xiaozhong Liu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.08014", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "RL-offline", "signals_used": [ "CoT-enhanced policy", "privacy risk" ], "cost_reduction_reported": null, "benchmarks": [ "three benchmark datasets" ], "models_studied": [], "production_evidence": false, "summary": "Proposes a chain-of-thought-enhanced policy for deciding when an on-device LLM should defer to a server LLM, accounting for privacy leakage as well as cost and quality.", "key_idea": "Privacy-aware deferral policy for local-to-cloud cascades.", "relevance_score": 3, "relevance_rationale": "Cascade deferral with extra enterprise constraint (privacy).", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "zhang-2024-xlam", "title": "xLAM: A Family of Large Action Models to Empower AI Agent Systems", "authors": [ "Jianguo Zhang", "Tian Lan", "Ming Zhu", "Zuxin Liu", "Thai Hoang", "Shirley Kokane", "Weiran Yao", "Juntao Tan", "Akshara Prabhakar", "Haolin Chen", "Zhiwei Liu", "Yihao Feng", "Tulika Awalgaonkar", "Rithesh Murthy", "Eric Hu", "Zeyuan Chen", "Ran Xu", "Juan Carlos Niebles", "Shelby Heinecke", "Huan Wang", "Silvio Savarese", "Caiming Xiong" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2409.03215", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "unified augmented agent data" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "xLAM 1B-8x22B" ], "production_evidence": false, "summary": "Releases xLAM large action models (1B to 8x22B) trained with a unified pipeline for augmenting and synthesizing agent data, with strong function-calling results.", "key_idea": "Open family of action models for agents.", "relevance_score": 3, "relevance_rationale": "Candidate small agent backbones.", "found_via": "seed", "cites_or_cited_by": [ "liu-2024-apigen" ] }, { "id": "zhao-2024-saup", "title": "SAUP: Situation Awareness Uncertainty Propagation on LLM Agent", "authors": [ "Q. C. Zhao", "Xujiang Zhao", "Yanchi Liu", "Wei Cheng", "Yiyou Sun", "Mika Oishi", "Takao Osaki", "Katsushi Matsuda", "Huaxiu Yao", "Haifeng Chen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.01033", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "step uncertainty propagation", "situational weights" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Propagates uncertainty across every step of an agent's decision process with situation-aware weights, improving AUROC by up to 20% over final-step estimates.", "key_idea": "Aggregate per-step uncertainty over agent trajectories.", "relevance_score": 3, "relevance_rationale": "Trajectory-level uncertainty for escalation.", "found_via": "keyword: uncertainty LLM agents", "cites_or_cited_by": [] }, { "id": "zhao-2024-wildchat", "title": "WildChat: 1M ChatGPT Interaction Logs in the Wild", "authors": [ "Wenting Zhao", "Xiang Ren", "Jack Hessel", "Claire Cardie", "Yejin Choi", "Yuntian Deng" ], "year": 2024, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2405.01470", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "real user conversations" ], "cost_reduction_reported": null, "benchmarks": [ "WildChat" ], "models_studied": [ "ChatGPT", "GPT-4" ], "production_evidence": true, "summary": "WildChat is a corpus of 1M opt-in user-ChatGPT conversations (over 2.5M turns) with timestamps and demographic metadata.", "key_idea": "Real-world ChatGPT interaction logs.", "relevance_score": 3, "relevance_rationale": "Public production-like logs for router training and analysis.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "li-2024-arena-hard", "shi-2024-wildfeedback", "zheng-2023-lmsys-chat-1m" ] }, { "id": "zhuge-2024-gptswarm", "title": "Language Agents as Optimizable Graphs", "authors": [ "Mingchen Zhuge", "Wenyi Wang", "Louis Kirsch", "Francesco Faccio", "Dmitrii Khizbullin", "Jürgen Schmidhuber" ], "year": 2024, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2402.16823", "code_url": "https://github.com/metauto-ai/gptswarm", "topic_clusters": [ "C3", "C4", "C12" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "task utility" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "GPTSwarm represents agents as computational graphs whose nodes query LLMs and whose edges carry information, and optimizes both node prompts and edge connectivity automatically. Edge optimization prunes or adds inter-agent links, making orchestration itself learnable.", "key_idea": "Treat multi-agent systems as optimizable graphs and learn which edges to keep.", "relevance_score": 3, "relevance_rationale": "Foundation for topology/pruning work (AgentPrune, G-Designer); not cost-driven itself.", "found_via": "arXiv title search: Language Agents as Optimizable Graphs | seed: GPTSwarm", "cites_or_cited_by": [] }, { "id": "aggarwal-2023-adaptive-consistency", "title": "Let's Sample Step by Step: Adaptive-Consistency for Efficient Reasoning and Coding with LLMs", "authors": [ "Pranjal Aggarwal", "Aman Madaan", "Yiming Yang", "Mausam" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.11860", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "sample agreement stopping criterion", "vote agreement stopping rule" ], "cost_reduction_reported": "Reduces sample budget by up to 7.9x with average accuracy drop below 0.1% (17 datasets, three LLMs)", "benchmarks": [ "17 reasoning and code datasets" ], "models_studied": [ "three LLMs" ], "production_evidence": false, "summary": "Adjusts the number of self-consistency samples per question with a lightweight stopping rule based on agreement so far.", "key_idea": "Adaptive number of samples per question.", "relevance_score": 3, "relevance_rationale": "Early foundational adaptive sampling method.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking) | hf-papers search: self-consistency cost efficient adaptive", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "aksitov-2023-rest-meets-react", "title": "ReST meets ReAct: Self-Improvement for Multi-Step Reasoning LLM Agent", "authors": [ "Renat Aksitov", "Sobhan Miryoosefi", "Zonglin Li", "Daliang Li", "Sheila Babayan", "Kavya Kopparapu", "Zachary Fisher", "Ruiqi Guo", "Sushant Prakash", "Pranesh Srinivasan", "Manzil Zaheer", "Felix Yu", "Sanjiv Kumar" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2312.10003", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "AI feedback ranking", "self-distillation" ], "cost_reduction_reported": null, "benchmarks": [ "Bamboogle" ], "models_studied": [], "production_evidence": false, "summary": "Iterative ReST-style self-improvement of a search agent with AI-feedback ranking, then distilling into models two orders of magnitude smaller with comparable performance.", "key_idea": "Self-improve then distill agent into much smaller models.", "relevance_score": 3, "relevance_rationale": "Agent distillation with AI feedback.", "found_via": "keyword: agent distillation", "cites_or_cited_by": [] }, { "id": "angelopoulos-2023-ppi", "title": "Prediction-powered inference", "authors": [ "Anastasios Nikolas Angelopoulos", "Stephen Bates", "Clara Fannjiang", "Michael I. Jordan", "Tijana Zrnic" ], "year": 2023, "venue_or_source": "Science", "source_type": "paper", "url": "https://arxiv.org/abs/2301.09633", "code_url": null, "topic_clusters": [ "C6", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "ML predictions", "small labeled set" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Introduces prediction-powered inference, which combines many model predictions with a small labeled set to produce valid confidence intervals for population quantities.", "key_idea": "Valid inference from predictions plus a small labeled correction set.", "relevance_score": 3, "relevance_rationale": "Foundation for judge-assisted evaluation of routing policies with guarantees.", "found_via": "snowball: related work of xiang-2026-otrope", "cites_or_cited_by": [ "xiang-2026-otrope" ] }, { "id": "angelopoulos-2023-ppi-plus-plus", "title": "PPI++: Efficient Prediction-Powered Inference", "authors": [ "Anastasios Nikolas Angelopoulos", "John C. Duchi", "Tijana Zrnic" ], "year": 2023, "venue_or_source": "Annals of Applied Statistics", "source_type": "paper", "url": "https://arxiv.org/abs/2311.01453", "code_url": null, "topic_clusters": [ "C6", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "ML predictions", "small labeled set" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "PPI++ makes prediction-powered inference computationally light and adapts automatically to prediction quality, so its intervals always improve on labeled-only ones.", "key_idea": "Adaptive, efficient prediction-powered inference.", "relevance_score": 3, "relevance_rationale": "Practical estimator for evaluating routing with judge labels.", "found_via": "snowball: related work of xiang-2026-otrope", "cites_or_cited_by": [ "xiang-2026-otrope" ] }, { "id": "bang-2023-gptcache", "title": "GPTCache: An Open-Source Semantic Cache for LLM Applications Enabling Faster Answers and Cost Savings", "authors": [ "Fu Bang" ], "year": 2023, "venue_or_source": "NLP-OSS Workshop 2023 (ACL Anthology)", "source_type": "paper", "url": "https://aclanthology.org/2023.nlposs-1.24/", "code_url": "https://github.com/zilliztech/GPTCache", "topic_clusters": [ "C10", "C14" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query embedding similarity", "pluggable similarity evaluator" ], "cost_reduction_reported": "response speed improves 2-10x on cache hits", "benchmarks": [], "models_studied": [ "ChatGPT" ], "production_evidence": false, "summary": "Describes GPTCache, an open-source library that intercepts LLM calls, embeds the query, and returns a stored response when a semantically similar query is found in a vector store. It supports multiple embedding models, vector stores and similarity evaluators and integrates with LangChain and LlamaIndex.", "key_idea": "Embedding-similarity cache in front of an LLM API to skip repeated or paraphrased calls.", "relevance_score": 3, "relevance_rationale": "Canonical open-source semantic cache; a baseline cost lever at the query level.", "found_via": "GPTCache semantic cache LLM", "cites_or_cited_by": [ "basu-2026-w5h2-intent-caching", "cheema-2025-tweakllm", "gill-2024-meancache", "iyengar-2025-generative-caching", "regmi-2024-gpt-semantic-cache", "schroeder-2025-vcache" ] }, { "id": "chen-2023-chatgpt-behavior-over-time", "title": "How is ChatGPT's behavior changing over time?", "authors": [ "Lingjiao Chen", "Matei A. Zaharia", "James Y. Zou" ], "year": 2023, "venue_or_source": "Special Issue 5: Grappling With the Generative AI Revolution", "source_type": "paper", "url": "https://arxiv.org/abs/2307.09009", "code_url": null, "topic_clusters": [ "C5", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "math", "code generation", "USMLE", "multi-hop QA", "visual reasoning" ], "models_studied": [ "GPT-3.5", "GPT-4" ], "production_evidence": false, "summary": "Compares March and June 2023 versions of GPT-3.5 and GPT-4 on several tasks and finds large behavior changes, such as GPT-4 prime identification falling from 84% to 51% accuracy. Highlights the need for continuous monitoring of LLM services.", "key_idea": "The same API model can change substantially over months.", "relevance_score": 3, "relevance_rationale": "Canonical evidence for non-stationarity in the arms a router selects among.", "found_via": "snowball: related work of taberner-miller-2026-paretobandit", "cites_or_cited_by": [ "taberner-miller-2026-paretobandit" ] }, { "id": "chen-2023-punica", "title": "Punica: Multi-Tenant LoRA Serving", "authors": [ "Lequn Chen", "Zihao Ye", "Yongji Wu", "Danyang Zhuo", "Luis Ceze", "Arvind Krishnamurthy" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.18547", "code_url": "https://github.com/punica-ai/punica", "topic_clusters": [ "C8", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "batched multi-LoRA kernels" ], "cost_reduction_reported": "12x higher throughput serving multiple LoRA models vs state-of-the-art systems with 2 ms added latency per token", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Multi-tenant LoRA serving with a CUDA kernel that batches operations across different adapters, holding one copy of the base model per GPU.", "key_idea": "Batch across LoRA adapters sharing one base model.", "relevance_score": 3, "relevance_rationale": "Multi-specialist serving economics.", "found_via": "task seed", "cites_or_cited_by": [ "sheng-2023-s-lora" ] }, { "id": "gim-2023-prompt-cache", "title": "Prompt Cache: Modular Attention Reuse for Low-Latency Inference", "authors": [ "In Gim", "Guojun Chen", "Seung-seob Lee", "Nikhil Sarda", "Anurag Khandelwal", "Lin Zhong" ], "year": 2023, "venue_or_source": "MLSys 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2311.04934", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "prompt modules (schema)" ], "cost_reduction_reported": "time-to-first-token improved 8x on GPU and 60x on CPU for long prompts while maintaining output accuracy", "benchmarks": [ "document QA", "recommendation" ], "models_studied": [], "production_evidence": false, "summary": "Prompt Cache precomputes and stores attention states for reusable prompt segments (system messages, templates, documents) declared via a markup schema, reusing them whenever they appear in a prompt, not only as an exact prefix, while keeping positions consistent.", "key_idea": "Modular, position-aware reuse of attention states for recurring prompt segments.", "relevance_score": 3, "relevance_rationale": "Foundational KV reuse beyond exact prefixes; relevant for shared agent context.", "found_via": "Prompt Cache modular attention reuse low-latency inference", "cites_or_cited_by": [ "fang-2026-recache", "gim-2025-pie", "yao-2024-cacheblend" ] }, { "id": "hakhamaneshi-2023-anyscale-finetuning-llama2", "title": "Fine-Tuning Llama-2: A Comprehensive Case Study for Tailoring Models to Unique Applications", "authors": [ "Kourosh Hakhamaneshi", "Rehaan Ahmad" ], "year": 2023, "venue_or_source": "Anyscale blog", "source_type": "blog", "url": "https://www.anyscale.com/blog/fine-tuning-llama-2-a-comprehensive-case-study-for-tailoring-models-to-unique-applications", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "task fine-tuning" ], "cost_reduction_reported": "Fine-tuned Llama-2-13B reaches 98% on ViGGO functional representation and fine-tuned 7B/13B about 90% on SQL generation, outperforming GPT-4; on GSM8K fine-tuned 13B improves 28% to 47% but stays behind GPT-4", "benchmarks": [ "ViGGO", "SQL generation", "GSM8K" ], "models_studied": [ "Llama-2-7B", "Llama-2-13B", "Llama-2-70B-chat", "GPT-4" ], "production_evidence": false, "summary": "Practitioner case study fine-tuning Llama-2 on three tasks: structured-output tasks (functional representation, SQL) see small fine-tuned models beat GPT-4, while math reasoning improves but remains behind GPT-4.", "key_idea": "Specialization wins on structured tasks, less on open reasoning.", "relevance_score": 3, "relevance_rationale": "Practical guide to where small fine-tuned models can replace frontier calls.", "found_via": "known-item fetch: Anyscale blog fine-tuning Llama-2", "cites_or_cited_by": [] }, { "id": "hari-2023-tryage", "title": "Tryage: Real-time, intelligent Routing of User Prompts to Large Language Models", "authors": [ "Surya Narayanan Hari", "Matt Thomson" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2308.11601", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "predicted per-model loss", "user constraint flags (size, recency)" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Tryage uses a language-model 'perceptive router' to predict downstream performance of expert models from a library on each prompt, then combines these predictions with user-specified constraints such as model size or recency. Users can move along a Pareto front between accuracy and secondary objectives.", "key_idea": "Predict per-expert performance on a prompt and optimize a constraint-aware objective over a model library.", "relevance_score": 3, "relevance_rationale": "Early model-library router; less relevant to frontier API cost but introduces the Pareto framing.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "hsieh-2023-distilling-step-by-step", "title": "Distilling Step-by-Step! Outperforming Larger Language Models with Less Training Data and Smaller Model Sizes", "authors": [ "Cheng-Yu Hsieh", "Chun-Liang Li", "Chih-Kuan Yeh", "Hootan Nakhost", "Yasuhisa Fujii", "Alexander Ratner", "Ranjay Krishna", "Chen-Yu Lee", "Tomas Pfister" ], "year": 2023, "venue_or_source": "Findings of ACL 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.02301", "code_url": "https://github.com/google-research/distilling-step-by-step", "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "LLM rationales" ], "cost_reduction_reported": "Fine-tuned 770M T5 outperforms few-shot 540B PaLM using only 80% of available data on a benchmark", "benchmarks": [], "models_studied": [ "T5", "PaLM 540B" ], "production_evidence": false, "summary": "Extracts rationales from an LLM and trains small task-specific models with multi-task label and rationale prediction, beating LLMs with much less training data.", "key_idea": "Use teacher rationales as extra supervision for small task models.", "relevance_score": 3, "relevance_rationale": "Canonical specialization-by-distillation result.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "jiang-2023-llmlingua", "title": "LLMLingua: Compressing Prompts for Accelerated Inference of Large Language Models", "authors": [ "Huiqiang Jiang", "Qianhui Wu", "Chin-Yew Lin", "Yuqing Yang", "Lili Qiu" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2310.05736", "code_url": "https://github.com/microsoft/LLMLingua", "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "small-LM token perplexity", "budget controller" ], "cost_reduction_reported": "up to 20x prompt compression with little performance loss on GSM8K/BBH/ShareGPT/Arxiv-March23", "benchmarks": [ "GSM8K", "BBH", "ShareGPT", "Arxiv-March23" ], "models_studied": [], "production_evidence": false, "summary": "Coarse-to-fine prompt compression that uses a small language model to drop low-information tokens. A budget controller allocates compression across prompt parts (instructions, demonstrations, question), and an iterative token-level algorithm models dependencies between kept tokens; the small model is instruction-tuned to align with the target LLM.", "key_idea": "Use a small LM's perplexity to prune prompt tokens before sending to an expensive black-box LLM.", "relevance_score": 3, "relevance_rationale": "Core prompt-compression technique that directly cuts input-token cost for API-served agents.", "found_via": "LLMLingua prompt compression arXiv", "cites_or_cited_by": [ "jha-2024-prompt-compression-characterization", "jiang-2023-longllmlingua", "li-2023-selective-context", "li-2024-prompt-compression-survey", "pan-2024-llmlingua-2" ] }, { "id": "jiang-2023-longllmlingua", "title": "LongLLMLingua: Accelerating and Enhancing LLMs in Long Context Scenarios via Prompt Compression", "authors": [ "Huiqiang Jiang", "Qianhui Wu", "Xufang Luo", "Dongsheng Li", "Chin-Yew Lin", "Yuqing Yang", "Lili Qiu" ], "year": 2023, "venue_or_source": "ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.06839", "code_url": "https://github.com/microsoft/LLMLingua", "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "question-aware perplexity", "document reordering" ], "cost_reduction_reported": "up to 17.1% accuracy gain over the original prompt with ~4x fewer tokens on NaturalQuestions (GPT-3.5-Turbo); cost savings of $28.5 and $27.4 per 1,000 samples on LongBench and ZeroScrolls", "benchmarks": [ "NaturalQuestions", "LongBench", "ZeroScrolls" ], "models_studied": [ "GPT-3.5-Turbo" ], "production_evidence": false, "summary": "Extends LLMLingua to long-context settings by making compression question-aware, reordering documents to counter position bias, and allocating dynamic compression ratios across documents. It shows compression can both cut cost and improve accuracy in long-context RAG.", "key_idea": "Question-aware compression plus reordering fixes both cost and lost-in-the-middle effects.", "relevance_score": 3, "relevance_rationale": "Long-context compression directly applicable to RAG-heavy and tool-output-heavy agent steps.", "found_via": "LongLLMLingua arXiv", "cites_or_cited_by": [ "jiang-2023-llmlingua", "liu-2023-lost-in-the-middle" ] }, { "id": "jitkrittum-2023-confidence-cascade-deferral", "title": "When Does Confidence-Based Cascade Deferral Suffice?", "authors": [ "Wittawat Jitkrittum", "Neha Gupta", "Aditya Krishna Menon", "Harikrishna Narasimhan", "Ankit Singh Rawat", "Sanjiv Kumar" ], "year": 2023, "venue_or_source": "NeurIPS 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2307.02764", "code_url": null, "topic_clusters": [ "C2", "C15" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "max softmax confidence", "post-hoc deferral features" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Characterises the optimal cascade deferral rule and shows when simple confidence-based deferral fails: specialist downstream models, label noise and distribution shift. Post-hoc learned deferral fixes these cases.", "key_idea": "Theory of when confidence thresholds suffice for cascades.", "relevance_score": 3, "relevance_rationale": "Foundational theory used by LLM cascade papers.", "found_via": "known foundational work (cited by 2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "kuhn-2023-semantic-uncertainty", "title": "Semantic Uncertainty: Linguistic Invariances for Uncertainty Estimation in Natural Language Generation", "authors": [ "Lorenz Kuhn", "Yarin Gal", "Sebastian Farquhar" ], "year": 2023, "venue_or_source": "ICLR 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2302.09664", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "n/a", "learning_regime": "training-free", "signals_used": [ "semantic entropy", "sampled generations" ], "cost_reduction_reported": null, "benchmarks": [ "TriviaQA", "CoQA" ], "models_studied": [ "OPT" ], "production_evidence": false, "summary": "Introduces semantic entropy, which clusters sampled answers by meaning before computing entropy, making uncertainty more predictive of QA accuracy.", "key_idea": "Semantic entropy over meaning clusters.", "relevance_score": 3, "relevance_rationale": "Signal behind semantic-agreement cascades.", "found_via": "snowball: related work of cascade papers (verified on arXiv) | seed", "cites_or_cited_by": [ "kossen-2024-semantic-entropy-probes", "nikitin-2024-kernel-language-entropy" ] }, { "id": "kwon-2023-vllm", "title": "Efficient Memory Management for Large Language Model Serving with PagedAttention", "authors": [ "Woosuk Kwon", "Zhuohan Li", "Siyuan Zhuang", "Ying Sheng", "Lianmin Zheng", "Cody Hao Yu", "Joseph E. Gonzalez", "Hao Zhang", "Ion Stoica" ], "year": 2023, "venue_or_source": "SOSP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2309.06180", "code_url": "https://github.com/vllm-project/vllm", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "KV block tables" ], "cost_reduction_reported": "2-4x throughput at the same latency vs FasterTransformer and Orca", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "PagedAttention stores the KV cache in fixed-size blocks managed like OS virtual memory pages, nearly eliminating fragmentation and enabling KV sharing within and across requests; vLLM builds a high-throughput serving engine on it.", "key_idea": "Page the KV cache to eliminate memory waste and enable sharing.", "relevance_score": 3, "relevance_rationale": "De facto open serving substrate; the baseline for nearly all agent-serving work.", "found_via": "task seed", "cites_or_cited_by": [ "agrawal-2024-sarathi-serve", "zhen-2025-taming-the-titans", "zheng-2023-sglang" ] }, { "id": "lee-2023-orchestrallm", "title": "OrchestraLLM: Efficient Orchestration of Language Models for Dialogue State Tracking", "authors": [ "Chia-Hsuan Lee", "Hao Cheng", "Mari Ostendorf" ], "year": 2023, "venue_or_source": "NAACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2311.09758", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "kNN exemplar retrieval", "majority vote" ], "cost_reduction_reported": "Reduces computational costs by over 50% while improving performance vs LLM only (abstract)", "benchmarks": [ "dialogue state tracking" ], "models_studied": [], "production_evidence": false, "summary": "OrchestraLLM routes dialogue state tracking instances between a small and large LM by retrieving nearest exemplars from pools where each model was reliable and taking a majority vote.", "key_idea": "Exemplar-retrieval SLM/LLM routing.", "relevance_score": 3, "relevance_rationale": "Simple kNN routing in a structured task.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [] }, { "id": "li-2023-selective-context", "title": "Compressing Context to Enhance Inference Efficiency of Large Language Models", "authors": [ "Yucheng Li", "Bo Dong", "Chenghua Lin", "Frank Guerin" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2310.06201", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "self-information from a small LM" ], "cost_reduction_reported": "50% reduction in context cost, 36% lower inference memory and 32% lower inference time, with a 0.023 BERTScore and 0.038 faithfulness drop", "benchmarks": [ "summarisation", "question answering", "response generation" ], "models_studied": [], "production_evidence": false, "summary": "Selective Context prunes lexical units (tokens, phrases, sentences) with low self-information computed by a small causal LM. It is one of the earliest filtering-style prompt compression baselines.", "key_idea": "Drop low self-information spans from the context.", "relevance_score": 3, "relevance_rationale": "Baseline training-free compression technique that later agent-context methods compare against.", "found_via": "prompt compression survey LLM 2024 2025", "cites_or_cited_by": [ "jiang-2023-llmlingua", "li-2024-prompt-compression-survey" ] }, { "id": "lightman-2023-lets-verify-step-by-step", "title": "Let's Verify Step by Step", "authors": [ "Hunter Lightman", "Vineet Kosaraju", "Yura Burda", "Harri Edwards", "Bowen Baker", "Teddy Lee", "Jan Leike", "John Schulman", "Ilya Sutskever", "Karl Cobbe" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2305.20050", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward", "step labels" ], "cost_reduction_reported": null, "benchmarks": [ "MATH" ], "models_studied": [ "GPT-4" ], "production_evidence": false, "summary": "Compares outcome and process supervision for training reward models on math reasoning and finds process-supervised reward models are markedly better at selecting correct solutions. Releases PRM800K step-level human labels and shows active learning improves label efficiency.", "key_idea": "Step-level (process) reward models beat outcome reward models for verification.", "relevance_score": 3, "relevance_rationale": "Canonical PRM paper; the basis for step-level quality signals in agent trajectories.", "found_via": "seed", "cites_or_cited_by": [ "chae-2025-web-shepherd", "choudhury-2025-agentprm-choudhury", "wang-2023-math-shepherd" ] }, { "id": "lin-2023-generating-with-confidence", "title": "Generating with Confidence: Uncertainty Quantification for Black-box Large Language Models", "authors": [ "Zhen Lin", "Shubhendu Trivedi", "Jimeng Sun" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.19187", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "sampling dispersion", "semantic similarity" ], "cost_reduction_reported": null, "benchmarks": [ "CoQA", "TriviaQA", "NQ" ], "models_studied": [ "LLaMA", "OPT", "GPT-3.5" ], "production_evidence": false, "summary": "Studies black-box uncertainty and confidence metrics for NLG based on dispersion among sampled responses, applied to selective generation where low-confidence outputs are rejected or sent for further assessment.", "key_idea": "Black-box UQ for selective generation.", "relevance_score": 3, "relevance_rationale": "Directly applicable to escalation with API-only models.", "found_via": "keyword: uncertainty quantification LLM", "cites_or_cited_by": [] }, { "id": "liu-2023-dylan", "title": "A Dynamic LLM-Powered Agent Network for Task-Oriented Agent Collaboration", "authors": [ "Zijun Liu", "Yanzhe Zhang", "Peng Li", "Yang Liu", "Diyi Yang" ], "year": 2023, "venue_or_source": "COLM 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.02170", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "agent importance score" ], "cost_reduction_reported": null, "benchmarks": [ "code generation", "decision-making", "general reasoning", "arithmetic reasoning", "MMLU" ], "models_studied": [], "production_evidence": false, "summary": "DyLAN picks a team of agents for a task via an unsupervised Agent Importance Score computed on a preliminary trial, then lets the selected agents collaborate over a dynamic communication structure. It reports better results than fixed-team baselines at moderate compute and up to 25% accuracy gains on some MMLU subjects from team selection alone.", "key_idea": "Select which agents participate using contribution scores from a trial run.", "relevance_score": 3, "relevance_rationale": "Agent-selection and pruning precursor to AgentPrune/AgentDropout; cost is secondary.", "found_via": "arXiv title search: DyLAN", "cites_or_cited_by": [] }, { "id": "ma-2023-prompt-getting-worse", "title": "(Why) Is My Prompt Getting Worse? Rethinking Regression Testing for Evolving LLM APIs", "authors": [ "Wanqing Ma", "Chenyang Yang", "Christian Kästner" ], "year": 2023, "venue_or_source": "2024 IEEE/ACM 3rd International Conference on AI Engineering – Software Engineering for AI (CAIN)", "source_type": "paper", "url": "https://arxiv.org/abs/2311.11123", "code_url": null, "topic_clusters": [ "C5", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Case study showing that silent updates and deprecations of LLM APIs cause regressions in downstream prompts (toxicity detection). Argues that regression testing for LLM APIs needs new approaches because of fuzzy correctness, prompt brittleness and non-determinism.", "key_idea": "LLM API updates silently regress downstream behavior, requiring dedicated regression testing.", "relevance_score": 3, "relevance_rationale": "Model-swap drift is a core risk for any learned router or config; motivates continuous re-evaluation.", "found_via": "snowball: related work of taberner-miller-2026-paretobandit", "cites_or_cited_by": [ "taberner-miller-2026-paretobandit" ] }, { "id": "manakul-2023-selfcheckgpt", "title": "SelfCheckGPT: Zero-Resource Black-Box Hallucination Detection for Generative Large Language Models", "authors": [ "Potsawee Manakul", "Adian Liusie", "Mark J. F. Gales" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2303.08896", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "sampling consistency" ], "cost_reduction_reported": null, "benchmarks": [ "WikiBio" ], "models_studied": [ "GPT-3" ], "production_evidence": false, "summary": "Detects hallucinations in black-box LLMs by sampling multiple responses and checking consistency of facts, needing no external database or logits.", "key_idea": "Inconsistency across samples flags hallucination.", "relevance_score": 3, "relevance_rationale": "Widely used zero-resource hallucination signal.", "found_via": "keyword: hallucination detection", "cites_or_cited_by": [] }, { "id": "packer-2023-memgpt", "title": "MemGPT: Towards LLMs as Operating Systems", "authors": [ "Charles Packer", "Sarah Wooders", "Kevin Lin", "Vivian Fang", "Shishir G. Patil", "Ion Stoica", "Joseph E. Gonzalez" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.08560", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "memory tier management", "interrupts" ], "cost_reduction_reported": null, "benchmarks": [ "document analysis", "multi-session chat" ], "models_studied": [], "production_evidence": false, "summary": "MemGPT applies OS-style virtual memory to LLMs: the model moves information between a small in-context working set and external storage via function calls, giving the illusion of unbounded context for long documents and multi-session conversations.", "key_idea": "Virtual context management: page information in and out of the context window.", "relevance_score": 3, "relevance_rationale": "Foundational memory architecture for keeping agent contexts small.", "found_via": "task seed", "cites_or_cited_by": [ "chhikara-2025-mem0" ] }, { "id": "pang-2023-implicit-feedback-deployment-dialogue", "title": "Leveraging Implicit Feedback from Deployment Data in Dialogue", "authors": [ "Richard Yuanzhe Pang", "Stephen Roller", "Kyunghyun Cho", "He He", "Jason Weston" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2307.14117", "code_url": null, "topic_clusters": [ "C6", "C14" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "user response length", "sentiment", "reactions" ], "cost_reduction_reported": null, "benchmarks": [ "BlenderBot deployment data" ], "models_studied": [ "BlenderBot" ], "production_evidence": true, "summary": "Uses publicly released BlenderBot deployment conversations to learn from implicit signals such as the length, sentiment and reactions of the following user turn. Human evaluation shows improvements, but some proxies such as conversation length push the model toward controversial outputs.", "key_idea": "Next-user-turn signals from deployment logs act as implicit quality labels, with proxy-hacking risks.", "relevance_score": 3, "relevance_rationale": "Early production-data study of implicit feedback proxies and their failure modes.", "found_via": "query: \"regenerate\" user dissatisfaction signal", "cites_or_cited_by": [] }, { "id": "patil-2023-gorilla", "title": "Gorilla: Large Language Model Connected with Massive APIs", "authors": [ "Shishir G. Patil", "Tianjun Zhang", "Xin Wang", "Joseph E. Gonzalez" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.15334", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "self-instruct API data", "retriever-aware training" ], "cost_reduction_reported": "Fine-tuned LLaMA surpasses GPT-4 on writing API calls", "benchmarks": [ "APIBench" ], "models_studied": [ "LLaMA-7B", "GPT-4" ], "production_evidence": false, "summary": "Fine-tunes LLaMA on API documentation-derived instructions with retriever-aware training to produce accurate API calls and reduce hallucinated usage, surpassing GPT-4 on APIBench.", "key_idea": "Specialized small model for API calls.", "relevance_score": 3, "relevance_rationale": "Early specialization beating a frontier model.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "qin-2023-toolllm", "title": "ToolLLM: Facilitating Large Language Models to Master 16000+ Real-world APIs", "authors": [ "Yujia Qin", "Shihao Liang", "Yining Ye", "Kunlun Zhu", "Lan Yan", "Yaxi Lu", "Yankai Lin", "Xin Cong", "Xiangru Tang", "Bill Qian", "Sihan Zhao", "Runchu Tian", "Ruobing Xie", "Jie Zhou", "Mark Gerstein", "Dahai Li", "Zhiyuan Liu", "Maosong Sun" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2307.16789", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "ChatGPT-generated tool-use traces", "DFSDT" ], "cost_reduction_reported": null, "benchmarks": [ "ToolBench" ], "models_studied": [ "LLaMA", "ChatGPT" ], "production_evidence": false, "summary": "Constructs ToolBench from 16,464 RapidAPI APIs with ChatGPT-generated instructions and solution paths (via depth-first search decision trees), fine-tuning LLaMA into ToolLLaMA with an API retriever.", "key_idea": "Large synthetic tool-use data enables open tool agents.", "relevance_score": 3, "relevance_rationale": "Canonical tool-use distillation.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "ren-2023-self-eval-selective-generation", "title": "Self-Evaluation Improves Selective Generation in Large Language Models", "authors": [ "Jie Ren", "Yao Zhao", "Tu Vu", "Peter J. Liu", "Balaji Lakshminarayanan" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2312.09300", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "self-evaluation scores", "self-evaluation score", "P(True)" ], "cost_reduction_reported": null, "benchmarks": [ "TruthfulQA", "TL;DR" ], "models_studied": [ "PaLM-2", "GPT-3" ], "production_evidence": false, "summary": "Turns open-ended generation into token-level self-evaluation (multi-way or point-wise, with a none-of-the-above option) to get better-calibrated scores for selective generation.", "key_idea": "Self-evaluation scores beat sequence likelihood for abstention.", "relevance_score": 3, "relevance_rationale": "Signal usable for deciding when to escalate.", "found_via": "arXiv title search: selective generation large language models | keyword: confidence estimation", "cites_or_cited_by": [] }, { "id": "sheng-2023-s-lora", "title": "S-LoRA: Serving Thousands of Concurrent LoRA Adapters", "authors": [ "Ying Sheng", "Shiyi Cao", "Dacheng Li", "Coleman Hooper", "Nicholas Lee", "Shuo Yang", "Christopher Chou", "Banghua Zhu", "Lianmin Zheng", "Kurt Keutzer", "Joseph E. Gonzalez", "Ion Stoica" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.03285", "code_url": "https://github.com/S-LoRA/S-LoRA", "topic_clusters": [ "C8", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "unified paging of adapters and KV" ], "cost_reduction_reported": "up to 4x throughput vs HuggingFace PEFT and vLLM with naive LoRA support, serving orders of magnitude more adapters", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Serves thousands of LoRA adapters on shared base weights with unified paging for adapter weights and KV caches plus custom kernels for heterogeneous batching.", "key_idea": "Serve many fine-tuned specialists cheaply on one base model.", "relevance_score": 3, "relevance_rationale": "Makes per-task specialized small models economical to serve.", "found_via": "task seed", "cites_or_cited_by": [ "chen-2023-punica" ] }, { "id": "sun-2023-datacope", "title": "When is Off-Policy Evaluation (Reward Modeling) Useful in Contextual Bandits? A Data-Centric Perspective", "authors": [ "Hao Sun", "Alex J. Chan", "Nabeel Seedat", "Alihan Hüyük", "Mihaela van der Schaar" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.14110", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "logged dataset" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes DataCOPE, a data-centric framework that forecasts how accurate off-policy evaluation will be on a given logged dataset, identifies subgroups where OPE is unreliable, and compares data-collection strategies.", "key_idea": "Assess whether a logged dataset can support OPE for a target policy before trusting it.", "relevance_score": 3, "relevance_rationale": "Diagnoses when router or config logs are adequate for OPE.", "found_via": "query: abs:\"off-policy evaluation\" AND abs:\"large language model\"", "cites_or_cited_by": [] }, { "id": "tian-2023-just-ask-for-calibration", "title": "Just Ask for Calibration: Strategies for Eliciting Calibrated Confidence Scores from Language Models Fine-Tuned with Human Feedback", "authors": [ "Katherine Tian", "Eric Mitchell", "Allan Zhou", "Archit Sharma", "Rafael Rafailov", "Huaxiu Yao", "Chelsea Finn", "Christopher D. Manning" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.14975", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "n/a", "learning_regime": "training-free", "signals_used": [ "verbalised confidence", "verbalized confidence" ], "cost_reduction_reported": null, "benchmarks": [ "TriviaQA", "SciQ", "TruthfulQA" ], "models_studied": [ "ChatGPT", "GPT-4", "Claude" ], "production_evidence": false, "summary": "For RLHF models, verbalised confidence is often better calibrated than token probabilities, cutting expected calibration error by about 50% relative.", "key_idea": "Verbalised confidence can be the better escalation signal for RLHF models.", "relevance_score": 3, "relevance_rationale": "Signal choice for black-box cascades.", "found_via": "snowball: related work of cascade papers (verified on arXiv) | keyword: verbalized confidence", "cites_or_cited_by": [] }, { "id": "wang-2023-math-shepherd", "title": "Math-Shepherd: Verify and Reinforce LLMs Step-by-step without Human Annotations", "authors": [ "Peiyi Wang", "Lei Li", "Zhihong Shao", "R. X. Xu", "Damai Dai", "Yifei Li", "Deli Chen", "Y. Wu", "Zhifang Sui" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2312.08935", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward", "Monte Carlo rollouts" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K", "MATH" ], "models_studied": [ "Mistral-7B" ], "production_evidence": false, "summary": "Builds a process reward model trained from automatically constructed step labels (via rollouts estimating whether a step leads to the correct answer), avoiding human annotation. Used both for reranking and step-wise PPO; e.g., Mistral-7B rises from 77.9% to 84.1% on GSM8K with PPO and to 89.1% with verification.", "key_idea": "Automatic step labels from completion rollouts train a PRM.", "relevance_score": 3, "relevance_rationale": "Shows PRMs can be built cheaply without human labels, relevant to agent PRMs.", "found_via": "keyword: process reward model", "cites_or_cited_by": [ "lightman-2023-lets-verify-step-by-step" ] }, { "id": "xiong-2023-llms-express-uncertainty", "title": "Can LLMs Express Their Uncertainty? An Empirical Evaluation of Confidence Elicitation in LLMs", "authors": [ "Miao Xiong", "Zhiyuan Hu", "Xinyang Lu", "Yifei Li", "Jie Fu", "Junxian He", "Bryan Hooi" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2306.13063", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "n/a", "learning_regime": "training-free", "signals_used": [ "verbalised confidence", "sampling consistency", "verbalized confidence" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-4", "LLaMA 2 Chat" ], "production_evidence": false, "summary": "Benchmarks black-box confidence elicitation (prompting, sampling, aggregation) for calibration and failure prediction; finds verbal overconfidence and that consistency across samples helps.", "key_idea": "Black-box confidence elicitation benchmark.", "relevance_score": 3, "relevance_rationale": "Background for choosing escalation signals.", "found_via": "snowball: related work of cascade papers (verified on arXiv) | keyword: confidence estimation", "cites_or_cited_by": [] }, { "id": "xu-2023-recomp", "title": "RECOMP: Improving Retrieval-Augmented LMs with Compression and Selective Augmentation", "authors": [ "Fangyuan Xu", "Weijia Shi", "Eunsol Choi" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.04408", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "extractive/abstractive compressor", "selective augmentation" ], "cost_reduction_reported": "compression rate as low as 6% with minimal loss in performance on language modeling and open-domain QA", "benchmarks": [ "language modeling", "open-domain QA" ], "models_studied": [], "production_evidence": false, "summary": "Trains extractive and abstractive compressors that turn retrieved documents into short summaries before they are placed in context; the compressor can output nothing when retrieval is unhelpful. This lowers RAG inference cost while often keeping accuracy.", "key_idea": "Compress retrieved documents into query-focused summaries, or skip them entirely.", "relevance_score": 3, "relevance_rationale": "Transferable to compressing tool/retrieval outputs inside agent loops.", "found_via": "RECOMP compressing retrieved documents selective augmentation", "cites_or_cited_by": [] }, { "id": "ye-2023-predictable-bigbench", "title": "How Predictable Are Large Language Model Capabilities? A Case Study on BIG-bench", "authors": [ "Qinyuan Ye", "Harvey Yiyun Fu", "Xiang Ren", "Robin Jia" ], "year": 2023, "venue_or_source": "EMNLP 2023 (Findings)", "source_type": "paper", "url": "https://arxiv.org/abs/2305.14947", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "experiment records: model family, size, task, shots" ], "cost_reduction_reported": null, "benchmarks": [ "BIG-bench" ], "models_studied": [], "production_evidence": false, "summary": "Trains an MLP on BIG-bench experiment records to predict LLM performance on new configurations (R^2 > 95% on random splits) and searches for a small informative task subset.", "key_idea": "Performance prediction from past experiment records.", "relevance_score": 3, "relevance_rationale": "Predicting configuration performance from logs of past runs.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "zhang-2024-collaborative-performance-prediction" ] }, { "id": "yin-2023-lumos", "title": "Agent Lumos: Unified and Modular Training for Open-Source Language Agents", "authors": [ "Da Yin", "Faeze Brahman", "Abhilasha Ravichander", "Khyathi Chandu", "Kai-Wei Chang", "Yejin Choi", "Bill Yuchen Lin" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.05657", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "subagent", "learning_regime": "supervised-offline", "signals_used": [ "unified annotations", "modular planning/grounding/execution" ], "cost_reduction_reported": "Lumos surpasses GPT-4/3.5-based agents on complex QA and web tasks", "benchmarks": [], "models_studied": [ "LLaMA-2-7B/13B" ], "production_evidence": false, "summary": "Trains modular open-source agents (planning, grounding, execution modules) on unified annotations converted from existing benchmarks with GPT-4.", "key_idea": "Modular small agents trained on unified data.", "relevance_score": 3, "relevance_rationale": "Module-level specialization.", "found_via": "keyword: fine-tuning small agents", "cites_or_cited_by": [] }, { "id": "zeng-2023-agenttuning", "title": "AgentTuning: Enabling Generalized Agent Abilities for LLMs", "authors": [ "Aohan Zeng", "Mingdao Liu", "Rui Lu", "Bowen Wang", "Xiao Liu", "Yuxiao Dong", "Jie Tang" ], "year": 2023, "venue_or_source": "Findings of ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.12823", "code_url": "https://github.com/THUDM/AgentTuning", "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "GPT-4 interaction trajectories", "general instruction data" ], "cost_reduction_reported": "AgentLM-70B comparable to GPT-3.5-turbo on unseen agent tasks", "benchmarks": [], "models_studied": [ "Llama 2", "GPT-4", "GPT-3.5-turbo" ], "production_evidence": false, "summary": "Builds AgentInstruct, a dataset of filtered GPT-4 interaction trajectories, and mixes it with general instructions to tune Llama 2 into AgentLM, preserving general ability while improving agent skills.", "key_idea": "Hybrid agent-trajectory plus general instruction tuning.", "relevance_score": 3, "relevance_rationale": "Classic trajectory-SFT recipe.", "found_via": "seed", "cites_or_cited_by": [ "chen-2024-agent-flan", "song-2024-agentbank" ] }, { "id": "zheng-2023-lmsys-chat-1m", "title": "LMSYS-Chat-1M: A Large-Scale Real-World LLM Conversation Dataset", "authors": [ "Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng", "Tianle Li", "Siyuan Zhuang", "Zhanghao Wu", "Yonghao Zhuang", "Zhuohan Li", "Lin Zi", "Eric P. Xing", "Joseph E. Gonzalez", "Ion Gabriel Stoica", "Hao Zhang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2309.11998", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "real user conversations" ], "cost_reduction_reported": null, "benchmarks": [ "LMSYS-Chat-1M" ], "models_studied": [ "25 LLMs" ], "production_evidence": true, "summary": "Releases one million real conversations with 25 LLMs collected from the Vicuna demo and Chatbot Arena, with use cases such as moderation models and benchmark building.", "key_idea": "Large real-world LLM conversation log dataset.", "relevance_score": 3, "relevance_rationale": "Public proxy for production traffic to train/evaluate routers.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "chiang-2024-chatbot-arena", "zhao-2024-wildchat" ] }, { "id": "zheng-2023-mt-bench", "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "authors": [ "Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng", "Siyuan Zhuang", "Zhanghao Wu", "Yonghao Zhuang", "Lin Zi", "Zhuohan Li", "Dacheng Li", "Eric P. Xing", "Hao Zhang", "Joseph E. Gonzalez", "Ion Gabriel Stoica", "Zi Lin", "Eric. P Xing", "Ion Stoica" ], "year": 2023, "venue_or_source": "NeurIPS 2023 (Datasets and Benchmarks)", "source_type": "paper", "url": "https://arxiv.org/abs/2306.05685", "code_url": "https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge", "topic_clusters": [ "C5", "C7", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "LLM judge", "human preference", "LLM judge score" ], "cost_reduction_reported": null, "benchmarks": [ "MT-Bench", "Chatbot Arena" ], "models_studied": [ "GPT-4", "LLaMA", "Vicuna" ], "production_evidence": false, "summary": "Shows strong LLM judges (GPT-4) agree with human preferences over 80% of the time, similar to human-human agreement, using MT-Bench and early Chatbot Arena data; releases votes and conversations.", "key_idea": "LLM-as-a-judge validated against crowdsourced preferences.", "relevance_score": 3, "relevance_rationale": "Judge signal and Arena data underpin router training and online evaluation.", "found_via": "OpenAlex title: Chatbot Arena | keyword: LLM-as-a-judge", "cites_or_cited_by": [ "chiang-2024-chatbot-arena", "zhuge-2024-agent-as-a-judge" ] }, { "id": "zhu-2023-judgelm", "title": "JudgeLM: Fine-tuned Large Language Models are Scalable Judges", "authors": [ "Lianghui Zhu", "Xinggang Wang", "Xinlong Wang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.17631", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "judge score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Fine-tunes LLMs (7B-33B) on GPT-4 judgments to create scalable judges, addressing position, knowledge and format biases with swap augmentation and reference support/drop.", "key_idea": "Distill GPT-4 judging into smaller judge models.", "relevance_score": 3, "relevance_rationale": "Cheap judge alternative; relevant to C8 as distillation of a judge.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "kadavath-2022-lms-know-what-they-know", "title": "Language Models (Mostly) Know What They Know", "authors": [ "Saurav Kadavath", "Tom Conerly", "Amanda Askell", "Tom Henighan", "Dawn Drain", "Ethan Perez", "Nicholas Schiefer", "Zac Hatfield-Dodds", "Nova DasSarma", "Eli Tran-Johnson", "Scott Johnston", "Sheer El-Showk", "Andy Jones", "Nelson Elhage", "Tristan Hume", "et al.", "Anna Chen", "Yuntao Bai", "Sam Bowman", "Stanislav Fort", "Deep Ganguli", "Danny Hernandez", "Josh Jacobson", "Jackson Kernion", "Shauna Kravec", "Liane Lovitt", "Kamal Ndousse", "Catherine Olsson", "Sam Ringer", "Dario Amodei", "Tom Brown", "Jack Clark", "Nicholas Joseph", "Ben Mann", "Sam McCandlish", "Chris Olah", "Jared Kaplan" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2207.05221", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "P(True)", "P(IK)", "self-evaluation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Anthropic LMs up to 52B" ], "production_evidence": false, "summary": "Shows large models are fairly calibrated on multiple-choice formats and can self-evaluate answers via P(True); also trains P(IK) to predict whether the model knows the answer before answering.", "key_idea": "Self-evaluation and \"I know\" probability as confidence signals.", "relevance_score": 3, "relevance_rationale": "Foundational for confidence-based escalation.", "found_via": "snowball: related work of cascade papers (verified on arXiv) | seed (P(True))", "cites_or_cited_by": [] }, { "id": "leviathan-2022-speculative-decoding", "title": "Fast Inference from Transformers via Speculative Decoding", "authors": [ "Yaniv Leviathan", "Matan Kalman", "Yossi Matias" ], "year": 2022, "venue_or_source": "ICML 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2211.17192", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "draft-model proposals", "target-model verification" ], "cost_reduction_reported": "2x-3x acceleration of T5-XXL vs standard T5X implementation with identical outputs", "benchmarks": [], "models_studied": [ "T5-XXL" ], "production_evidence": false, "summary": "A small draft model proposes several tokens that the large model verifies in parallel, with a sampling scheme that keeps the output distribution exactly unchanged.", "key_idea": "Draft with a small model, verify with the large one, losslessly.", "relevance_score": 3, "relevance_rationale": "Token-level large/small model collaboration; lossless latency/cost reduction.", "found_via": "task seed", "cites_or_cited_by": [ "cai-2024-medusa", "chen-2023-speculative-sampling", "li-2024-eagle", "miao-2023-specinfer", "xia-2024-speculative-decoding-survey" ] }, { "id": "saito-2022-ope-large-action-embeddings", "title": "Off-Policy Evaluation for Large Action Spaces via Embeddings", "authors": [ "Yuta Saito", "Thorsten Joachims" ], "year": 2022, "venue_or_source": "ICML 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2202.06317", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "action embeddings", "marginalized importance weights" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes an OPE estimator using marginalized importance weights over action embeddings, which avoids IPS collapse when the action space is huge, as with language outputs.", "key_idea": "Marginalized IPS via action embeddings for large action spaces.", "relevance_score": 3, "relevance_rationale": "Foundational estimator for OPE over many LLMs/prompts/responses.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "dudik-2011-doubly-robust", "kiyohara-2025-prompt-opt-logged-bandit" ] }, { "id": "wang-2022-self-consistency", "title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models", "authors": [ "Xuezhi Wang", "Jason Wei", "Dale Schuurmans", "Quoc Le", "Ed Chi", "Sharan Narang", "Aakanksha Chowdhery", "Denny Zhou" ], "year": 2022, "venue_or_source": "ICLR 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2203.11171", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "answer agreement", "majority vote" ], "cost_reduction_reported": "Accuracy gains reported: GSM8K +17.9%, SVAMP +11.0%, AQuA +12.2% over greedy CoT (cost: multiple samples)", "benchmarks": [ "GSM8K", "SVAMP", "AQuA", "StrategyQA", "ARC-challenge" ], "models_studied": [ "PaLM", "GPT-3", "LaMDA", "UL2" ], "production_evidence": false, "summary": "Samples many chain-of-thought reasoning paths and returns the most frequent final answer instead of greedy decoding. Produces large accuracy gains on arithmetic and commonsense reasoning.", "key_idea": "Majority vote over sampled reasoning paths; agreement doubles as a confidence signal.", "relevance_score": 3, "relevance_rationale": "Agreement rate is a widely used ground-truth-free confidence signal driving cascades.", "found_via": "keyword: self-consistency", "cites_or_cited_by": [] }, { "id": "jiang-2021-enigma-dialog-ope", "title": "Towards Automatic Evaluation of Dialog Systems: A Model-Free Off-Policy Evaluation Approach", "authors": [ "Haoming Jiang", "Bo Dai", "Mengjiao Yang", "Tuo Zhao", "Wei Wei" ], "year": 2021, "venue_or_source": "EMNLP 2021", "source_type": "paper", "url": "https://arxiv.org/abs/2102.10242", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "pre-collected experience data" ], "cost_reduction_reported": null, "benchmarks": [ "dialog evaluation" ], "models_studied": [], "production_evidence": false, "summary": "ENIGMA estimates human evaluation scores for dialog systems via model-free off-policy evaluation using only a small amount of pre-collected experience data.", "key_idea": "OPE to evaluate dialog policies without live humans.", "relevance_score": 3, "relevance_rationale": "Early OPE for language systems.", "found_via": "OpenAlex title search", "cites_or_cited_by": [] }, { "id": "saito-2020-open-bandit-pipeline", "title": "Open Bandit Dataset and Pipeline: Towards Realistic and Reproducible Off-Policy Evaluation", "authors": [ "Yuta Saito", "Shunsuke Aihara", "Megumi Matsutani", "Yusuke Narita" ], "year": 2020, "venue_or_source": "NeurIPS 2021 (Datasets and Benchmarks)", "source_type": "paper", "url": "https://arxiv.org/abs/2008.07146", "code_url": "https://github.com/st-tech/zr-obp", "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "logged bandit feedback" ], "cost_reduction_reported": null, "benchmarks": [ "Open Bandit Dataset (ZOZOTOWN)" ], "models_studied": [], "production_evidence": true, "summary": "Releases a real logged bandit dataset from a large fashion e-commerce platform collected under multiple policies, plus Open Bandit Pipeline, a library implementing OPE estimators and batch bandit algorithms.", "key_idea": "Real multi-policy logged data and a reference OPE library.", "relevance_score": 3, "relevance_rationale": "Tooling reusable for OPE of routing policies.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "dudik-2011-doubly-robust", "swaminathan-2015-snips" ] }, { "id": "anthropic-message-batches", "title": "Batch processing (Message Batches API)", "authors": [ "Anthropic" ], "year": null, "venue_or_source": "Claude Platform docs", "source_type": "docs", "url": "https://docs.anthropic.com/en/docs/build-with-claude/batch-processing", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) 50% cost reduction for asynchronous batches", "benchmarks": [], "models_studied": [ "Claude models" ], "production_evidence": false, "summary": "Asynchronous Messages batches at half price with higher throughput; launched via the Message Batches API announcement (anthropic.com/news/message-batches-api).", "key_idea": "Async batch at 50% price.", "relevance_score": 3, "relevance_rationale": "Seed; batch API cost lever.", "found_via": "seed: provider batch APIs", "cites_or_cited_by": [] }, { "id": "aws-bedrock-model-distillation", "title": "Amazon Bedrock Model Distillation", "authors": [ "Amazon Web Services" ], "year": null, "venue_or_source": "AWS product page", "source_type": "product", "url": "https://aws.amazon.com/bedrock/model-distillation/", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "teacher responses", "production prompts" ], "cost_reduction_reported": "(vendor-reported) Distilled models up to 500% faster and up to 75% less expensive with <2% accuracy loss for use cases like RAG", "benchmarks": [ "RAG (vendor-stated)" ], "models_studied": [ "teacher/student model pairs on Bedrock" ], "production_evidence": false, "summary": "Managed distillation that generates teacher responses on customer prompts and fine-tunes a smaller student model.", "key_idea": "Managed teacher-student distillation.", "relevance_score": 3, "relevance_rationale": "Production-data distillation product.", "found_via": "snowball: related work of aws-2025-bedrock-intelligent-prompt-routing", "cites_or_cited_by": [] }, { "id": "notdiamond-product", "title": "Not Diamond: Model Routing for Coding Agents", "authors": [ "Not Diamond" ], "year": null, "venue_or_source": "Not Diamond website", "source_type": "product", "url": "https://www.notdiamond.ai", "code_url": "https://github.com/Not-Diamond/RoRF", "topic_clusters": [ "C1", "C3", "C14" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "provider", "domain", "reasoning complexity", "sub-agents", "cache", "context" ], "cost_reduction_reported": "(vendor-reported) Claims 5%+ accuracy gains and 20%+ cost savings; Rootly case: +39% average accuracy across benchmarks", "benchmarks": [], "models_studied": [ "multiple frontier models" ], "production_evidence": false, "summary": "Commercial router now positioned for coding agents, predicting the best model per input and integrating with existing gateways and harnesses. Marketing claims and customer quotes (Rootly, Replicated, OpenRouter) are vendor-reported without methodology. Also open-sourced a random-forest router (github.com/Not-Diamond/RoRF).", "key_idea": "Commercial per-call router for agent workloads.", "relevance_score": 3, "relevance_rationale": "Seed; vendor claims only.", "found_via": "seed: Not Diamond", "cites_or_cited_by": [] }, { "id": "openai-batch-api", "title": "Batch API", "authors": [ "OpenAI" ], "year": null, "venue_or_source": "OpenAI API docs", "source_type": "docs", "url": "https://platform.openai.com/docs/guides/batch", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) 50% discount vs synchronous APIs, 24-hour completion window", "benchmarks": [], "models_studied": [ "OpenAI models" ], "production_evidence": false, "summary": "Asynchronous batch processing of request files at half price with a 24-hour window and separate rate limits.", "key_idea": "Trade latency for 50% price.", "relevance_score": 3, "relevance_rationale": "Provider cost lever for offline agent steps and evals.", "found_via": "seed: provider batch APIs", "cites_or_cited_by": [ "openai-cost-optimization", "openai-flex-processing" ] }, { "id": "openai-flex-processing", "title": "Flex processing", "authors": [ "OpenAI" ], "year": null, "venue_or_source": "OpenAI API docs", "source_type": "docs", "url": "https://platform.openai.com/docs/guides/flex-processing", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) Priced at Batch API rates for synchronous-style requests with slower responses and occasional unavailability", "benchmarks": [], "models_studied": [ "selected OpenAI models" ], "production_evidence": false, "summary": "Service tier that bills at batch rates in exchange for slower responses and possible 429 unavailability, aimed at evals and non-urgent workloads.", "key_idea": "Cheaper best-effort service tier.", "relevance_score": 3, "relevance_rationale": "Provider cost lever for latency-tolerant subagent calls.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [ "openai-batch-api", "openai-cost-optimization" ] }, { "id": "openai-prompt-caching", "title": "Prompt caching", "authors": [ "OpenAI" ], "year": null, "venue_or_source": "OpenAI API docs", "source_type": "docs", "url": "https://platform.openai.com/docs/guides/prompt-caching", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "prompt prefix hash", "prompt_cache_key" ], "cost_reduction_reported": "(vendor-reported) Cached input discounted up to 95% (0.1x standard rate for recent models)", "benchmarks": [], "models_studied": [ "OpenAI models" ], "production_evidence": false, "summary": "Automatic prefix caching of KV states for prompts above a minimum length, routed by a hash of the prefix and an optional cache key; the docs note overflow routing above about 15 requests per minute per prefix.", "key_idea": "Automatic prefix KV caching with discounted cached tokens.", "relevance_score": 3, "relevance_rationale": "Key cost lever for agents with long shared prefixes.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [] }, { "id": "amin-2026-bayesian-orchestration-multi-llm", "title": "Bayesian Orchestration of Multi-LLM Agents for Cost-Aware Sequential Decision-Making", "authors": [ "Danial Amin" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.01522", "code_url": null, "topic_clusters": [ "C3", "C7" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "LLM-elicited likelihoods", "value of information", "disagreement" ], "cost_reduction_reported": "Total cost reduced by $294,000 (34%) vs best single-LLM baseline on 1000-resume screening simulation", "benchmarks": [ "resume screening (1000 resumes)" ], "models_studied": [ "GPT-4o", "Claude 4.5 Sonnet", "Gemini Pro", "Grok", "DeepSeek" ], "production_evidence": false, "summary": "Treats several LLMs as likelihood models and combines them with Bayes rule for sequential decisions under asymmetric error costs, choosing actions by expected cost and gathering information by value of information. In a resume-screening simulation it lowers total decision cost by 34%.", "key_idea": "Bayesian aggregation of multiple LLMs for cost-aware sequential decisions.", "relevance_score": 2, "relevance_rationale": "Decision cost rather than inference cost; tangentially relevant orchestration.", "found_via": "arXiv search: single vs multi-agent", "cites_or_cited_by": [] }, { "id": "anthropic-extended-thinking-docs", "title": "Extended thinking (Claude Platform docs)", "authors": [ "Anthropic" ], "year": 2026, "venue_or_source": "Claude Platform documentation", "source_type": "docs", "url": "https://platform.claude.com/docs/en/build-with-claude/extended-thinking", "code_url": null, "topic_clusters": [ "C9", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude 4.x models" ], "production_evidence": false, "summary": "Documents manual extended thinking with a fixed budget_tokens thinking budget, now legacy: deprecated on Claude 4.6 models and rejected on 4.7 and later in favour of adaptive thinking controlled by effort. Useful as a record of how providers moved from explicit per-request thinking budgets to model-adaptive thinking.", "key_idea": "Fixed thinking-token budgets are being replaced by adaptive thinking plus effort levels.", "relevance_score": 2, "relevance_rationale": "Background on reasoning-budget controls available in production.", "found_via": "direct: platform.claude.com docs", "cites_or_cited_by": [ "anthropic-effort-docs" ] }, { "id": "chen-2026-agentic-energy-survey", "title": "Networking-Aware Energy Efficiency in Agentic AI Inference: A Survey", "authors": [ "Xiaojing Chen", "Haiqi Yu", "Wei Ni", "Dusit Niyato", "Ruichen Zhang", "Xin Wang", "Shunqing Zhang", "Shugong Xu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.07857", "code_url": null, "topic_clusters": [ "C11", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of energy efficiency in agentic AI inference with an accounting framework for compute and communication costs across the perception-reasoning-action loop.", "key_idea": "Energy accounting for agentic inference.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "search: efficient agents survey", "cites_or_cited_by": [] }, { "id": "chen-2026-governance-decay", "title": "Governance Decay: How Context Compaction Silently Erases Safety Constraints in Long-Horizon LLM Agents", "authors": [ "Shiyang Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.22528", "code_url": null, "topic_clusters": [ "C7", "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "constraint pinning" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows that context compaction can silently drop safety or policy constraints from agent context (violations up to 38% when dropped), introduces a compaction-eviction attack, and proposes pinning governance constraints outside lossy compaction.", "key_idea": "Compaction can erase policy constraints; pin them.", "relevance_score": 2, "relevance_rationale": "Risk to weigh when adding compaction for cost.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [] }, { "id": "chen-2026-olp-routing-serving", "title": "Online Linear Programming for Multi-Objective Routing in LLM Serving", "authors": [ "Zixi Chen", "Y. Ye", "Zijie Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.03948", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "none", "learning_regime": "bandit-online", "signals_used": [ "SLO-weighted benefit", "KV-cache and batch constraints" ], "cost_reduction_reported": null, "benchmarks": [ "Vidur simulator" ], "models_studied": [], "production_evidence": false, "summary": "Casts dispatch of requests to decode workers as online linear programming with bid-price control and warm-started dual updates, improving latency/throughput SLOs in the Vidur simulator.", "key_idea": "Online LP bid-price control for LLM request dispatch.", "relevance_score": 2, "relevance_rationale": "Serving-layer online optimization; background.", "found_via": "OpenAlex title: online routing LLM", "cites_or_cited_by": [] }, { "id": "cheng-2026-optimas-mas", "title": "OptiMAS: Automatically Optimize Multi-Agent System", "authors": [ "Yuxin Cheng", "Chang Liu", "Hanxin Yu", "Haochen Tan", "Taiqiang Wu", "Weiqiang Jin", "Jie Ran", "Kaibo Wang", "Xiaoguang Li", "Haoli Bai", "Graziano Chesi", "Ngai Wong" ], "year": 2026, "venue_or_source": "EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2608.21918", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "OptiMAS automatically optimizes multi-agent systems over extended horizons and matches or beats domain-specialized hand-crafted systems and evolutionary methods on four agentic benchmarks across three LLM backbones.", "key_idea": "Long-horizon automated MAS optimization.", "relevance_score": 2, "relevance_rationale": "Automated MAS optimization; cost not primary.", "found_via": "search: Automated Design of Agentic Systems ADAS Meta Agent Search arXiv", "cites_or_cited_by": [] }, { "id": "cim-2026-parallel-context-compaction", "title": "Parallel Context Compaction for Long-Horizon LLM Agent Serving", "authors": [ "Musa Cim", "Burak Topcu", "Chita Das", "Mahmut Taylan Kandemir" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.23296", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "block-parallel summarization" ], "cost_reduction_reported": null, "benchmarks": [ "HotpotQA", "LoCoMo" ], "models_studied": [ "8B-120B dense and MoE backbones" ], "production_evidence": false, "summary": "Parallelizes context compaction by summarizing blocks of history concurrently, reducing wall time and giving operators control over summary volume.", "key_idea": "Parallel block-wise compaction for serving.", "relevance_score": 2, "relevance_rationale": "Serving-side compaction efficiency.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [] }, { "id": "ding-2026-calibrate-then-act", "title": "Calibrate-Then-Act: Cost-Aware Exploration in LLM Agents", "authors": [ "Wenxuan Ding", "Nicholas Tomlin", "Greg Durrett" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2602.16699", "code_url": null, "topic_clusters": [ "C9", "C12" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "inferred environment prior" ], "cost_reduction_reported": null, "benchmarks": [ "synthetic task", "retrieval-augmented QA", "file-reading coding task" ], "models_studied": [], "production_evidence": false, "summary": "Calibrate-Then-Act gives agents an inferred prior about latent environment state so they explicitly reason about cost-uncertainty trade-offs, such as when to stop exploring, improving decisions over standard RL training.", "key_idea": "Make cost-benefit trade-offs explicit in agent reasoning.", "relevance_score": 2, "relevance_rationale": "Background on cost-aware agent actions.", "found_via": "snowball: cited-by S2 citations of relevance-5 records", "cites_or_cited_by": [ "chen-2024-optima" ] }, { "id": "ding-2026-conserve", "title": "Observation, Not Prediction: Conversation-Level Disaggregated Scheduling for Agentic Serving", "authors": [ "Jianru Ding", "Ryien Hosseini", "Pouya Mahdi Gholami", "Mingyuan Xiang", "Henry Hoffmann" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2606.01839", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "observed conversation state" ], "cost_reduction_reported": "p95 time-to-first-effective-token reduced 51.08% and energy efficiency improved 7.51% vs per-turn prediction baseline; heterogeneous GPU tiers add 22.75% energy efficiency", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Conversation-level disaggregated scheduling for agentic serving: prefill once on a high-throughput prefiller, transfer KV once, and pin the conversation to one decoder, using observation rather than learned cost prediction.", "key_idea": "Disaggregate at conversation, not turn, granularity.", "relevance_score": 2, "relevance_rationale": "Serving policy for multi-turn agents.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "zhong-2024-distserve" ] }, { "id": "google-adk-agent-model-routing", "title": "Model routing and agent routing (Google Agent Development Kit docs: RoutedLlm, RoutedAgent)", "authors": [ "Google" ], "year": 2026, "venue_or_source": "Agent Development Kit (ADK) documentation", "source_type": "docs", "url": "https://adk.dev/agents/models/routing/", "code_url": "https://github.com/google/adk-js", "topic_clusters": [ "C3", "C14" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "developer-defined router function", "error context" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Gemini Flash", "Gemini Pro" ], "production_evidence": false, "summary": "ADK's experimental RoutedLlm lets an LlmAgent choose a model per request through a routing function (for fallback on error, A/B tests between models, or auto-routing by input complexity), and RoutedAgent does the same across whole agents (https://adk.dev/agents/routing/). Failover re-calls the router only if the chosen model or agent fails before emitting output. These sit alongside ADK's workflow agents (sequential, parallel, loop) and LLM-driven delegation for multi-agent apps.", "key_idea": "First-class per-request model/agent routing hooks inside an agent framework.", "relevance_score": 2, "relevance_rationale": "Framework support for step-level model routing; no evaluation.", "found_via": "direct: adk.dev docs (google.github.io/adk-docs redirects here)", "cites_or_cited_by": [ "openai-agents-sdk-handoffs" ] }, { "id": "google-gemini-thinking-docs", "title": "Gemini thinking (Gemini API docs: thinking levels and budgets)", "authors": [ "Google" ], "year": 2026, "venue_or_source": "Gemini API documentation", "source_type": "docs", "url": "https://ai.google.dev/gemini-api/docs/thinking", "code_url": null, "topic_clusters": [ "C9", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Gemini 3.x Flash", "Gemini 2.5" ], "production_evidence": false, "summary": "Explains that Gemini 3 and 2.5 models think dynamically by default, scaling reasoning to request complexity, and exposes a thinking_level parameter (minimal/low/medium/high depending on model; thinking budgets on 2.5) plus thought summaries and signatures for multi-turn tool use. Thinking tokens are billed as output.", "key_idea": "Per-request thinking level/budget control on Gemini models.", "relevance_score": 2, "relevance_rationale": "Background API control for reasoning-token cost.", "found_via": "direct: ai.google.dev docs", "cites_or_cited_by": [ "openai-reasoning-effort-docs" ] }, { "id": "guo-2026-saga", "title": "SAGA: Workflow-Atomic Scheduling for AI Agent Inference on GPU Clusters", "authors": [ "Dongxin Guo", "Jikun Wu", "Siu Ming Yiu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.00528", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "workflow atomicity" ], "cost_reduction_reported": "GPU memory utilization improved 1.22x with 99.2% SLO attainment under multi-tenant interference, at ~30% lower peak throughput than throughput-optimal batching", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Workflow-atomic scheduling of agent inference on GPU clusters, trading some peak throughput for latency in interactive compound AI deployments.", "key_idea": "Schedule whole workflows atomically for latency.", "relevance_score": 2, "relevance_rationale": "Compound AI serving tradeoff.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [] }, { "id": "han-2026-rideway", "title": "RideWay: Benchmarking Efficient Task Completion for Tool-Using Language Agents", "authors": [ "Qingnuan Han", "Boli Fang", "Mingzhi Hou", "Claire Liu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.17985", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "Efficiency Utility (excess tool calls and turns)" ], "cost_reduction_reported": null, "benchmarks": [ "RideWay (58 tasks)" ], "models_studied": [ "24 models" ], "production_evidence": false, "summary": "RideWay benchmarks tool-using ride-hailing agents on efficiency, discounting successes for excess tool calls and user turns relative to reference effort, calibrated with human preferences. Excess turns are penalized about twice as much as excess tool calls.", "key_idea": "Success-gated efficiency metric for tool-using agents.", "relevance_score": 2, "relevance_rationale": "Efficiency-aware agent evaluation.", "found_via": "arXiv title search: tool-use efficiency agent", "cites_or_cited_by": [] }, { "id": "lenton-2026-continual-arc", "title": "Why we built Continual-ARC", "authors": [ "Dan Lenton" ], "year": 2026, "venue_or_source": "Unify blog", "source_type": "benchmark", "url": "https://unify.ai/blog/why-we-built-continual-arc", "code_url": null, "topic_clusters": [ "C10", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Pooled over 600 instances: persistent memory changed prime-agent's budget use by +0.4/+0.1/-0.1 points (low/medium/high effort); hermes-agent's memory cost ~6 points at low effort; none beat a bare model with a demo-cache floor (vendor-reported)", "benchmarks": [ "Continual-ARC (ARC-AGI-1/2/3 streams)" ], "models_studied": [ "prime-agent", "openclaw", "hermes-agent" ], "production_evidence": false, "summary": "Introduces a benchmark streaming recurring ARC tasks with fresh inputs, where agents pay budget for extra examples, and compares three open agent harnesses with memory on vs wiped. Memory yielded essentially no savings; a floor that simply caches worked examples matched them, and a one-program-per-task library did far better.", "key_idea": "Agent memory features often fail to reduce cost on recurring work; simple example caches or stored programs do better.", "relevance_score": 2, "relevance_rationale": "Evidence on whether memory/context reuse actually cuts cost; niche benchmark.", "found_via": "direct: unify.ai/blog index", "cites_or_cited_by": [ "lenton-2026-distilling-repeated-work-into-code" ] }, { "id": "li-2026-herasys", "title": "HeraSys: Collaborative Serving of Multiple LLM Workflows via Fine-Grained End-to-End Optimization", "authors": [ "Size Li", "Zhiqing Tang", "Hongrui Liang", "Jianxiong Guo", "Jiong Lou", "Tian Wang", "Weijia Jia" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2607.22578", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "inter- and intra-query priority" ], "cost_reduction_reported": "P99 latency reduced up to 2.17x and serving throughput increased up to 1.85x under strict latency guarantees", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Collaborative serving of multiple LLM workflows with fine-grained end-to-end priority, resource skewing, adaptive batching and pipeline decomposition.", "key_idea": "Jointly serve many workflows with fine-grained priorities.", "relevance_score": 2, "relevance_rationale": "Multi-workflow serving.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [] }, { "id": "li-2026-polar", "title": "POLAR: Online Learning for LoRA Adapter Caching and Routing in Edge LLM Serving", "authors": [ "Shaoang Li", "Jian Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.16583", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "request context", "adapter residency" ], "cost_reduction_reported": null, "benchmarks": [ "15 real LoRA adapters for Qwen2.5-7B" ], "models_studied": [ "Qwen2.5-7B" ], "production_evidence": false, "summary": "Two-timescale online control of which LoRA adapters stay in GPU memory and how requests are routed to adapters, with contextual-bandit regret guarantees.", "key_idea": "Bandit routing plus caching of LoRA adapters.", "relevance_score": 2, "relevance_rationale": "Serving-level online routing background.", "found_via": "snowball: cited-by wei-2025-barp", "cites_or_cited_by": [] }, { "id": "liang-2026-lacache", "title": "LaCache: Robust Semantic Caching for LLM Serving", "authors": [ "Jiacheng Liang", "Yuhui Wang", "Tanqiu Jiang", "Ting Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.01718", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query similarity", "first-k decoded tokens similarity" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Makes semantic caching robust to collision attacks by also checking whether the first few speculatively decoded response tokens match the cached response, which provably blocks adversarial collisions and improves relevance of hits.", "key_idea": "Verify cache hits on early response tokens, not just the query.", "relevance_score": 2, "relevance_rationale": "Robust semantic caching design.", "found_via": "arXiv search: semantic cache LLM", "cites_or_cited_by": [ "zhang-2026-cacheattack" ] }, { "id": "merrill-2026-terminal-bench", "title": "Terminal-Bench: Benchmarking Agents on Hard, Realistic Tasks in Command Line Interfaces", "authors": [ "Mike A. Merrill", "Alexander G. Shaw", "Nicholas Carlini", "Boxuan Li", "Harsh Raj", "Ivan Bercovich", "Lin Shi", "Jeong Yeon Shin", "Thomas Walshe", "E. Kelly Buchanan", "Junhong Shen", "Guanghao Ye", "Haowei Lin", "Jason Poulos", "Maoyu Wang", "Marianna Nezhurina", "Jenia Jitsev", "Di Lu", "Orfeas Menis Mastromichalakis", "Zhiwei Xu", "Zizhao Chen", "Yue Liu", "Robert Zhang", "Leon Liangyu Chen", "Anurag Kashyap", "Jan-Lucas Uslu", "Jeffrey Li", "Jianbo Wu", "Minghao Yan", "Song Bian", "Vedang Sharma", "Ke Sun", "Steven Dillmann", "Akshay Anand", "Andrew Lanpouthakoun", "Bardia Koopah", "Changran Hu", "Etash Guha", "Gabriel H. S. Dreiman", "Jiacheng Zhu", "Karl Krauth", "Li Zhong", "Niklas Muennighoff", "Robert Amanfu", "Shangyin Tan", "Shreyas Pimpalgaonkar", "Tushar Aggarwal", "Xiangning Lin", "Xin Lan", "Xuandong Zhao", "Yiqing Liang", "Yuanli Wang", "Zilong Wang", "Changzhi Zhou", "David Heineman", "Hange Liu", "Harsh Trivedi", "John Yang", "Junhong Lin", "Manish Shetty", "Michael Yang", "Nabil Omi", "Negin Raoof", "Shanda Li", "Terry Yue Zhuo", "Wuwei Lin", "Yiwei Dai", "Yuxin Wang", "Wenhao Chai", "Shang Zhou", "Dariush Wahdany", "Ziyu She", "Jiaming Hu", "Zhikang Dong", "Yuxuan Zhu", "Sasha Cui", "Ahson Saiyed", "Arinbjörn Kolbeinsson", "Jesse Hu", "Christopher Michael Rytting", "Ryan Marten", "Yixin Wang", "Alex Dimakis", "Andy Konwinski", "Ludwig Schmidt" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.11868", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "verification test suites" ], "cost_reduction_reported": null, "benchmarks": [ "Terminal-Bench 2.0 (89 tasks)" ], "models_studied": [ "frontier models and agents" ], "production_evidence": false, "summary": "Curated set of 89 hard command-line tasks, each with its own container environment, human-written solution and tests, spanning software engineering, ML, security, sysadmin and data science. Frontier agents score below 65%.", "key_idea": "Hard, verified terminal tasks for agents.", "relevance_score": 2, "relevance_rationale": "Background agent benchmark.", "found_via": "web search: Terminal-Bench", "cites_or_cited_by": [] }, { "id": "mitra-2026-cascades-under-perturbations", "title": "Accuracy and Robustness of Model Cascades Under Data Perturbations", "authors": [ "Pallavi Mitra", "Jai Kushwaha", "Felix Biessmann" ], "year": 2026, "venue_or_source": "ECML-PKDD 2026 GREEN-AI workshop", "source_type": "paper", "url": "https://arxiv.org/abs/2608.17711", "code_url": null, "topic_clusters": [ "C2", "C13" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "confidence" ], "cost_reduction_reported": "Selected cascade achieves up to 10-fold decrease in CO2 emissions", "benchmarks": [ "image classification with corruptions" ], "models_studied": [], "production_evidence": false, "summary": "Studies how input corruptions and sequential perturbations break confidence-based deferral in cascades, identifying three failure modes.", "key_idea": "Cascade routing reliability degrades under distribution shift.", "relevance_score": 2, "relevance_rationale": "Robustness caveat for deployed cascades.", "found_via": "arXiv title search: model cascading", "cites_or_cited_by": [] }, { "id": "nguyen-2026-cliffcompaction", "title": "CliffCompaction: Cost-Efficient Compaction for Long-Horizon Coding Agents", "authors": [ "Trang Nguyen", "Eulrang Cho", "Bingqing Chen", "Tim Dettmers" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.26779", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "compaction trigger" ], "cost_reduction_reported": null, "benchmarks": [ "KernelBench" ], "models_studied": [], "production_evidence": false, "summary": "Cost-efficient compaction for long-horizon coding agents that sustains sessions beyond a million tokens; released as an API proxy usable with Claude Code and Codex, and reported CUDA-kernel speedups of 2.23x after 200 steps and 3.58x after 400 on KernelBench.", "key_idea": "Harness-agnostic compaction proxy for very long coding sessions.", "relevance_score": 2, "relevance_rationale": "Practical compaction tool; cost numbers not in abstract.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [] }, { "id": "openai-2026-compaction-docs", "title": "Compaction (OpenAI API guide)", "authors": [ "OpenAI" ], "year": 2026, "venue_or_source": "OpenAI API documentation", "source_type": "docs", "url": "https://developers.openai.com/api/docs/guides/compaction", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "rendered token count threshold" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "OpenAI reasoning models" ], "production_evidence": false, "summary": "OpenAI's Responses API guide to compaction: setting context_management with a compact_threshold makes the server run a compaction pass when the rendered context crosses the threshold, emitting an opaque encrypted compaction item that carries prior state and reasoning forward in fewer tokens. Items before the latest compaction item can then be dropped to cut request size and tail latency.", "key_idea": "Threshold-triggered server-side compaction of agent history into an opaque state item.", "relevance_score": 2, "relevance_rationale": "Background API feature for context cost control; no numbers.", "found_via": "direct: developers.openai.com docs", "cites_or_cited_by": [ "anthropic-2026-compaction-docs" ] }, { "id": "petrova-2026-agents-dont-paginate", "title": "Agents Don't Paginate: First-Chunk Selection for LLM Tool Responses", "authors": [ "Tatiana Petrova", "Andrei Mazniak", "Radu State" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.26130", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "keyword scorer over tool-response chunks" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies which chunk of a long tool response to show an agent first; a keyword scorer improves rank-1 chunk selection (24.2% to 35.0%) but downstream accuracy does not change, since agents rarely page further.", "key_idea": "Tool-response truncation decisions matter because agents do not paginate.", "relevance_score": 2, "relevance_rationale": "Evidence on tool-output truncation behaviour.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [] }, { "id": "qiu-2026-dataprm", "title": "Rewarding the Scientific Process: Process-Level Reward Modeling for Agentic Data Analysis", "authors": [ "Zhisong Qiu", "Shuofei Qiao", "Kewei Xu", "Yuqi Zhu", "Lun Du", "Ningyu Zhang", "Huajun Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.24198", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "environment-aware generative PRM" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "DataPRM is a generative process reward model for agentic data analysis that actively probes execution state to catch silent errors, with a ternary reward separating correctable and irrecoverable errors.", "key_idea": "Active-verifier PRM for data-analysis agents.", "relevance_score": 2, "relevance_rationale": "Domain PRM example.", "found_via": "WebSearch: process reward model for LLM agents", "cites_or_cited_by": [] }, { "id": "sampath-2026-adaptive-orchestration-concierge", "title": "Adaptive Orchestration: Scalable Self-Evolving Multi-Agent Systems", "authors": [ "Sathish Sampath", "Anuradha Baskaran" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2601.09742", "code_url": null, "topic_clusters": [ "C3", "C12" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "capability-gap detection" ], "cost_reduction_reported": "Maintains high task success while minimizing token consumption vs static agent swarms (no number in abstract)", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes a self-evolving concierge that hires specialized sub-agents on demand based on live conversation analysis, evicts them with an LRU policy, and prunes history. Aims to avoid both monolithic context pollution and static swarm overhead.", "key_idea": "Hire and evict sub-agents dynamically instead of running a static swarm.", "relevance_score": 2, "relevance_rationale": "Architecture idea with limited evaluation detail.", "found_via": "OpenAlex search: agent pruning", "cites_or_cited_by": [] }, { "id": "sun-2026-cascade-adversarial-attack", "title": "When Efficiency Backfires: Cascading LLMs Trigger Cascade Failure under Adversarial Attack", "authors": [ "Zehan Sun", "Dingfan Chen", "Songze Li" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.17288", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows LLM cascades are vulnerable to adversarial suffixes that target the lightweight model and deferral mechanism together, degrading both accuracy and cost savings.", "key_idea": "Cascades expand the attack surface.", "relevance_score": 2, "relevance_rationale": "Security risk of cascade deployments.", "found_via": "arXiv search: \"LLM cascade\"", "cites_or_cited_by": [] }, { "id": "together-2026-ab-test-models-in-production", "title": "A/B test models in production", "authors": [ "Zain Hasan", "Zarni Phyo", "Nikitha Suryadevara", "Ted Cui" ], "year": 2026, "venue_or_source": "Together AI blog", "source_type": "product", "url": "https://www.together.ai/blog/a-b-test-models-in-production", "code_url": null, "topic_clusters": [ "C5", "C14" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "fixed traffic split" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Describes endpoint-level A/B experiments that split live traffic into one control and up to 20 variants with fixed percentage shares independent of replica counts, a ramp schedule (95/5, 80/20, 50/50), blue-green promotion and one-call rollback. The walkthrough verifies observed traffic shares on a live endpoint.", "key_idea": "Serving-layer traffic splitting for evaluating cheaper candidate models on real users.", "relevance_score": 2, "relevance_rationale": "Infrastructure for online evaluation of model swaps; fixed splits rather than adaptive bandits.", "found_via": "direct: together.ai/blog index", "cites_or_cited_by": [] }, { "id": "together-2026-cache-aware-pd-disaggregation", "title": "Cache-aware prefill-decode disaggregation (CPD) for up to 40% faster long-context LLM serving", "authors": [ "Jiejing Zhang", "Yubo Wang", "Yinghui Liu", "Mourya Vangala Srinivasa", "Chenxi Li", "Jue Wang", "Yineng Zhang", "Shuaiwen Leon Song", "Ce Zhang" ], "year": 2026, "venue_or_source": "Together AI blog", "source_type": "blog", "url": "https://www.together.ai/blog/cache-aware-disaggregated-inference", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "prefix-cache hit rate" ], "cost_reduction_reported": "Up to 35-40% higher sustainable QPS than existing disaggregated designs (~1.1-1.15 QPS per GPU) with lower TTFT on long-context mixed traffic (vendor-reported)", "benchmarks": [ "Together long-context serving benchmark" ], "models_studied": [], "production_evidence": false, "summary": "Separates cold (low cache-hit, heavy prefill) from warm (high cache-hit) requests into different prefill paths backed by a distributed KV cache, so context reuse in multi-turn, coding and agent workloads is served fast while cold long prompts do not block it.", "key_idea": "Route requests to prefill pools by cache warmth to exploit context reuse.", "relevance_score": 2, "relevance_rationale": "Serving background relevant to agentic context reuse.", "found_via": "direct: together.ai/blog index", "cites_or_cited_by": [] }, { "id": "vllm-2026-tiered-kv-offloading", "title": "Tiered KV Cache Offloading in vLLM", "authors": [ "Or Ozeri", "Danny Harnik", "Ronen Schaffer", "Itay Etelis", "Varun Sundar Rabindranath" ], "year": 2026, "venue_or_source": "vLLM blog", "source_type": "blog", "url": "https://blog.vllm.ai/blog/2026-09-10-tiered-kv-offloading", "code_url": "https://github.com/vllm-project/vllm", "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "multi-turn conversation benchmark on Qwen3.6-35B-A3B, 2x H100" ], "models_studied": [ "Qwen3.6-35B-A3B" ], "production_evidence": false, "summary": "Describes vLLM's host-centric tiered KV offloading, where evicted KV blocks flow through CPU memory to filesystem, object storage or remote peers and are reloaded instead of recomputed, with KV events enabling cache-aware routing in llm-d and Dynamo. A multi-turn benchmark shows throughput holding as the number of concurrent conversations exceeds HBM, then host, capacity.", "key_idea": "Reloading KV from cheaper tiers beats re-prefill for multi-turn and agentic sessions.", "relevance_score": 2, "relevance_rationale": "Serving-layer caching background for long agent sessions.", "found_via": "direct: blog.vllm.ai index", "cites_or_cited_by": [ "nvidia-dynamo-repo", "vllm-2026-agentx-agentic-serving" ] }, { "id": "wang-2026-actkv", "title": "ActKV: Efficient LLM Agents through Action-Guided KV Cache Management", "authors": [ "Zihan Wang", "Cheng Tang", "Lei Gong", "Chao Wang", "Wenqi Lou", "Teng Wang", "Xuehai Zhou" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.31395", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "action-guided importance", "LLM confidence" ], "cost_reduction_reported": "retains 98.53% of FullKV accuracy with 25.98% of peak KV memory and 3.97x token / 3.58x task throughput on long-trace tasks", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Action-guided KV cache management for agents with confidence-driven budget allocation and page-aware compression kernels.", "key_idea": "Compress agent KV cache guided by action relevance.", "relevance_score": 2, "relevance_rationale": "Agent-specific KV compression.", "found_via": "snowball: cited-by kang-2025-acon", "cites_or_cited_by": [ "zhang-2023-h2o" ] }, { "id": "wang-2026-agentspec", "title": "AgentSpec: Speculative Decoding for Batch Inference of LLM Agents", "authors": [ "Xin Wang", "Ziming Miao", "Yi Zhu", "Hui Shen", "Zhongwei Wan", "Fan Yang", "Mi Zhang" ], "year": 2026, "venue_or_source": "EMNLP 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2608.24004", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "agent-level semantic paths", "redundancy-aware draft budget" ], "cost_reduction_reported": null, "benchmarks": [ "five agent workloads" ], "models_studied": [ "four models from four LLM families" ], "production_evidence": false, "summary": "Speculative decoding tailored to batch inference of LLM agents: drafts are pruned along irrelevant semantic paths for low rejection rates, and a redundancy-aware budget reuses spare token budget, implemented in vLLM.", "key_idea": "Agent-aware speculative decoding for batch agent inference.", "relevance_score": 2, "relevance_rationale": "Speculative decoding specialized for agents.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [ "li-2024-eagle" ] }, { "id": "wang-2026-plan-rewardbench", "title": "Aligning Agents via Planning: A Benchmark for Trajectory-Level Reward Modeling", "authors": [ "Jiaxuan Wang", "Yulan Hu", "Wenjin Yang", "Zheng Pan", "Xin Li", "Lan-Zhe Guo" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2604.08178", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "trajectory preference" ], "cost_reduction_reported": null, "benchmarks": [ "Plan-RewardBench" ], "models_studied": [], "production_evidence": false, "summary": "Trajectory-level preference benchmark to evaluate how well judges and reward models distinguish preferred from distractor tool-using agent trajectories.", "key_idea": "Benchmark for trajectory-level agent RMs.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "WebSearch: agent trajectory evaluation step-level reward benchmark", "cites_or_cited_by": [] }, { "id": "wang-2026-reflect-judges-research-agents", "title": "Time to REFLECT: Can We Trust LLM Judges for Evidence-based Research Agents?", "authors": [ "Leyao Wang", "Yanan He", "Peng Chen", "Asaf Yehudai", "Yixin Liu", "Rex Ying", "Michal Shmueli-Scheuer", "Arman Cohan" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.19196", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judge" ], "cost_reduction_reported": null, "benchmarks": [ "REFLECT" ], "models_studied": [], "production_evidence": false, "summary": "Meta-evaluation showing LLM judges of deep-research agents remain unreliable (below 55% accuracy), especially on evidence verification.", "key_idea": "Judges for deep-research agents are unreliable.", "relevance_score": 2, "relevance_rationale": "Caveat.", "found_via": "hf-papers search: agent failure detection", "cites_or_cited_by": [] }, { "id": "wang-2026-smetric", "title": "SMetric: Rethink LLM Scheduling for Serving Agents with Balanced Session-centric Scheduling", "authors": [ "Jiahao Wang", "Kaizhan Lin", "Kaixi Zhang", "Jinbo Han", "Xingda Wei", "Sijie Shen", "Chenguang Fang", "Wenyuan Yu", "Rong Chen", "Haibo Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.08565", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "session stickiness", "instance SLO feasibility" ], "cost_reduction_reported": "peak TPS improved 9-15% under prefill-decode colocation and peak prefill TPS 9% under disaggregation on real-world traces", "benchmarks": [ "real-world traces" ], "models_studied": [], "production_evidence": false, "summary": "Session-centric scheduling for serving agents that keeps follow-up turns on the instance holding their cache only when SLOs can be met, otherwise migrating sessions to balance load.", "key_idea": "Balance cache affinity against load for agent sessions.", "relevance_score": 2, "relevance_rationale": "Session-level scheduling.", "found_via": "serving engine LLM agents agentic workflows scheduling arXiv 2025", "cites_or_cited_by": [] }, { "id": "wu-2026-litegui", "title": "LiteGUI: Distilling Compact GUI Agents with Reinforcement Learning", "authors": [ "Yubin Wu", "Zicheng Cai", "Liping Ning", "Hua Wang", "Zhi Chen", "Yaohua Tang", "Hao Chen" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.07505", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "structured on-policy distillation", "oracle trajectories" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "2B/3B VLMs" ], "production_evidence": false, "summary": "Distills compact 2B/3B GUI agents with structured on-policy distillation and dual-level exploration RL.", "key_idea": "RL distillation for on-device GUI agents.", "relevance_score": 2, "relevance_rationale": "Small agent specialization.", "found_via": "hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [] }, { "id": "yuan-2026-bayesflow", "title": "BayesFlow: A Probability Inference Framework for Meta-Agent Assisted Workflow Generation", "authors": [ "Bo Yuan", "Yun Zhou", "Zhichao Xu", "Kiran Ramnath", "Aosong Feng", "Balasubramaniam Srinivasan" ], "year": 2026, "venue_or_source": "Findings of EACL 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.22305", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "look-ahead rollouts", "importance weights" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "BayesFlow casts workflow generation as Bayesian inference over workflows with parallel look-ahead rollouts and an in-loop refiner, improving accuracy by up to 9 points over workflow-generation baselines.", "key_idea": "Workflow generation as posterior sampling.", "relevance_score": 2, "relevance_rationale": "Workflow optimization method; cost not central.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [] }, { "id": "zeng-2026-autoragtuner", "title": "AutoRAGTuner: A Declarative Framework for Automatic Optimization of RAG Pipelines", "authors": [ "Xintan Zeng", "Yongchao Liu", "Yice Luo", "Jiajun Zhen" ], "year": 2026, "venue_or_source": "EuroSys 2026 (poster)", "source_type": "paper", "url": "https://arxiv.org/abs/2605.02967", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "RAG metrics" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "AutoRAGTuner is a declarative framework covering RAG construction, execution, evaluation and optimization, with an adaptive Bayesian optimization engine for end-to-end hyperparameter tuning across vanilla to graph-based RAG.", "key_idea": "Declarative RAG pipeline spec plus adaptive BO tuning.", "relevance_score": 2, "relevance_rationale": "RAG config tuning infrastructure; no explicit cost objective.", "found_via": "search: AutoRAG automated RAG pipeline optimization arXiv", "cites_or_cited_by": [] }, { "id": "zeng-2026-when-multi-agent-rl-helps", "title": "When Does Multi-Agent RL Improve LLM Workflows? Workflow, Scale, and Policy-Sharing Tradeoffs", "authors": [ "Yifan Zeng", "Yiran Wu", "Yaolun Zhang", "Wentian Zhao", "Kun Wan", "Qingyun Wu", "Huazheng Wang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2605.24202", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "math", "code" ], "models_studied": [ "0.6B", "1.7B", "4B models" ], "production_evidence": false, "summary": "Studies when end-to-end RL training improves multi-agent workflows (Eval-Opt, Voting, Orch-Workers) at small scales, comparing shared and isolated policies. Gains depend on workflow, task and scale, and each policy-sharing choice has distinct failure modes.", "key_idea": "Workflow- and scale-dependent value of multi-agent RL training.", "relevance_score": 2, "relevance_rationale": "Background on training MAS roles.", "found_via": "OpenAlex search: per-step model routing", "cites_or_cited_by": [] }, { "id": "zhang-2026-cacheattack", "title": "From Similarity to Vulnerability: Key Collision Attack on LLM Semantic Caching", "authors": [ "Zhixiang Zhang", "Zesen Liu", "Yuchong Xie", "Quanfeng Huang", "Dongdong She" ], "year": 2026, "venue_or_source": "ICML 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2601.23088", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "embedding key collisions" ], "cost_reduction_reported": null, "benchmarks": [ "security-critical tasks", "agentic workflows", "financial agent case study" ], "models_studied": [], "production_evidence": false, "summary": "First systematic study of integrity attacks on semantic caches: CacheAttack crafts black-box queries whose embeddings collide with benign ones, hijacking responses (86% hit rate) and inducing malicious agent behaviour, with transfer across embedding models.", "key_idea": "Semantic caches are fuzzy hashes and can be poisoned via collisions.", "relevance_score": 2, "relevance_rationale": "Security risk to weigh when adding semantic caching to production agents.", "found_via": "semantic caching LLM false positive hits correctness study", "cites_or_cited_by": [ "liang-2026-lacache", "zhang-2026-semantic-cache-poisoning" ] }, { "id": "zhang-2026-cclub-prompt-routing", "title": "Steering Frozen LLMs: Adaptive Social Alignment via Online Prompt Routing", "authors": [ "Zeyu Zhang", "Xiangxiang Dai", "Ziyi Han", "Xutong Liu", "John C. S. Lui" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2603.15647", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "query", "learning_regime": "bandit-online", "signals_used": [ "utility and safety similarity graphs" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CCLUB routes queries among system prompts with a consensus-clustering LinUCB bandit to adapt safety alignment at inference time, with sublinear regret.", "key_idea": "Clustered LinUCB routing across system prompts.", "relevance_score": 2, "relevance_rationale": "Online configuration routing, safety not cost.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [ "li-2010-linucb" ] }, { "id": "zhang-2026-semantic-cache-poisoning", "title": "Similarity Is Not Validity: Defending LLM Semantic Caches Against Poisoning", "authors": [ "Zihan Zhang", "Shuangjie Yao", "Zesen Liu", "Zhixiang Zhang", "Wai Ip Lai", "Dung Hiu Hilton Yeung", "Chun Kit Zhang", "Fuchen Ma", "Yuanyuan Yuan", "Yu Jiang", "Dongdong She" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2609.35908", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "deletion gain", "answer check" ], "cost_reduction_reported": "blocks 82.0%-98.2% of poisoned cache entries at a 5% false-positive rate with negligible serving overhead", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defends semantic caches against poisoning by exploiting the rewrite-plus-residual structure of adversarial queries: it searches shortened variants of cached queries for similarity gains and checks whether removed text affects the stored answer.", "key_idea": "Detect poisoned cache entries via deletion-based similarity tests.", "relevance_score": 2, "relevance_rationale": "Security hardening for semantic caches.", "found_via": "arXiv search: semantic cache LLM", "cites_or_cited_by": [ "zhang-2026-cacheattack" ] }, { "id": "zhang-2026-specbox", "title": "SpecBox: Speculative Sandbox Scheduling for Efficient LLM Agent Serving", "authors": [ "Yihui Zhang", "Tianyu Wo", "Jinghao Wang", "Xiaoyang Sun", "Menghao Zhang", "Cangzhou Yuan", "Li Li", "Chunming Hu", "Albert Y. Zomaya", "Renyu Yang" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2607.23933", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "sandbox dependency graph", "stochastic prefetching", "semantic result cache" ], "cost_reduction_reported": "P99 end-to-end latency reduced up to 2.9x vs on-demand sandboxes and peak memory reduced 45.9% vs permanently reserved sandboxes", "benchmarks": [ "multi-turn agent traces" ], "models_studied": [], "production_evidence": false, "summary": "Speculatively prewarms agent sandboxes by forecasting upcoming sandbox switches, prunes repeated invocations via a result cache, and uses shared-memory transfers.", "key_idea": "Speculative sandbox prewarming for agent tool execution.", "relevance_score": 2, "relevance_rationale": "Tool-execution side of agent serving.", "found_via": "arXiv search: semantic cache LLM", "cites_or_cited_by": [] }, { "id": "zhao-2026-catchbench", "title": "CatchBench: When Can an Agent Failure Be Caught?", "authors": [ "Yue Zhao", "Mengyuan Li", "Ruolin Li", "Xiongye Xiao", "Xiyang Hu" ], "year": 2026, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2608.22808", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "rule scanners", "LLM judges" ], "cost_reduction_reported": null, "benchmarks": [ "CatchBench" ], "models_studied": [ "GPT", "Claude", "Gemini", "Qwen", "Llama" ], "production_evidence": false, "summary": "Benchmark/arena evaluating 72 failure detectors (rule scanners, structural models, eleven LLM judges) on when agent failures can be caught, finding most contrasts unresolved.", "key_idea": "When are agent failures detectable at all.", "relevance_score": 2, "relevance_rationale": "Meta-evaluation of detectors.", "found_via": "hf-papers search: agent failure detection", "cites_or_cited_by": [] }, { "id": "ashiga-2025-ensemble-survey", "title": "Ensemble Learning for Large Language Models in Text and Code Generation: A Survey", "authors": [ "Mari Ashiga", "Wei Jie", "Fan Wu", "Vardan Voskanyan", "Fateme Dinmohammadi", "Paul Brookes", "Jingzhi Gong", "Zheng Wang" ], "year": 2025, "venue_or_source": "IEEE TAI 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2503.13505", "code_url": null, "topic_clusters": [ "C1", "C2", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Reviews LLM ensemble techniques for text and code generation in seven categories including routing and cascading.", "key_idea": "Ensemble survey with routing and cascading sections.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "chen-2025-kairos", "title": "Kairos: Low-latency Multi-Agent Serving with Shared LLMs and Excessive Loads in the Public Cloud", "authors": [ "Jinyuan Chen", "Jiuchen Shi", "Quan Chen", "Minyi Guo" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.06948", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "agent latency characteristics", "memory demand" ], "cost_reduction_reported": "end-to-end latency reduced 17.8% to 28.4% vs state-of-the-art", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Multi-agent serving on shared LLMs under heavy load via a workflow orchestrator, latency-aware priority scheduler and memory-aware dispatcher.", "key_idea": "Latency- and memory-aware dispatch for multi-agent traffic.", "relevance_score": 2, "relevance_rationale": "Multi-agent serving.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [] }, { "id": "cheng-2025-agent-r1", "title": "Agent-R1: A Unified and Modular Framework for Agentic Reinforcement Learning", "authors": [ "Mingyue Cheng", "Jie Ouyang", "Shuo Yu", "Ruiran Yan", "Yucong Luo", "Zirui Liu", "Daoyu Wang", "Qi Liu", "Enhong Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.14460", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "environment rewards" ], "cost_reduction_reported": null, "benchmarks": [ "multi-hop QA" ], "models_studied": [], "production_evidence": false, "summary": "Clarifies RL formulations for LLM agents and releases Agent-R1, a modular framework for end-to-end RL training of tool-using agents.", "key_idea": "Modular end-to-end agent RL framework.", "relevance_score": 2, "relevance_rationale": "Tooling for RL-specializing small agents.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "chung-2025-mlenergy", "title": "The ML.ENERGY Benchmark: Toward Automated Inference Energy Measurement and Optimization", "authors": [ "Jae-Won Chung", "Jeff J. Ma", "Ruofan Wu", "Jiachen Liu", "Oh Jun Kweon", "Yuxuan Xia", "Zhiyu Wu", "Mosharaf Chowdhury" ], "year": 2025, "venue_or_source": "NeurIPS 2025 Datasets and Benchmarks (Spotlight)", "source_type": "paper", "url": "https://arxiv.org/abs/2505.06371", "code_url": null, "topic_clusters": [ "C11", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "energy", "latency" ], "cost_reduction_reported": "Automated optimization recommendations sometimes save more than 40% energy", "benchmarks": [ "ML.ENERGY Benchmark (40 models, 6 tasks)" ], "models_studied": [ "40 generative model architectures" ], "production_evidence": false, "summary": "Benchmark suite and leaderboard for inference energy under realistic serving, with design principles and case studies on how model and serving choices affect energy.", "key_idea": "Energy-per-request benchmarking and leaderboard.", "relevance_score": 2, "relevance_rationale": "Efficiency leaderboard; serving cost proxy.", "found_via": "web search: ML.ENERGY leaderboard", "cites_or_cited_by": [] }, { "id": "dai-2025-scuba", "title": "SCUBA: Salesforce Computer Use Benchmark", "authors": [ "Yutong Dai", "Krithika Ramakrishnan", "Jing Gu", "Matthew Fernandez", "Yanqi Luo", "Viraj Prabhu", "Zhenyu Hu", "Silvio Savarese", "Caiming Xiong", "Zeyuan Chen", "Ran Xu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.26506", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "milestones" ], "cost_reduction_reported": null, "benchmarks": [ "SCUBA (300 tasks on Salesforce)" ], "models_studied": [ "computer-use agents" ], "production_evidence": false, "summary": "Benchmark of 300 CRM tasks from user interviews on Salesforce sandboxes for computer-use agents, with milestone metrics; finds large gaps between agent paradigms and between open and closed models.", "key_idea": "Enterprise CRM computer-use benchmark.", "relevance_score": 2, "relevance_rationale": "Enterprise UI workflow benchmark.", "found_via": "web search: CRMArena-Pro", "cites_or_cited_by": [] }, { "id": "dang-2025-rl-small-llms-reasoning", "title": "Reinforcement Learning for Reasoning in Small LLMs: What Works and What Doesn't", "authors": [ "Quy-Anh Dang", "Chris Ngo" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.16219", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "GRPO" ], "cost_reduction_reported": "AMC23 63% to 80%, AIME24 46.7% using 7,000 samples and $42 training cost", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows RL on a 1.5B model with small curated data and minimal budget gives large reasoning gains.", "key_idea": "Cheap RL for small reasoning models.", "relevance_score": 2, "relevance_rationale": "Training-cost evidence.", "found_via": "hf-papers search: RL fine-tuning small LLM tool use", "cites_or_cited_by": [] }, { "id": "deepseekai-2025-deepseek-r1", "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "authors": [ "DeepSeek-AI", "Daya Guo", "Dejian Yang", "Haowei Zhang", "Junxiao Song", "Ruoyu Zhang", "Runxin Xu", "Qihao Zhu", "Shirong Ma", "Peiyi Wang", "Xiao Bi", "Xiaokang Zhang", "Xingkai Yu", "Yu Wu", "Z. F. Wu", "Zhibin Gou", "Zhihong Shao", "Zhuoshu Li", "Ziyi Gao", "Aixin Liu", "Bing Xue", "Bingxuan Wang", "Bochao Wu", "Bei Feng", "Chengda Lu", "Chenggang Zhao", "Chengqi Deng", "Chenyu Zhang", "Chong Ruan", "Damai Dai", "Deli Chen", "Dongjie Ji", "Erhang Li", "Fangyun Lin", "Fucong Dai", "Fuli Luo", "Guangbo Hao", "Guanting Chen", "Guowei Li", "H. Zhang", "Han Bao", "Hanwei Xu", "Haocheng Wang", "Honghui Ding", "Huajian Xin", "Huazuo Gao", "Hui Qu", "Hui Li", "Jianzhong Guo", "Jiashi Li", "Jiawei Wang", "Jingchang Chen", "Jingyang Yuan", "Junjie Qiu", "Junlong Li", "J. L. Cai", "Jiaqi Ni", "Jian Liang", "Jin Chen", "Kai Dong", "Kai Hu", "Kaige Gao", "Kang Guan", "Kexin Huang", "Kuai Yu", "Lean Wang", "Lecong Zhang", "Liang Zhao", "Litong Wang", "Liyue Zhang", "Lei Xu", "Leyi Xia", "Mingchuan Zhang", "Minghua Zhang", "Minghui Tang", "Meng Li", "Miaojun Wang", "Mingming Li", "Ning Tian", "Panpan Huang", "Peng Zhang", "Qiancheng Wang", "Qinyu Chen", "Qiushi Du", "Ruiqi Ge", "Ruisong Zhang", "Ruizhe Pan", "Runji Wang", "R. J. Chen", "R. L. Jin", "Ruyi Chen", "Shanghao Lu", "Shangyan Zhou", "Shanhuang Chen", "Shengfeng Ye", "Shiyu Wang", "Shuiping Yu", "Shunfeng Zhou", "Shuting Pan", "S. S. Li", "Shuang Zhou", "Shaoqing Wu", "Shengfeng Ye", "Tao Yun", "Tian Pei", "Tianyu Sun", "T. Wang", "Wangding Zeng", "Wanjia Zhao", "Wen Liu", "Wenfeng Liang", "Wenjun Gao", "Wenqin Yu", "Wentao Zhang", "W. L. Xiao", "Wei An", "Xiaodong Liu", "Xiaohan Wang", "Xiaokang Chen", "Xiaotao Nie", "Xin Cheng", "Xin Liu", "Xin Xie", "Xingchao Liu", "Xinyu Yang", "Xinyuan Li", "Xuecheng Su", "Xuheng Lin", "X. Q. Li", "Xiangyue Jin", "Xiaojin Shen", "Xiaosha Chen", "Xiaowen Sun", "Xiaoxiang Wang", "Xinnan Song", "Xinyi Zhou", "Xianzu Wang", "Xinxia Shan", "Y. K. Li", "Y. Q. Wang", "Y. X. Wei", "Yang Zhang", "Yanhong Xu", "Yao Li", "Yao Zhao", "Yaofeng Sun", "Yaohui Wang", "Yi Yu", "Yichao Zhang", "Yifan Shi", "Yiliang Xiong", "Ying He", "Yishi Piao", "Yisong Wang", "Yixuan Tan", "Yiyang Ma", "Yiyuan Liu", "Yongqiang Guo", "Yuan Ou", "Yuduan Wang", "Yue Gong", "Yuheng Zou", "Yujia He", "Yunfan Xiong", "Yuxiang Luo", "Yuxiang You", "Yuxuan Liu", "Yuyang Zhou", "Y. X. Zhu", "Yanhong Xu", "Yanping Huang", "Yaohui Li", "Yi Zheng", "Yuchen Zhu", "Yunxian Ma", "Ying Tang", "Yukun Zha", "Yuting Yan", "Z. Z. Ren", "Zehui Ren", "Zhangli Sha", "Zhe Fu", "Zhean Xu", "Zhenda Xie", "Zhengyan Zhang", "Zhewen Hao", "Zhicheng Ma", "Zhigang Yan", "Zhiyu Wu", "Zihui Gu", "Zijia Zhu", "Zijun Liu", "Zilin Li", "Ziwei Xie", "Ziyang Song", "Zizheng Pan", "Zhen Huang", "Zhipeng Xu", "Zhongyu Zhang", "Zhen Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.12948", "code_url": null, "topic_clusters": [ "C8", "C9" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "rule-based rewards", "distilled traces" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "DeepSeek-V3", "Qwen", "Llama" ], "production_evidence": false, "summary": "Large-scale RL for reasoning plus distillation of R1 reasoning traces into small dense Qwen/Llama models.", "key_idea": "RL reasoning and distillation into small models.", "relevance_score": 2, "relevance_rationale": "Reference for trace distillation.", "found_via": "keyword: distillation reasoning", "cites_or_cited_by": [] }, { "id": "estornell-2025-acc-collab", "title": "ACC-Collab: An Actor-Critic Approach to Multi-Agent LLM Collaboration", "authors": [ "Andrew Estornell", "Jean-Francois Ton", "Yuanshun Yao", "Yang Liu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.00053", "code_url": null, "topic_clusters": [ "C7", "C8" ], "routing_granularity": "subagent", "learning_regime": "RL-offline", "signals_used": [ "critic agent" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Trains an actor-critic pair of LLM agents specialized for collaboration, outperforming multi-agent debate baselines.", "key_idea": "Learned critic agent for multi-agent collaboration.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "known-item lookup: critic agent", "cites_or_cited_by": [] }, { "id": "feng-2025-gigpo", "title": "Group-in-Group Policy Optimization for LLM Agent Training", "authors": [ "Lang Feng", "Zhenghai Xue", "Tingcong Liu", "Bo An" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.10978", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "step", "learning_regime": "RL-online", "signals_used": [ "anchor-state step grouping" ], "cost_reduction_reported": "> 12% on ALFWorld and > 9% on WebShop over GRPO at same compute", "benchmarks": [ "ALFWorld", "WebShop" ], "models_studied": [ "Qwen2.5-1.5B", "Qwen2.5-7B" ], "production_evidence": false, "summary": "Group-in-group policy optimization adds step-level relative advantages from repeated anchor states to GRPO for long-horizon agents.", "key_idea": "Critic-free step-level credit assignment.", "relevance_score": 2, "relevance_rationale": "RL method for small agents.", "found_via": "keyword: RL fine-tuning for agents", "cites_or_cited_by": [] }, { "id": "feng-2025-retool", "title": "ReTool: Reinforcement Learning for Strategic Tool Use in LLMs", "authors": [ "Jiazhan Feng", "Shijue Huang", "Xingwei Qu", "Ge Zhang", "Yujia Qin", "Baoquan Zhong", "Chengquan Jiang", "Jinxin Chi", "Wanjun Zhong" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.11536", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "code interpreter feedback" ], "cost_reduction_reported": "32B reaches 67% on AIME with 400 steps vs 40% for text RL with 1080 steps", "benchmarks": [ "AIME" ], "models_studied": [], "production_evidence": false, "summary": "RL with interleaved code execution to teach strategic tool use for math reasoning.", "key_idea": "RL for tool-integrated reasoning.", "relevance_score": 2, "relevance_rationale": "RL specialization.", "found_via": "keyword: RL tool use", "cites_or_cited_by": [] }, { "id": "fu-2025-agentrefine", "title": "AgentRefine: Enhancing Agent Generalization through Refinement Tuning", "authors": [ "Dayuan Fu", "Keqing He", "Yejie Wang", "Wentao Hong", "Zhuoma Gongque", "Weihao Zeng", "Wei Wang", "Jingang Wang", "Xunliang Cai", "Weiran Xu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.01702", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "refinement trajectories" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Tunes agents on data that includes correction of their mistakes from environment feedback, improving generalization.", "key_idea": "Refinement tuning for agent generalization.", "relevance_score": 2, "relevance_rationale": "SFT recipe.", "found_via": "keyword: fine-tuning small agents", "cites_or_cited_by": [] }, { "id": "gu-2025-auditing-prompt-caching", "title": "Auditing Prompt Caching in Language Model APIs", "authors": [ "Chenchen Gu", "Xiang Lisa Li", "Rohith Kuditipudi", "Percy Liang", "Tatsunori Hashimoto" ], "year": 2025, "venue_or_source": "ICML 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.07776", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "timing side channel" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "seven API providers including OpenAI" ], "production_evidence": true, "summary": "Develops statistical timing audits that detect prompt caching in real LLM APIs and finds global cache sharing across users at seven providers, which can leak information about other users' prompts and model architecture.", "key_idea": "Prompt caching is detectable via timing and can leak across users.", "relevance_score": 2, "relevance_rationale": "Privacy/security consideration for shared prompt caches in enterprise deployments.", "found_via": "snowball: related work of lumer-2026-dont-break-the-cache", "cites_or_cited_by": [] }, { "id": "guan-2025-glow", "title": "GLOW: Graph-Language Co-Encoding for Agentic Workflow Performance Prediction", "authors": [ "Wei Guan", "Jian Cao", "Jinyu Cai", "Qiqi Cai", "Jianqi Gao", "See-Kiong Ng" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.15751", "code_url": null, "topic_clusters": [ "C4", "C6" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "graph and language co-encoding" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "GLOW combines graph and language encoders to predict performance of candidate agentic workflows, reducing the need for costly LLM calls during workflow generation.", "key_idea": "Graph-language surrogate for workflow performance.", "relevance_score": 2, "relevance_rationale": "Surrogate evaluation for config search.", "found_via": "search: agentic workflow performance predictor surrogate cheap evaluation arXiv", "cites_or_cited_by": [] }, { "id": "haque-2025-tinyllm-edge-agents", "title": "TinyLLM: Evaluation and Optimization of Small Language Models for Agentic Tasks on Edge Devices", "authors": [ "Mohd Ariful Haque", "Fahad Rahman", "Kishor Datta Gupta", "Khalil Shujaee", "Roy George" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.22138", "code_url": null, "topic_clusters": [ "C8", "C13" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "function-calling fine-tuning" ], "cost_reduction_reported": null, "benchmarks": [ "BFCL" ], "models_studied": [ "<1B-3B SLMs" ], "production_evidence": false, "summary": "Evaluates and optimizes small models for function calling on edge devices; 1-3B models reach up to 65.74% overall BFCL accuracy with hybrid optimization.", "key_idea": "SLM function calling at the edge.", "relevance_score": 2, "relevance_rationale": "Small agent evidence.", "found_via": "hf-papers search: small language model function calling", "cites_or_cited_by": [] }, { "id": "iacobacci-2025-thinking-budget-not-all", "title": "Increasing the Thinking Budget is Not All You Need", "authors": [ "Ignacio Iacobacci", "Zhaozhi Qian", "Faroq AL-Tam", "Muhammad AL-Qurishi", "Riad Souissi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2512.19585", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "thinking budget", "self-consistency", "reflection" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Short study finding that raising the thinking budget is not the most efficient use of compute; self-consistency and self-reflection configurations can reach higher accuracy.", "key_idea": "Spend compute on sampling/reflection before longer thinking.", "relevance_score": 2, "relevance_rationale": "Background evidence.", "found_via": "arXiv title search: thinking budget", "cites_or_cited_by": [] }, { "id": "iyengar-2025-generative-caching", "title": "A Generative Caching System for Large Language Models", "authors": [ "Arun Iyengar", "Ashish Kundu", "Ramana Kompella", "Sai Nandan Mamidi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.17603", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query embedding similarity", "cost/latency/quality tradeoff" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Presents a caching system that, beyond serving semantic hits, can synthesize answers for unseen queries by combining several cached responses (\"generative caching\"), and tunes caching decisions to balance cost, latency and response quality. Reports being considerably faster than GPTCache.", "key_idea": "Combine multiple cached answers to respond to new queries.", "relevance_score": 2, "relevance_rationale": "Variant of semantic caching with explicit cost/quality balancing.", "found_via": "GPTCache semantic cache LLM", "cites_or_cited_by": [ "bang-2023-gptcache" ] }, { "id": "jin-2025-search-r1", "title": "Search-R1: Training LLMs to Reason and Leverage Search Engines with Reinforcement Learning", "authors": [ "Bowen Jin", "Hansi Zeng", "Zhenrui Yue", "Dong Wang", "Hamed Zamani", "Jiawei Han" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.09516", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "outcome reward" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Trains LLMs via RL alone to interleave search-engine calls with reasoning, using retrieved-token masking and outcome rewards.", "key_idea": "RL-trained search agents.", "relevance_score": 2, "relevance_rationale": "RL specialization of small search agents.", "found_via": "hf-papers search: multi-turn reinforcement learning agents", "cites_or_cited_by": [] }, { "id": "joey-2025-temporal-predictors-outcome", "title": "Temporal Predictors of Outcome in Reasoning Language Models", "authors": [ "David, Joey" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.14773", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state probe over CoT" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies how early during chain-of-thought an LLM's hidden states commit to its eventual outcome, using probes on intermediate states.", "key_idea": "Outcome predictable early in CoT.", "relevance_score": 2, "relevance_rationale": "Supports early-exit/escalation signals.", "found_via": "WebSearch: probes hidden states LLM correctness prediction routing escalation", "cites_or_cited_by": [] }, { "id": "kartal-2025-ragsmith", "title": "RAGSmith: A Framework for Finding the Optimal Composition of Retrieval-Augmented Generation Methods Across Datasets", "authors": [ "Muhammed Yusuf Kartal", "Suha Kagan Kose", "Korhan Sevinç", "Burak Aktas" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.01386", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "retrieval metrics", "LLM judge" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RAGSmith treats RAG design as architecture search over nine technique families and 46,080 configurations using a genetic algorithm with a combined retrieval-generation objective, gaining +3.8% on average over naive RAG while exploring about 0.2% of the space.", "key_idea": "Genetic search over RAG technique compositions.", "relevance_score": 2, "relevance_rationale": "RAG config search; quality-only.", "found_via": "search: AutoRAG automated RAG pipeline optimization arXiv", "cites_or_cited_by": [] }, { "id": "khalifa-2025-thinkprm", "title": "Process Reward Models That Think", "authors": [ "Muhammad Khalifa", "Rishabh Agarwal", "Lajanugen Logeswaran", "Jaekyeom Kim", "Hao Peng", "Moontae Lee", "Honglak Lee", "Lu Wang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.16828", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "generative PRM", "verification CoT" ], "cost_reduction_reported": null, "benchmarks": [ "ProcessBench", "MATH-500", "AIME 24", "GPQA-Diamond", "LiveCodeBench" ], "models_studied": [], "production_evidence": false, "summary": "Long-CoT generative PRM fine-tuned on 1% of PRM800K labels that beats discriminative PRMs and LLM-as-judge.", "key_idea": "Data-efficient generative PRMs.", "relevance_score": 2, "relevance_rationale": "PRM design option.", "found_via": "keyword: process reward model", "cites_or_cited_by": [] }, { "id": "kirchhof-2025-uq-agents-position", "title": "Position: Uncertainty Quantification Needs Reassessment for Large-language Model Agents", "authors": [ "Michael Kirchhof", "Gjergji Kasneci", "Enkelejda Kasneci" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.22655", "code_url": null, "topic_clusters": [ "C7", "C15" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "uncertainty" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Position paper arguing aleatoric/epistemic dichotomies are ill-suited to interactive LLM agents and proposing research on underspecification, interactive and output uncertainty.", "key_idea": "UQ must be rethought for agents.", "relevance_score": 2, "relevance_rationale": "Framing for agent UQ.", "found_via": "keyword: uncertainty LLM agents", "cites_or_cited_by": [] }, { "id": "koh-2025-rbridge", "title": "Predicting LLM Reasoning Performance with Small Proxy Model", "authors": [ "Woosung Koh", "Juyoung Suk", "Sungjun Han", "Se-Young Yun", "Jamin Shin" ], "year": 2025, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2509.21013", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "small proxy model NLL weighted by task alignment" ], "cost_reduction_reported": "reduces dataset ranking costs by over 100x relative to the best baseline", "benchmarks": [ "six reasoning benchmarks" ], "models_studied": [ "1B-32B models" ], "production_evidence": false, "summary": "rBridge uses small (<=1B) proxy models with task-aligned likelihood to predict larger models' reasoning performance for pretraining data decisions.", "key_idea": "Small proxies predict large-model reasoning.", "relevance_score": 2, "relevance_rationale": "Background on performance prediction.", "found_via": "snowball: cited-by pacchiardi-2025-predictaboard", "cites_or_cited_by": [] }, { "id": "langchain-2025-context-engineering", "title": "Context Engineering (for agents)", "authors": [ "LangChain Team" ], "year": 2025, "venue_or_source": "LangChain blog", "source_type": "blog", "url": "https://www.langchain.com/blog/context-engineering-for-agents", "code_url": "https://github.com/langchain-ai/context_engineering", "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Organizes agent context engineering into four strategies: write context outside the window (scratchpads, memories), select relevant context back in (retrieval, tool selection), compress it (summarization, trimming), and isolate it (multi-agent, sandboxes, state fields), with LangGraph examples.", "key_idea": "Write, select, compress, isolate.", "relevance_score": 2, "relevance_rationale": "Useful taxonomy; little quantitative evidence.", "found_via": "LangChain blog context engineering for agents write select compress isolate", "cites_or_cited_by": [ "rajasekaran-2025-effective-context-engineering" ] }, { "id": "li-2025-eagle-3", "title": "EAGLE-3: Scaling up Inference Acceleration of Large Language Models via Training-Time Test", "authors": [ "Yuhui Li", "Fangyun Wei", "Chao Zhang", "Hongyang Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.01840", "code_url": "https://github.com/SafeAILab/EAGLE", "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "multi-layer feature fusion", "training-time test" ], "cost_reduction_reported": "speedup up to 6.5x (about 1.4x over EAGLE-2); 1.38x throughput at batch size 64 in SGLang", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Drops feature prediction in favour of direct token prediction with multi-layer feature fusion trained via training-time test, letting the draft scale with more training data; evaluated on chat and reasoning models.", "key_idea": "Scale draft training for larger speculative speedups.", "relevance_score": 2, "relevance_rationale": "Current state of speculative decoding in production engines.", "found_via": "task seed", "cites_or_cited_by": [ "li-2024-eagle", "li-2024-eagle-2" ] }, { "id": "li-2025-edge-cloud-survey", "title": "Collaborative Inference and Learning between Edge SLMs and Cloud LLMs: A Survey of Algorithms, Execution, and Open Challenges", "authors": [ "Senyao Li", "Haozhao Wang", "Wenchao Xu", "Rui Zhang", "Song Guo", "Jingling Yuan", "Xian Zhong", "Tianwei Zhang", "Ruixuan Li" ], "year": 2025, "venue_or_source": "ACM Computing Surveys (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2507.16731", "code_url": null, "topic_clusters": [ "C1", "C11", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of edge SLM and cloud LLM collaboration for inference and training, covering task assignment, division and mixture at task and token levels.", "key_idea": "Edge-cloud collaboration taxonomy.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "li-2025-websailor", "title": "WebSailor: Navigating Super-human Reasoning for Web Agent", "authors": [ "Kuan Li", "Zhongwang Zhang", "Huifeng Yin", "Liwen Zhang", "Litu Ou", "Jialong Wu", "Wenbiao Yin", "Baixuan Li", "Zhengwei Tao", "Xinyu Wang", "Weizhou Shen", "Junkai Zhang", "Dingchu Zhang", "Xixi Wu", "Yong Jiang", "Ming Yan", "Pengjun Xie", "Fei Huang", "Jingren Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.02592", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "RFT cold start", "DUPO RL" ], "cost_reduction_reported": null, "benchmarks": [ "BrowseComp" ], "models_studied": [], "production_evidence": false, "summary": "Post-training recipe for web agents with high-uncertainty synthetic tasks, rejection-sampling fine-tuning and efficient agentic RL.", "key_idea": "Synthetic hard tasks plus RL for deep-research agents.", "relevance_score": 2, "relevance_rationale": "Open specialized research agents.", "found_via": "keyword: agent distillation", "cites_or_cited_by": [] }, { "id": "lin-2025-router-vulnerabilities", "title": "Life-Cycle Routing Vulnerabilities of LLM Router", "authors": [ "Qiqi Lin", "Xiaoyang Ji", "Shengfang Zhai", "Qingni Shen", "Zhi Zhang", "Yuejian Fang", "Yansong Gao" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.08704", "code_url": null, "topic_clusters": [ "C1" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "adversarial/backdoor attacks" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies adversarial and backdoor robustness of LLM routers across their lifecycle, finding DNN-based routers weakest and training-free routers most robust.", "key_idea": "Security of routers.", "relevance_score": 2, "relevance_rationale": "Background risk.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "liu-2025-deepseek-grm", "title": "Inference-Time Scaling for Generalist Reward Modeling", "authors": [ "Zijun Liu", "Peiyi Wang", "Runxin Xu", "Shirong Ma", "Chong Ruan", "Peng Li", "Yang Liu", "Yu Wu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.02495", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "generative reward", "principles and critiques" ], "cost_reduction_reported": null, "benchmarks": [ "RewardBench", "PPE", "RMB" ], "models_studied": [ "DeepSeek-GRM-27B" ], "production_evidence": false, "summary": "Studies inference-time scaling of generalist generative reward models with Self-Principled Critique Tuning, sampling multiple principles/critiques and aggregating with a meta RM.", "key_idea": "Reward models that improve with more inference compute.", "relevance_score": 2, "relevance_rationale": "Background on scalable judges.", "found_via": "keyword: outcome reward model", "cites_or_cited_by": [] }, { "id": "liu-2025-efficient-inference-lrm-survey", "title": "Efficient Inference for Large Reasoning Models: A Survey", "authors": [ "Yue Liu", "Jiaying Wu", "Yufei He", "Ruihan Gong", "Jun Xia", "Liang Li", "Hongcheng Gao", "Hongyu Chen", "Baolong Bi", "Jiaheng Zhang", "Zhiqi Huang", "Bryan Hooi", "Stan Z. Li", "Keqin Li" ], "year": 2025, "venue_or_source": "IEEE TPAMI (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2503.23077", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of efficient inference for large reasoning models, dividing methods into explicit compact CoT and implicit latent CoT, with empirical analysis.", "key_idea": "Token-efficient LRM inference survey.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "nvidia-dynamo-repo", "title": "NVIDIA Dynamo: A Datacenter Scale Distributed Inference Serving Framework", "authors": [ "NVIDIA" ], "year": 2025, "venue_or_source": "GitHub (ai-dynamo/dynamo)", "source_type": "repo", "url": "https://github.com/ai-dynamo/dynamo", "code_url": "https://github.com/ai-dynamo/dynamo", "topic_clusters": [ "C10", "C11" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "worker load", "KV-cache overlap" ], "cost_reduction_reported": "Repo-listed results: 2x faster TTFT with KV-aware routing (Qwen3-Coder 480B, Baseten benchmark); 80% fewer SLA breaches from Planner autoscaling at 5% lower TCO (Alibaba, APSARA 2025); 7x higher throughput per GPU for DeepSeek R1 on GB200 NVL72 vs B200 without Dynamo (vendor/partner-reported)", "benchmarks": [ "InferenceX", "partner benchmarks" ], "models_studied": [ "DeepSeek R1", "DeepSeek-V3", "Qwen3-Coder 480B" ], "production_evidence": true, "summary": "Open-source serving layer over vLLM, SGLang and TensorRT-LLM providing KV-cache-overlap-aware request routing, a KV block manager offloading GPU->CPU->SSD->remote, prefill/decode disaggregation, fast weight streaming for cold starts and an SLA-driven autoscaling planner. The repo lists partner deployment results, including Alibaba's autoscaling outcome.", "key_idea": "Cache-aware routing and disaggregated serving to cut the cost of prefill-heavy multi-turn traffic.", "relevance_score": 2, "relevance_rationale": "Serving infrastructure background; companion to the index's Dynamo agentic-inference blog.", "found_via": "direct: github.com/ai-dynamo/dynamo", "cites_or_cited_by": [ "vllm-2026-tiered-kv-offloading" ] }, { "id": "openai-agents-sdk-handoffs", "title": "Handoffs (OpenAI Agents SDK documentation)", "authors": [ "OpenAI" ], "year": 2025, "venue_or_source": "OpenAI Agents SDK documentation", "source_type": "docs", "url": "https://openai.github.io/openai-agents-python/handoffs/", "code_url": "https://github.com/openai/openai-agents-python", "topic_clusters": [ "C3", "C10", "C12" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "LLM tool-call choice" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Documents handoffs, in which an agent delegates to a specialised agent through a transfer_to_ tool. Each agent can use its own model, and input filters, nested handoff history and history mappers control how much prior conversation the receiving agent sees, which is the SDK's lever for per-subagent model choice and context size.", "key_idea": "LLM-driven delegation as tool calls, with configurable history passed to each specialist (and its model).", "relevance_score": 2, "relevance_rationale": "Framework primitive for subagent-level model assignment and context control; no results.", "found_via": "direct: openai.github.io docs", "cites_or_cited_by": [ "google-adk-agent-model-routing" ] }, { "id": "pagonas-2025-cortex-workflow-serving", "title": "Cortex: Workflow-Aware Resource Pooling and Scheduling for Agentic Serving", "authors": [ "Nikos Pagonas", "Yeounoh Chung", "Kostis Kaffes", "Arvind Krishnamurthy" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.14126", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "subagent", "learning_regime": "n/a", "signals_used": [ "workflow stages" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Position/design paper arguing for workflow-aware resource pooling where each agentic stage gets its own resource pool and scheduling, reducing inter-stage interference and improving KV utilization; outlines malleable resources, speculative branches and shared multi-tier agent-state caches.", "key_idea": "Stage-isolated resource pools for agentic serving.", "relevance_score": 2, "relevance_rationale": "Design direction for agent-native serving.", "found_via": "snowball: cited-by luo-2025-autellix", "cites_or_cited_by": [] }, { "id": "pahuja-2025-explorer", "title": "Explorer: Scaling Exploration-driven Web Trajectory Synthesis for Multimodal Web Agents", "authors": [ "Vardaan Pahuja", "Yadong Lu", "Corby Rosset", "Boyu Gou", "Arindam Mitra", "Spencer Whitehead", "Yu Su", "Ahmed Awadallah" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.11357", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "exploration-driven synthetic trajectories" ], "cost_reduction_reported": "Average synthesis cost 28 cents per successful trajectory", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Scalable recipe producing 94K multimodal web trajectories at 28 cents each, used to train web agents.", "key_idea": "Cheap large-scale web trajectory synthesis.", "relevance_score": 2, "relevance_rationale": "Data cost evidence.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "peng-2025-agentic-reward-modeling", "title": "Agentic Reward Modeling: Integrating Human Preferences with Verifiable Correctness Signals for Reliable Reward Systems", "authors": [ "Hao Peng", "Yunjia Qi", "Xiaozhi Wang", "Zijun Yao", "Bin Xu", "Lei Hou", "Juanzi Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.19328", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "reward model", "verifiable signals" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RewardAgent combines preference reward models with verifiable factuality and instruction-following checks, outperforming vanilla RMs and producing better DPO training pairs.", "key_idea": "Combine RM with verifiable correctness checkers.", "relevance_score": 2, "relevance_rationale": "Composite reward signal.", "found_via": "arXiv API title search: agentic reward modeling", "cites_or_cited_by": [] }, { "id": "qian-2025-toolrl", "title": "ToolRL: Reward is All Tool Learning Needs", "authors": [ "Cheng Qian", "Emre Can Acikgoz", "Qi He", "Hongru Wang", "Xiusi Chen", "Dilek Hakkani-Tür", "Gokhan Tur", "Heng Ji" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.13958", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "fine-grained tool reward" ], "cost_reduction_reported": "17% over base and 15% over SFT models", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies reward design for tool-use RL and shows principled rewards with GRPO beat SFT.", "key_idea": "Reward design for tool-use RL.", "relevance_score": 2, "relevance_rationale": "RL specialization.", "found_via": "keyword: RL tool use", "cites_or_cited_by": [] }, { "id": "redis-langcache", "title": "Redis LangCache (semantic caching service)", "authors": [ "Redis" ], "year": 2025, "venue_or_source": "Redis documentation", "source_type": "product", "url": "https://redis.io/docs/latest/develop/ai/context-engine/langcache/", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "prompt embedding similarity threshold" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A managed or self-hosted semantic cache that embeds incoming prompts, returns a stored response when a semantically similar prompt was answered before, and otherwise stores the new LLM result. It exposes similarity thresholds, TTLs and eviction policies, supports caching intermediate results in multi-step agent workflows, and reports hit rates and savings.", "key_idea": "Managed semantic caching with tunable similarity thresholds in front of LLM calls.", "relevance_score": 2, "relevance_rationale": "Productised caching lever; no quantitative results on the page.", "found_via": "direct: redis.io docs", "cites_or_cited_by": [] }, { "id": "ren-2025-predicting-zero-shot-success", "title": "Predicting Language Models' Success at Zero-Shot Probabilistic Prediction", "authors": [ "Kevin Ren", "Santiago Cortes-Gomez", "Carlos Andrés Carmona Patiño", "Ananya Joshi", "Ruiqi Lyu", "Jingjing Tang", "Alistair Turcan", "Khurram Yamin", "Steven Yu-Ping Wu", "Bryan Wilder" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.15356", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "predicted probabilities", "label-free task metrics" ], "cost_reduction_reported": null, "benchmarks": [ "tabular prediction tasks" ], "models_studied": [], "production_evidence": false, "summary": "Large study of LLM zero-shot probabilistic prediction on tabular tasks; builds label-free metrics that predict at the task level whether an LLM will do well.", "key_idea": "Label-free task-level prediction of LLM suitability.", "relevance_score": 2, "relevance_rationale": "Background on predicting LLM success without labels.", "found_via": "OpenAlex title: predicting success language model", "cites_or_cited_by": [] }, { "id": "song-2025-prmbench", "title": "PRMBench: A Fine-grained and Challenging Benchmark for Process-Level Reward Models", "authors": [ "Mingyang Song", "Zhaochen Su", "Xiaoye Qu", "Jiawei Zhou", "Yu Cheng" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.03124", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "process reward" ], "cost_reduction_reported": null, "benchmarks": [ "PRMBench" ], "models_studied": [], "production_evidence": false, "summary": "Fine-grained benchmark with 6,216 problems and 83,456 step labels for evaluating PRM error detection across simplicity, soundness and sensitivity.", "key_idea": "Benchmark for PRM error detection.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "keyword: process reward model benchmark", "cites_or_cited_by": [] }, { "id": "spangher-2025-chatbot-arena-estimate", "title": "Chatbot Arena Estimate: towards a generalized performance benchmark for LLM capabilities", "authors": [ "Lucas Spangher", "Tianle Li", "William F. Arnold", "Nick Masiewicki", "Xerxes Dotiwalla", "Rama Kumar Pasumarthi", "Peter Grabowski", "Eugene Ie", "Daniel Gruhl" ], "year": 2025, "venue_or_source": "NAACL 2025 (Industry Track)", "source_type": "paper", "url": "https://doi.org/10.18653/v1/2025.naacl-industry.77", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "Arena preference data" ], "cost_reduction_reported": null, "benchmarks": [ "Chatbot Arena" ], "models_studied": [], "production_evidence": false, "summary": "Industry-track paper proposing a way to estimate Chatbot Arena-style performance as a general benchmark for LLM capabilities (abstract not retrieved; described from title and venue only).", "key_idea": "Estimating Arena scores as a benchmark.", "relevance_score": 2, "relevance_rationale": "Performance prediction from preference data; limited detail verified.", "found_via": "OpenAlex title: Chatbot Arena", "cites_or_cited_by": [ "chiang-2024-chatbot-arena" ] }, { "id": "su-2025-underthinking-overthinking-length", "title": "Between Underthinking and Overthinking: An Empirical Study of Reasoning Length and correctness in LLMs", "authors": [ "Jinyan Su", "Jennifer Healey", "Preslav Nakov", "Claire Cardie" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.00127", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "preference for shorter responses" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Empirical study showing LLMs overthink easy problems and underthink hard ones, i.e. misjudge difficulty; preferring shorter responses via preference optimisation cuts length with acceptable accuracy.", "key_idea": "Models miscalibrate reasoning length to difficulty.", "relevance_score": 2, "relevance_rationale": "Background evidence.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "team-2025-kimi-k15", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "authors": [ "Kimi Team", "Angang Du", "Bofei Gao", "Bowei Xing", "Changjiu Jiang", "Cheng Chen", "Cheng Li", "Chenjun Xiao", "Chenzhuang Du", "Chonghua Liao", "Chuning Tang", "Congcong Wang", "Dehao Zhang", "Enming Yuan", "Enzhe Lu", "et al." ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.12599", "code_url": null, "topic_clusters": [ "C8", "C9" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "length penalty (long2short)" ], "cost_reduction_reported": "long2short short-CoT results: 60.8 AIME, 94.6 MATH500, 47.3 LiveCodeBench, outperforming GPT-4o and Claude Sonnet 3.5 by up to +550%", "benchmarks": [ "AIME", "MATH 500", "Codeforces", "MathVista", "LiveCodeBench" ], "models_studied": [ "Kimi k1.5", "GPT-4o", "Claude Sonnet 3.5" ], "production_evidence": false, "summary": "Technical report on RL training of Kimi k1.5, including long2short methods that transfer long-CoT capability into efficient short-CoT models.", "key_idea": "long2short transfer to cheaper short-CoT reasoning.", "relevance_score": 2, "relevance_rationale": "Background; industrial recipe for concise reasoning.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "together-2025-atlas-adaptive-speculator", "title": "AdapTive-LeArning Speculator System (ATLAS): A New Paradigm in LLM Inference via Runtime-Learning Accelerators", "authors": [ "Junxiong Wang", "Shirley Wu", "Zelei Shao", "Vikranth Srivatsa", "Jue Wang", "Roy Yuan", "Qingyang Wu", "Alpay Ariyak", "Rupert Wu", "Wai Tong Chung", "Chenfeng Xu", "Yonatan Oren", "Pragaash Ponnusamy", "Yineng Zhang", "Avner May", "Leon Song", "Tri Dao", "Percy Liang", "Ce Zhang", "Ben Athiwaratkun" ], "year": 2025, "venue_or_source": "Together AI blog", "source_type": "blog", "url": "https://www.together.ai/blog/adaptive-learning-speculator-system-atlas", "code_url": null, "topic_clusters": [ "C5", "C11" ], "routing_granularity": "token", "learning_regime": "hybrid", "signals_used": [ "live traffic token distribution", "draft acceptance rate" ], "cost_reduction_reported": "Up to 500 TPS on DeepSeek-V3.1 and 460 TPS on Kimi-K2 when fully adapted, 2.65x faster than standard decoding; RL rollouts: acceptance rate rose from <10% to >80% over 1.4k steps (vendor-reported)", "benchmarks": [ "Together internal throughput benchmarks" ], "models_studied": [ "DeepSeek-V3.1", "Kimi-K2" ], "production_evidence": false, "summary": "ATLAS pairs a static speculator with a lightweight speculator that keeps learning from live traffic, plus a confidence-aware controller that chooses between them and adjusts lookahead. The adaptive component tracks workload drift, which the authors also apply to RL rollouts where the policy distribution shifts.", "key_idea": "Speculative-decoding drafters that adapt online to the serving workload.", "relevance_score": 2, "relevance_rationale": "Serving-cost lever with online adaptation to traffic.", "found_via": "direct: together.ai/blog index", "cites_or_cited_by": [] }, { "id": "together-2025-batch-api", "title": "Introducing the Together AI Batch API: Process Thousands of LLM Requests at 50% Lower Cost", "authors": [ "Alay Dilipbhai Shah", "Rajas Bansal", "Mark Jones", "Yogish Baliga", "Ted Cui", "Ameen Patel", "Derek Dowling", "Jordan Kail", "Justin Foutts", "Bryan Wade", "Will Van Eaton", "Anirudh Jain" ], "year": 2025, "venue_or_source": "Together AI blog", "source_type": "blog", "url": "https://www.together.ai/blog/batch-api", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) 50% lower cost than real-time API, best-effort 24h completion", "benchmarks": [], "models_studied": [ "DeepSeek-R1", "DeepSeek-V3", "Llama 8B-405B", "Qwen3-235B", "Mixtral" ], "production_evidence": false, "summary": "Batch API for open models at half price, up to 50,000 requests per file and separate rate limits.", "key_idea": "Open-model batch inference at 50% price.", "relevance_score": 2, "relevance_rationale": "Batch lever for open-weight models.", "found_via": "seed: Together blogs on cost", "cites_or_cited_by": [] }, { "id": "varda-2025-cloudflare-code-mode", "title": "Code Mode: the better way to use MCP", "authors": [ "Kenton Varda", "Sunil Pai" ], "year": 2025, "venue_or_source": "Cloudflare blog", "source_type": "blog", "url": "https://blog.cloudflare.com/code-mode/", "code_url": null, "topic_clusters": [ "C10", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Cloudflare converts MCP tools into a TypeScript API and has the LLM write code against it, run in a sandboxed isolate, rather than emitting tool-call tokens. Chained calls no longer route every intermediate result back through the model, saving tokens and time, and models handle more and more complex tools because they have seen far more code than tool-call examples.", "key_idea": "Agents call tools more efficiently by writing code against a generated API than by emitting tool calls.", "relevance_score": 2, "relevance_rationale": "Architecture-level context lever (cited by Anthropic's code-execution post); qualitative.", "found_via": "snowball: cited by anthropic-2025-code-execution-with-mcp", "cites_or_cited_by": [ "anthropic-2025-code-execution-with-mcp" ] }, { "id": "vllm-2025-vllm-router", "title": "vLLM Router: A High-Performance and Prefill/Decode Aware Load Balancer for Large-scale Serving", "authors": [ "vLLM Team" ], "year": 2025, "venue_or_source": "vLLM blog", "source_type": "blog", "url": "https://blog.vllm.ai/blog/2025-12-13-vllm-router-release", "code_url": "https://github.com/vllm-project/router", "topic_clusters": [ "C11" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "KV-cache/prefix affinity", "load" ], "cost_reduction_reported": "Llama 3.1 8B with 8 prefill + 8 decode pods: request throughput 25% higher than llm-d and 100% higher than the K8s-native load balancer (project-reported)", "benchmarks": [ "project throughput/TTFT benchmarks" ], "models_studied": [ "Llama 3.1 8B", "DeepSeek V3" ], "production_evidence": false, "summary": "Introduces a Rust load balancer for vLLM fleets, forked from the SGLang model gateway, with cache-aware and prefill/decode-disaggregation-aware request placement. Benchmarks compare throughput and TTFT against llm-d and Kubernetes-native balancing.", "key_idea": "Replica-level routing that is aware of KV-cache locality and P/D disaggregation raises throughput per GPU.", "relevance_score": 2, "relevance_rationale": "Serving infrastructure background (replica routing, not model routing).", "found_via": "direct: blog.vllm.ai index", "cites_or_cited_by": [] }, { "id": "wang-2025-klear-agentforge", "title": "Klear-AgentForge: Forging Agentic Intelligence through Posttraining Scaling", "authors": [ "Qi Wang", "Hongzhi Zhang", "Jia Fu", "Kai Fu", "Yahui Liu", "Tinghai Zhang", "Chenxi Sun", "Gangwei Jiang", "Jingyi Tang", "Xingguang Ji", "Yang Yue", "Jingyuan Zhang", "Fuzheng Zhang", "Kun Gai", "Guorui Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2511.05951", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "SFT plus multi-turn RL" ], "cost_reduction_reported": null, "benchmarks": [ "tool-use and coding benchmarks" ], "models_studied": [ "Qwen3-8B" ], "production_evidence": false, "summary": "Open post-training pipeline (SFT and multi-turn RL) producing an 8B agentic model competitive with much larger models.", "key_idea": "Full open agent post-training pipeline.", "relevance_score": 2, "relevance_rationale": "Recipe reference.", "found_via": "hf-papers search: fine-tuning small agent trajectories", "cites_or_cited_by": [] }, { "id": "wang-2025-ragen-starpo", "title": "RAGEN: Understanding Self-Evolution in LLM Agents via Multi-Turn Reinforcement Learning", "authors": [ "Zihan Wang", "Kangrui Wang", "Qineng Wang", "Pingyue Zhang", "Linjie Li", "Zhengyuan Yang", "Kefan Yu", "Minh Nhat Nguyen", "Licheng Liu", "Eli Gottlieb", "Monica Lam", "Yiping Lu", "Kyunghyun Cho", "Jiajun Wu", "Li Fei-Fei", "Lijuan Wang", "Yejin Choi", "Manling Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.20073", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-online", "signals_used": [ "trajectory-level rewards" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes StarPO for trajectory-level agent RL and the RAGEN system, identifying the 'echo trap' instability and remedies for multi-turn RL.", "key_idea": "Trajectory-level RL and its failure modes.", "relevance_score": 2, "relevance_rationale": "Practical RL recipe knowledge.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "wang-2025-talkhier", "title": "Talk Structurally, Act Hierarchically: A Collaborative Framework for LLM Multi-Agent Systems", "authors": [ "Zhao Wang", "Sota Moriyama", "Wei-Yao Wang", "Briti Gangopadhyay", "Shingo Takamatsu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2502.11098", "code_url": "https://github.com/sony/talkhier", "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "open-domain QA", "domain-specific selective questioning", "advertisement text generation" ], "models_studied": [ "GPT-4o", "OpenAI-o1 (baseline)" ], "production_evidence": false, "summary": "TalkHier introduces a structured communication protocol and hierarchical refinement for multi-agent LLM systems and reports gains over o1, AgentVerse and majority voting on QA and ad-text tasks. Relevant mainly as an orchestration pattern; cost is not reduced.", "key_idea": "Structured messages plus hierarchical evaluator teams.", "relevance_score": 2, "relevance_rationale": "Background orchestration pattern.", "found_via": "arXiv title search", "cites_or_cited_by": [] }, { "id": "wang-2025-underthinking-o1", "title": "Thoughts Are All Over the Place: On the Underthinking of o1-Like LLMs", "authors": [ "Yue Wang", "Qiuzhi Liu", "Jiahao Xu", "Tian Liang", "Xingyu Chen", "Zhiwei He", "Linfeng Song", "Dian Yu", "Juntao Li", "Zhuosheng Zhang", "Rui Wang", "Zhaopeng Tu", "Haitao Mi", "Dong Yu" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.18585", "code_url": null, "topic_clusters": [ "C9" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "thought-switching penalty" ], "cost_reduction_reported": null, "benchmarks": [ "three challenging math test sets" ], "models_studied": [ "two open-source o1-like models", "DeepSeek-R1" ], "production_evidence": false, "summary": "Identifies underthinking, where o1-like models switch reasoning paths too often without exploring them, and proposes a decoding penalty on thought switching that improves accuracy without fine-tuning.", "key_idea": "Penalise premature thought switching.", "relevance_score": 2, "relevance_rationale": "Complementary failure mode to overthinking.", "found_via": "web search: Do NOT Think That Much (related results)", "cites_or_cited_by": [] }, { "id": "wei-2025-browsecomp", "title": "BrowseComp: A Simple Yet Challenging Benchmark for Browsing Agents", "authors": [ "Jason Wei", "Zhiqing Sun", "Spencer Papay", "Scott McKinney", "Jeffrey Han", "Isa Fulford", "Hyung Won Chung", "Alex Tachard Passos", "William Fedus", "Amelia Glaese" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.12516", "code_url": "https://github.com/openai/simple-evals", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "BrowseComp (1,266 questions)" ], "models_studied": [ "GPT-4o", "GPT-4.5", "OpenAI Deep Research" ], "production_evidence": false, "summary": "OpenAI benchmark of 1,266 hard-to-find, entangled questions whose short answers are easy to verify, testing persistent and creative web browsing. Non-browsing models score near zero while an agent trained for browsing does much better.", "key_idea": "Short-answer verifiable benchmark for deep browsing agents.", "relevance_score": 2, "relevance_rationale": "Used in multi-agent research-system cost analyses (e.g., token usage explains variance).", "found_via": "web search: BrowseComp", "cites_or_cited_by": [ "hadfield-2025-multi-agent-research-system" ] }, { "id": "wu-2025-webdancer", "title": "WebDancer: Towards Autonomous Information Seeking Agency", "authors": [ "Jialong Wu", "Baixuan Li", "Runnan Fang", "Wenbiao Yin", "Liwen Zhang", "Zhengwei Tao", "Dingchu Zhang", "Zekun Xi", "Yong Jiang", "Pengjun Xie", "Fei Huang", "Jingren Zhou" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.22648", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "sampled trajectories", "SFT then RL" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Four-stage recipe (data construction, trajectory sampling, SFT cold start, RL) for autonomous information-seeking agents.", "key_idea": "SFT-then-RL recipe for web agents.", "relevance_score": 2, "relevance_rationale": "Training recipe.", "found_via": "keyword: agent distillation", "cites_or_cited_by": [] }, { "id": "xie-2025-swe-fixer", "title": "SWE-Fixer: Training Open-Source LLMs for Effective and Efficient GitHub Issue Resolution", "authors": [ "Chengxing Xie", "Bowen Li", "Chang Gao", "He Du", "Wai Lam", "Difan Zou", "Kai Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.05040", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "110K issues-patch pairs" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Open pipeline-based issue resolver with lightweight retrieval and a fine-tuned editing model for efficient issue resolution.", "key_idea": "Efficient fine-tuned SWE pipeline.", "relevance_score": 2, "relevance_rationale": "Specialization for coding tasks.", "found_via": "keyword: SFT on agent traces", "cites_or_cited_by": [] }, { "id": "xu-2025-a-mem", "title": "A-MEM: Agentic Memory for LLM Agents", "authors": [ "Wujiang Xu", "Zujie Liang", "Kai Mei", "Hang Gao", "Juntao Tan", "Yongfeng Zhang" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2502.12110", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "Zettelkasten-style linked notes" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "six foundation models" ], "production_evidence": false, "summary": "Agentic memory system that creates linked notes for each experience and lets the agent update existing memories as new ones arrive, improving over prior memory baselines across six foundation models.", "key_idea": "Self-organizing linked memory for agents.", "relevance_score": 2, "relevance_rationale": "Memory background; limited cost analysis.", "found_via": "snowball: related work of chhikara-2025-mem0", "cites_or_cited_by": [ "chhikara-2025-mem0" ] }, { "id": "xu-2025-doubly-robust-alignment", "title": "Doubly Robust Alignment for Large Language Models", "authors": [ "Erhan Xu", "Kai Ye", "Hongyi Zhou", "Luhan Zhu", "Francesco Quinzan", "Chengchun Shi" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2506.01183", "code_url": "https://github.com/DRPO4LLM/DRPO4LLM", "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "preference model", "reference policy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Doubly robust preference optimization for RLHF that stays consistent if either the preference model or the reference policy is correct.", "key_idea": "Doubly robust RLHF.", "relevance_score": 2, "relevance_rationale": "OPE-style robustness in LLM training; background.", "found_via": "OpenAlex title: doubly robust language", "cites_or_cited_by": [] }, { "id": "xu-2025-robustflow", "title": "RobustFlow: Towards Robust Agentic Workflow Generation", "authors": [ "Shengxiang Xu", "Jiayi Zhang", "Shimin Di", "Yuyu Luo", "Liang Yao", "Hanmo Liu", "Jia Zhu", "Min-Ling Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.21834", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "robustness to instruction variation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "RobustFlow studies the robustness of generated agentic workflows to semantically equivalent task instructions and trains generators to produce consistent workflows.", "key_idea": "Robustness of automated workflow generation.", "relevance_score": 2, "relevance_rationale": "Background on reliability of workflow optimizers.", "found_via": "search: ScoreFlow MaAS agentic supernet arXiv", "cites_or_cited_by": [] }, { "id": "yang-2025-swe-smith", "title": "SWE-smith: Scaling Data for Software Engineering Agents", "authors": [ "John Yang", "Kilian Leret", "Carlos E. Jimenez", "Alexander Wettig", "Kabir Khandpur", "Yanzhe Zhang", "Binyuan Hui", "Ofir Press", "Ludwig Schmidt", "Diyi Yang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.21798", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "synthetic task instances", "expert trajectories" ], "cost_reduction_reported": "SWE-agent-LM-32B reaches 40.2% Pass@1 on SWE-bench Verified", "benchmarks": [ "SWE-bench Verified" ], "models_studied": [ "Qwen2.5-Coder-32B" ], "production_evidence": false, "summary": "Synthesizes 50k SWE task instances from 128 repositories and trains SWE-agent-LM-32B on resulting trajectories.", "key_idea": "Scalable synthetic SWE training data.", "relevance_score": 2, "relevance_rationale": "Data scaling for coding agents.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "yoon-2025-reasoning-models-confidence", "title": "Reasoning Models Better Express Their Confidence", "authors": [ "Dongkeun Yoon", "Seungone Kim", "Sohee Yang", "Sunkyoung Kim", "Soyeon Kim", "Yongil Kim", "Eunbi Choi", "Yireun Kim", "Minjoon Seo" ], "year": 2025, "venue_or_source": "NeurIPS 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2505.14489", "code_url": null, "topic_clusters": [ "C2", "C7", "C9" ], "routing_granularity": "n/a", "learning_regime": "training-free", "signals_used": [ "verbalised confidence" ], "cost_reduction_reported": null, "benchmarks": [ "six datasets" ], "models_studied": [ "six reasoning models" ], "production_evidence": false, "summary": "Reasoning models are better calibrated than non-reasoning counterparts in 33 of 36 settings, with calibration improving as the chain of thought unfolds due to slow-thinking behaviours.", "key_idea": "Slow thinking improves confidence calibration.", "relevance_score": 2, "relevance_rationale": "Signal quality for escalation from reasoning models.", "found_via": "snowball: cited-by jitkrittum/gupta token-level cascades (2404.10136)", "cites_or_cited_by": [ "gupta-2024-lm-cascades-token-uncertainty" ] }, { "id": "yuan-2025-agent-r", "title": "Agent-R: Training Language Model Agents to Reflect via Iterative Self-Training", "authors": [ "Siyu Yuan", "Zehui Chen", "Zhiheng Xi", "Junjie Ye", "Zhengyin Du", "Jiecao Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.11425", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "hybrid", "signals_used": [ "MCTS error correction" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Iterative self-training with MCTS-constructed revision trajectories teaches agents to reflect and recover from errors.", "key_idea": "Self-training for reflection.", "relevance_score": 2, "relevance_rationale": "Self-improvement.", "found_via": "keyword: fine-tuning small agents", "cites_or_cited_by": [] }, { "id": "yue-2025-dont-overthink-survey", "title": "Don't Overthink It: A Survey of Efficient R1-style Large Reasoning Models", "authors": [ "Linan Yue", "Yichao Du", "Yizhi Wang", "Weibo Gao", "Fangzhou Yao", "Li Wang", "Ye Liu", "Ziyu Xu", "Qi Liu", "Shimin Di", "Min-Ling Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2508.02120", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of efficient reasoning methods for R1-style large reasoning models aimed at shortening reasoning paths without losing accuracy.", "key_idea": "R1-style efficiency survey.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "zhang-2025-dgm", "title": "Darwin Godel Machine: Open-Ended Evolution of Self-Improving Agents", "authors": [ "Jenny Zhang", "Shengran Hu", "Cong Lu", "Robert Lange", "Jeff Clune" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.22954", "code_url": "https://github.com/jennyzzt/dgm", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "coding benchmark score" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "The Darwin Gödel Machine is a self-improving coding agent that rewrites its own code and validates changes on coding benchmarks, maintaining an open-ended archive of agents.", "key_idea": "Open-ended evolutionary self-modification of agent code.", "relevance_score": 2, "relevance_rationale": "Automated agent design background; very compute-heavy.", "found_via": "search: Darwin Gödel Machine self-improving agent design arXiv", "cites_or_cited_by": [ "hu-2024-adas" ] }, { "id": "zhang-2025-flora-bench", "title": "GNNs as Predictors of Agentic Workflow Performances", "authors": [ "Yuanshuo Zhang", "Yuchen Hou", "Bohan Tang", "Shuo Chen", "Muhan Zhang", "Xiaowen Dong", "Siheng Chen" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2503.11301", "code_url": null, "topic_clusters": [ "C4", "C6" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "GNN over workflow graph" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies GNNs as cheap predictors of agentic workflow performance and builds a benchmark for this prediction task, to reduce reliance on repeated LLM execution during workflow optimization.", "key_idea": "GNN surrogates for workflow performance.", "relevance_score": 2, "relevance_rationale": "Surrogate evaluation for config search.", "found_via": "search: agentic workflow performance predictor surrogate cheap evaluation arXiv", "cites_or_cited_by": [] }, { "id": "zhang-2025-lessons-developing-prms", "title": "The Lessons of Developing Process Reward Models in Mathematical Reasoning", "authors": [ "Zhenru Zhang", "Chujie Zheng", "Yangzhen Wu", "Beichen Zhang", "Runji Lin", "Bowen Yu", "Dayiheng Liu", "Jingren Zhou", "Junyang Lin" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2501.07301", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward", "LLM-as-judge labels" ], "cost_reduction_reported": null, "benchmarks": [ "ProcessBench", "best-of-N math" ], "models_studied": [ "Qwen2.5-Math" ], "production_evidence": false, "summary": "Finds Monte Carlo-estimated step labels produce weaker PRMs than LLM-as-judge or human labels, and identifies biases in best-of-N evaluation of PRMs; proposes consensus filtering.", "key_idea": "MC step labels are noisy; combine with judge consensus.", "relevance_score": 2, "relevance_rationale": "Practical guidance for building PRMs.", "found_via": "keyword: process reward model", "cites_or_cited_by": [] }, { "id": "zhao-2025-a2flow", "title": "A2Flow: Automating Agentic Workflow Generation via Self-Adaptive Abstraction Operators", "authors": [ "Mingming Zhao", "Xiaokang Wei", "Yuanqi Shao", "Kaiwen Zhou", "Lin Yang", "Siwei Rao", "Junhui Zhan", "Zhitang Chen" ], "year": 2025, "venue_or_source": "AAAI 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2511.20693", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "execution feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "A2Flow automatically extracts reusable abstract operators from demonstrations for workflow generation, removing the need for manually defined operators as in AFlow.", "key_idea": "Self-adaptive operator abstraction for workflow search.", "relevance_score": 2, "relevance_rationale": "Extension of AFlow.", "found_via": "snowball: cited-by saadfalcon-2024-archon", "cites_or_cited_by": [ "zhang-2024-aflow" ] }, { "id": "zhao-2025-cochain", "title": "Cochain: Balancing Insufficient and Excessive Collaboration in LLM Agent Workflows", "authors": [ "Jiaxing Zhao", "Hongbin Xie", "Yuzhen Lei", "Xuan Song", "Zhuoran Shi", "Lianxin Li", "Shuangxue Liu", "Linguo Xie", "Haoran Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2505.10936", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": "Small model with Cochain outperforms GPT-4 in expert evaluation", "benchmarks": [ "business workflow datasets" ], "models_studied": [], "production_evidence": false, "summary": "Cochain replaces costly multi-agent collaboration in business workflows with a knowledge graph and a prompts tree that inject cross-stage knowledge into a single model's prompts, beating multi-agent baselines at lower cost.", "key_idea": "Prompt-level cross-stage knowledge instead of multi-agent chatter.", "relevance_score": 2, "relevance_rationale": "Alternative to MAS for workflows.", "found_via": "arXiv API search", "cites_or_cited_by": [] }, { "id": "zhen-2025-taming-the-titans", "title": "Taming the Titans: A Survey of Efficient LLM Inference Serving", "authors": [ "Ranran Zhen", "Juntao Li", "Yixin Ji", "Zhenlin Yang", "Tong Liu", "Qingrong Xia", "Xinyu Duan", "Zhefeng Wang", "Baoxing Huai", "Min Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2504.19720", "code_url": null, "topic_clusters": [ "C11", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of efficient LLM inference serving at instance level (placement, scheduling, length prediction, storage, disaggregation), cluster level (deployment, load balancing, cloud) and emerging scenarios.", "key_idea": "Map of LLM serving techniques.", "relevance_score": 2, "relevance_rationale": "Orientation survey for serving.", "found_via": "KV cache compression survey LLM inference efficiency", "cites_or_cited_by": [ "kwon-2023-vllm" ] }, { "id": "zheng-2025-survey-process-reward-models", "title": "A Survey of Process Reward Models: From Outcome Signals to Process Supervisions for Large Language Models", "authors": [ "Congmin Zheng", "Jiachen Zhu", "Ou, Zhuoying", "Yuxiang Chen", "Kangning Zhang", "Rong Shan", "Zeyu Zheng", "Mengyue Yang", "Jianghao Lin", "Yong Hai Yu", "Weinan Zhang" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2510.08049", "code_url": null, "topic_clusters": [ "C7", "C15" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "process reward" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of process reward models covering data generation, model building and usage from outcome to process supervision.", "key_idea": "PRM survey.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "WebSearch: process reward model for LLM agents", "cites_or_cited_by": [] }, { "id": "zhu-2025-concise-adaptive-thinking-survey", "title": "Towards Concise and Adaptive Thinking in Large Reasoning Models: A Survey", "authors": [ "Jason Zhu", "Hongyu Li" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2507.09662", "code_url": null, "topic_clusters": [ "C9", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of concise and adaptive thinking in LRMs, including methods to switch between fast and slow thinking based on input difficulty.", "key_idea": "Adaptive thinking survey.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ding-2024-hybrid-llm", "ong-2024-routellm" ] }, { "id": "zhu-2025-planner-r1", "title": "Planner-R1: Reward Shaping Enables Efficient Agentic RL with Smaller LLMs", "authors": [ "Siyu Zhu", "Yanbin Jiang", "Hejian Sang", "Shao Tang", "Qingquan Song", "Biao He", "Rohit Jain", "Zhipeng Wang", "Alborz Geramifard" ], "year": 2025, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2509.25779", "code_url": null, "topic_clusters": [ "C3", "C8" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "dense process-level reward" ], "cost_reduction_reported": "8B models 3.5x more compute-efficient and 1.5x more memory-efficient than 32B with competitive performance; 56.9% final pass vs GPT-5 21.2% on TravelPlanner", "benchmarks": [ "TravelPlanner" ], "models_studied": [ "8B and 32B models", "GPT-5 (baseline)" ], "production_evidence": false, "summary": "Agentic RL on TravelPlanner with reward shaping reaches 56.9% final pass with 180 training queries; shaped dense rewards let 8B models match 32B at much lower compute.", "key_idea": "Reward shaping makes small models efficient agentic planners.", "relevance_score": 2, "relevance_rationale": "Background on training small agent models.", "found_via": "snowball: cited-by belcak-2025-slms-future-agentic-ai", "cites_or_cited_by": [ "belcak-2025-slms-future-agentic-ai" ] }, { "id": "ziems-2025-mmgrpo", "title": "Composing Policy Gradients and Prompt Optimization for Language Model Programs", "authors": [ "Noah Ziems", "Dilara Soylu", "Lakshya A Agrawal", "Isaac Miller", "Liheng Lai", "Chen Qian", "Kaiqiang Song", "Meng Jiang", "Dan Klein", "Matei Zaharia", "Karel D'Oosterlinck", "Christopher Potts", "Omar Khattab" ], "year": 2025, "venue_or_source": "ACM CAIS 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2508.04660", "code_url": null, "topic_clusters": [ "C4", "C8" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "task reward" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Composes policy-gradient training (a multi-module GRPO) with prompt optimization for LM programs, studying how RL on module weights and prompt search combine.", "key_idea": "Combine multi-module GRPO with prompt optimization.", "relevance_score": 2, "relevance_rationale": "Background on joint weight/prompt optimization of compound systems.", "found_via": "search: BetterTogether fine-tuning and prompt optimization LM programs arXiv", "cites_or_cited_by": [ "soylu-2024-bettertogether" ] }, { "id": "abdelaziz-2024-granite-function-calling", "title": "Granite-Function Calling Model: Introducing Function Calling Abilities via Multi-task Learning of Granular Tasks", "authors": [ "Ibrahim Abdelaziz", "Kinjal Basu", "Mayank Agarwal", "Sadhana Kumaravel", "Matthew Stallone", "Rameswar Panda", "Yara Rizk", "GP Bhargav", "Maxwell Crouse", "Chulaka Gunasekara", "Shajith Ikbal", "Sachin Joshi", "Hima Karanam", "Vineet Kumar", "Asim Munawar", "Sumit Neelam", "Dinesh Raghu", "Udit Sharma", "Adriana Meza Soria", "Dheeraj Sreedhar", "Praveen Venkateswaran", "Merve Unuvar", "David Cox", "Salim Roukos", "Luis Lastras", "Pavan Kapanipathi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.00121", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "multi-task function-calling data" ], "cost_reduction_reported": null, "benchmarks": [ "BFCL", "out-of-domain function-calling datasets" ], "models_studied": [ "Granite-20B" ], "production_evidence": false, "summary": "IBM Granite-20B function-calling model trained with multi-task learning over granular function-calling tasks, compared with 15+ models.", "key_idea": "Enterprise open function-calling model.", "relevance_score": 2, "relevance_rationale": "Industry specialization.", "found_via": "keyword: function calling small language model", "cites_or_cited_by": [] }, { "id": "agrawal-2024-sarathi-serve", "title": "Taming Throughput-Latency Tradeoff in LLM Inference with Sarathi-Serve", "authors": [ "Amey Agrawal", "Nitin Kedia", "Ashish Panwar", "Jayashree Mohan", "Nipun Kwatra", "Bhargav S. Gulavani", "Alexey Tumanov", "Ramachandran Ramjee" ], "year": 2024, "venue_or_source": "OSDI 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2403.02310", "code_url": "https://github.com/microsoft/sarathi-serve", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "chunked prefill" ], "cost_reduction_reported": "serving capacity 2.6x higher for Mistral-7B on one A100 and up to 3.7x for Yi-34B on two A100s vs vLLM; up to 5.6x for Falcon-180B with pipeline parallelism", "benchmarks": [], "models_studied": [ "Mistral-7B", "Yi-34B", "Falcon-180B" ], "production_evidence": false, "summary": "Splits prefills into chunks and builds stall-free batches that add new requests without pausing ongoing decodes, improving throughput under tail-latency SLOs.", "key_idea": "Chunked prefill with stall-free batching.", "relevance_score": 2, "relevance_rationale": "Serving background.", "found_via": "Sarathi-Serve throughput latency tradeoff chunked prefill", "cites_or_cited_by": [ "kwon-2023-vllm" ] }, { "id": "cai-2024-medusa", "title": "Medusa: Simple LLM Inference Acceleration Framework with Multiple Decoding Heads", "authors": [ "Tianle Cai", "Yuhong Li", "Zhengyang Geng", "Hongwu Peng", "Jason D. Lee", "Deming Chen", "Tri Dao" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2401.10774", "code_url": "https://github.com/FasterDecoding/Medusa", "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "multiple decoding heads", "tree attention" ], "cost_reduction_reported": "Medusa-1 over 2.2x speedup without quality loss; Medusa-2 2.3-3.6x", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Adds extra decoding heads that predict several future tokens, verified jointly with tree attention; variants train heads on a frozen backbone or jointly.", "key_idea": "Self-drafting via extra decoding heads.", "relevance_score": 2, "relevance_rationale": "Speculative decoding without a separate draft model.", "found_via": "task seed", "cites_or_cited_by": [ "leviathan-2022-speculative-decoding", "li-2024-eagle", "xia-2024-speculative-decoding-survey" ] }, { "id": "chan-2024-cache-augmented-generation", "title": "Don't Do RAG: When Cache-Augmented Generation is All You Need for Knowledge Tasks", "authors": [ "Brian J Chan", "Chao-Ting Chen", "Jui-Hung Cheng", "Hen-Hsen Huang" ], "year": 2024, "venue_or_source": "WWW 2025 (short paper)", "source_type": "paper", "url": "https://arxiv.org/abs/2412.15605", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "preloaded KV cache" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes cache-augmented generation: for a bounded knowledge base, preload all documents into a long-context model once and cache the KV state, then answer queries without retrieval. Finds it matches or beats RAG in such settings with less complexity.", "key_idea": "Preload and cache a small knowledge base instead of retrieving.", "relevance_score": 2, "relevance_rationale": "Design alternative for agents with a stable, small knowledge base.", "found_via": "Cache-Augmented Generation don't do RAG long context KV preload", "cites_or_cited_by": [ "jin-2024-ragcache" ] }, { "id": "chen-2024-inside-eigenscore", "title": "INSIDE: LLMs' Internal States Retain the Power of Hallucination Detection", "authors": [ "Chao Chen", "Kai Liu", "Ze Chen", "Yi Gu", "Yue Wu", "Mingyuan Tao", "Zhihang Fu", "Jieping Ye" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.03744", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "internal states", "EigenScore" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Uses covariance eigenvalues of internal embeddings across samples (EigenScore) and feature clipping for hallucination detection.", "key_idea": "Internal-state consistency for hallucination detection.", "relevance_score": 2, "relevance_rationale": "UQ variant.", "found_via": "keyword: hallucination detection", "cites_or_cited_by": [] }, { "id": "fu-2024-lookahead-decoding", "title": "Break the Sequential Dependency of LLM Inference Using Lookahead Decoding", "authors": [ "Yichao Fu", "Peter Bailis", "Ion Stoica", "Hao Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.02057", "code_url": "https://github.com/hao-ai-lab/LookaheadDecoding", "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "n-gram pool from Jacobi iterations" ], "cost_reduction_reported": "up to 1.8x speedup on MT-bench and 4x with strong scaling on multiple GPUs for code completion", "benchmarks": [ "MT-bench", "code completion" ], "models_studied": [], "production_evidence": false, "summary": "Exact parallel decoding that trades extra per-step FLOPs for fewer decoding steps, without draft models or data stores.", "key_idea": "Draft-free exact parallel decoding.", "relevance_score": 2, "relevance_rationale": "Speculative-decoding alternative.", "found_via": "speculative decoding survey Xia unlocking efficiency", "cites_or_cited_by": [] }, { "id": "fu-2024-serverlessllm", "title": "ServerlessLLM: Low-Latency Serverless Inference for Large Language Models", "authors": [ "Yao Fu", "Leyang Xue", "Yeqi Huang", "Andrei-Octavian Brabete", "Dmitrii Ustiugov", "Yuvraj Patel", "Luo Mai" ], "year": 2024, "venue_or_source": "OSDI 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2401.14351", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "checkpoint locality" ], "cost_reduction_reported": "latency reduced 10-200x vs state-of-the-art serverless systems across LLM inference workloads", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Low-latency serverless LLM inference via fast multi-tier checkpoint loading, live migration, and locality-aware startup scheduling.", "key_idea": "Fast cold starts for serverless LLMs.", "relevance_score": 2, "relevance_rationale": "Serverless serving of many models, e.g., long-tail specialists.", "found_via": "task seed", "cites_or_cited_by": [] }, { "id": "godaddy-2024-llm-from-the-trenches", "title": "LLM From the Trenches: 10 Lessons Learned Operationalizing Models at GoDaddy", "authors": [ "GoDaddy" ], "year": 2024, "venue_or_source": "GoDaddy blog", "source_type": "blog", "url": "https://www.godaddy.com/resources/news/llm-from-the-trenches-10-lessons-learned-operationalizing-models-at-godaddy", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-3.5 Turbo", "GPT-4" ], "production_evidence": true, "summary": "Lessons from production support bots: prompts do not transfer across models, GPT-4 latency of 3-5s with 30s outliers, ~1% provider failures, structured-output failure rates by model, deterministic human-handoff, and interest in adaptive model selection.", "key_idea": "Operational realities of multi-model production.", "relevance_score": 2, "relevance_rationale": "Production reliability and latency data.", "found_via": "seed list: engineering case studies", "cites_or_cited_by": [] }, { "id": "gu-2024-survey-llm-as-a-judge", "title": "A Survey on LLM-as-a-Judge", "authors": [ "Jiawei Gu", "Xuhui Jiang", "Zhichao Shi", "Hexiang Tan", "Xuehao Zhai", "Chengjin Xu", "Wei Li", "Yinghan Shen", "Shengjie Ma", "Honghao Liu", "Yuanzhuo Wang", "Jian Guo" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.15594", "code_url": null, "topic_clusters": [ "C7", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judge" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of LLM-as-a-Judge covering reliability, bias mitigation and meta-evaluation.", "key_idea": "Survey on building reliable LLM judges.", "relevance_score": 2, "relevance_rationale": "Background reference.", "found_via": "keyword: LLM-as-a-judge survey", "cites_or_cited_by": [] }, { "id": "hu-2024-evomac", "title": "Self-Evolving Multi-Agent Collaboration Networks for Software Development", "authors": [ "Yue Hu", "Yuzhu Cai", "Yaxin Du", "Xinyu Zhu", "Xiangrui Liu", "Zijie Yu", "Yuchen Hou", "Shuo Tang", "Siheng Chen" ], "year": 2024, "venue_or_source": "ICLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2410.16946", "code_url": null, "topic_clusters": [ "C3", "C4" ], "routing_granularity": "subagent", "learning_regime": "training-free", "signals_used": [ "unit-test feedback" ], "cost_reduction_reported": null, "benchmarks": [ "rSDE-Bench", "HumanEval" ], "models_studied": [], "production_evidence": false, "summary": "EvoMAC treats a multi-agent coding team like a network and updates agents and their links at test time via textual backpropagation from test-team feedback. It introduces rSDE-Bench for software-level requirements and outperforms prior multi-agent coders.", "key_idea": "Textual backpropagation to evolve a coding MAS at test time.", "relevance_score": 2, "relevance_rationale": "MAS self-evolution; cost not the focus.", "found_via": "web search: seed", "cites_or_cited_by": [] }, { "id": "hu-2024-mab-generative-eval", "title": "A Multi-Armed Bandit Approach to Online Selection and Evaluation of Generative Models", "authors": [ "Xiaoyan Hu", "Ho-fung Leung", "Farzan Farnia" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.07451", "code_url": "https://github.com/yannxiaoyanhu/dgm-online-eval", "topic_clusters": [ "C5", "C13" ], "routing_granularity": "none", "learning_regime": "bandit-online", "signals_used": [ "FID", "Inception Score" ], "cost_reduction_reported": null, "benchmarks": [ "standard image datasets" ], "models_studied": [ "deep generative models" ], "production_evidence": false, "summary": "Frames sample-efficient online evaluation and selection of generative models as a bandit, with UCB algorithms for FD and IS metrics and regret bounds.", "key_idea": "UCB bandits to pick the best generative model with minimal samples.", "relevance_score": 2, "relevance_rationale": "Background on bandit-based online model evaluation; image domain.", "found_via": "OpenAlex title: online model selection bandits", "cites_or_cited_by": [ "hu-2024-pak-ucb" ] }, { "id": "intercom-2024-fin-2-claude", "title": "Fin 2: Powered by Anthropic's Claude LLM", "authors": [ "Intercom" ], "year": 2024, "venue_or_source": "Intercom blog", "source_type": "blog", "url": "https://www.intercom.com/blog/fin-2/", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude" ], "production_evidence": true, "summary": "Announcement that Intercom's Fin 2 support agent runs on Anthropic's Claude, citing performance and reliability.", "key_idea": "Production model-provider switch.", "relevance_score": 2, "relevance_rationale": "Production context.", "found_via": "seed list: Intercom case study", "cites_or_cited_by": [] }, { "id": "jin-2024-ragcache", "title": "RAGCache: Efficient Knowledge Caching for Retrieval-Augmented Generation", "authors": [ "Chao Jin", "Zili Zhang", "Xuanlin Jiang", "Fangyue Liu", "Xin Liu", "Xuanzhe Liu", "Xin Jin" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.12457", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "document retrieval frequency" ], "cost_reduction_reported": "TTFT reduced by up to 4x and throughput improved by up to 2.1x vs vLLM integrated with Faiss", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Caches KV states of frequently retrieved documents in a knowledge tree spread across GPU and host memory, and overlaps retrieval with generation via speculative pipelining.", "key_idea": "Cache KV of popular retrieved documents across GPU/host memory.", "relevance_score": 2, "relevance_rationale": "RAG-specific KV reuse; background for agent retrieval steps.", "found_via": "RAGCache knowledge caching retrieval-augmented generation", "cites_or_cited_by": [ "chan-2024-cache-augmented-generation", "yao-2024-cacheblend" ] }, { "id": "juravsky-2024-hydragen", "title": "Hydragen: High-Throughput LLM Inference with Shared Prefixes", "authors": [ "Jordan Juravsky", "Bradley Brown", "Ryan Ehrlich", "Daniel Y. Fu", "Christopher Ré", "Azalia Mirhoseini" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.05099", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "shared-prefix attention decomposition" ], "cost_reduction_reported": "CodeLlama-13b end-to-end throughput up to 32x vs competitive baselines; 55% less inference time on competitive programming with tree-shaped sharing", "benchmarks": [], "models_studied": [ "CodeLlama-13b" ], "production_evidence": false, "summary": "Computes attention over a shared prefix once for a batch using matrix-matrix operations, then combines with per-sequence suffix attention.", "key_idea": "Batch attention over shared prefixes.", "relevance_score": 2, "relevance_rationale": "Efficient shared-prefix sampling (e.g., many agent samples).", "found_via": "snowball: related work of zheng-2023-sglang", "cites_or_cited_by": [ "zheng-2023-sglang" ] }, { "id": "kipnis-2024-metabench", "title": "metabench -- A Sparse Benchmark of Reasoning and Knowledge in Large Language Models", "authors": [ "Alex Kipnis", "Konstantinos Voudouris", "Luca M. Schulze Buschoff", "Eric R. Schulz" ], "year": 2024, "venue_or_source": "ICLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2407.12844", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "item-level accuracies from Open LLM Leaderboard", "IRT" ], "cost_reduction_reported": null, "benchmarks": [ "six Open LLM Leaderboard benchmarks" ], "models_studied": [ "Open LLM Leaderboard models" ], "production_evidence": false, "summary": "Builds a sparse benchmark from item-wise accuracies of many models on six Open LLM Leaderboard benchmarks, selecting a small item subset that predicts full benchmark scores.", "key_idea": "Small item subsets predict full-benchmark performance.", "relevance_score": 2, "relevance_rationale": "Background on cheap evaluation for model selection.", "found_via": "OpenAlex title: metabench; verified via arXiv id lookup", "cites_or_cited_by": [ "polo-2024-tinybenchmarks" ] }, { "id": "klarna-2024-ai-assistant", "title": "Klarna AI assistant handles two-thirds of customer service chats in its first month", "authors": [ "Klarna" ], "year": 2024, "venue_or_source": "Klarna press release", "source_type": "blog", "url": "https://www.klarna.com/international/press/klarna-ai-assistant-handles-two-thirds-of-customer-service-chats-in-its-first-month/", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) 2.3M chats (two-thirds of volume), work of ~700 FTE, resolution time 11 to <2 minutes, estimated $40M 2024 profit improvement", "benchmarks": [], "models_studied": [ "OpenAI-powered assistant" ], "production_evidence": true, "summary": "Company press release on its customer-service assistant's first month, claiming resolution speed, repeat-inquiry and satisfaction parity with humans. Press-release claims, not an engineering study.", "key_idea": "Business-impact claims of a support agent.", "relevance_score": 2, "relevance_rationale": "Production scale context; vendor/company-reported.", "found_via": "seed list: Klarna case study", "cites_or_cited_by": [] }, { "id": "kurtic-2024-bf16-or-death", "title": "\"Give Me BF16 or Give Me Death\"? Accuracy-Performance Trade-Offs in LLM Quantization", "authors": [ "Eldar Kurtic", "Alexandre Marques", "Shubhra Pandit", "Mark Kurtz", "Dan Alistarh" ], "year": 2024, "venue_or_source": "ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2411.02355", "code_url": null, "topic_clusters": [ "C11", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "FP8 W8A8 lossless across scales; tuned INT8 loses 1-3%; W4A16 most cost-efficient for synchronous deployments, W8A8 best for asynchronous continuous batching", "benchmarks": [ "academic benchmarks", "real-world tasks" ], "models_studied": [ "Llama-3.1 family" ], "production_evidence": false, "summary": "Large empirical study of FP8, INT8 and INT4 quantization on Llama-3.1 models measuring accuracy and vLLM inference performance, producing deployment guidelines.", "key_idea": "Quantization choice depends on deployment mode; FP8 is essentially free.", "relevance_score": 2, "relevance_rationale": "Practical cost-quality guidance for self-hosted models in agent systems.", "found_via": "quantization inference cost LLM serving tradeoff accuracy", "cites_or_cited_by": [ "frantar-2022-gptq", "lin-2023-awq" ] }, { "id": "lambert-2024-rewardbench", "title": "RewardBench: Evaluating Reward Models for Language Modeling", "authors": [ "Nathan Lambert", "Valentina Pyatkin", "Jacob Morrison", "LJ Miranda", "Bill Yuchen Lin", "Khyathi Chandu", "Nouha Dziri", "Sachin Kumar", "Tom Zick", "Yejin Choi", "Noah A. Smith", "Hannaneh Hajishirzi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2403.13787", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "reward model score" ], "cost_reduction_reported": null, "benchmarks": [ "RewardBench" ], "models_studied": [], "production_evidence": false, "summary": "Benchmark of prompt-chosen-rejected trios across chat, reasoning and safety for evaluating reward models.", "key_idea": "Standard reward model benchmark.", "relevance_score": 2, "relevance_rationale": "Background for selecting judges/RMs.", "found_via": "keyword: outcome reward model", "cites_or_cited_by": [] }, { "id": "levy-2024-st-webagentbench", "title": "ST-WebAgentBench: A Benchmark for Evaluating Safety and Trustworthiness in Web Agents", "authors": [ "Ido Levy", "Ben Wiesel", "Sami Marreed", "Alon Oved", "Avi Yaeli", "Nir Mashkif", "Segev Shlomov" ], "year": 2024, "venue_or_source": "ICLR 2026", "source_type": "paper", "url": "https://arxiv.org/abs/2410.06703", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "policy compliance" ], "cost_reduction_reported": null, "benchmarks": [ "ST-WebAgentBench (222 tasks)" ], "models_studied": [ "web agents" ], "production_evidence": false, "summary": "Enterprise-oriented web-agent benchmark scoring safety and trustworthiness under policies, with Completion-under-Policy and Risk Ratio metrics.", "key_idea": "Policy-compliant completion for enterprise web agents.", "relevance_score": 2, "relevance_rationale": "Enterprise evaluation dimension beyond accuracy.", "found_via": "web search: BrowserGym ecosystem", "cites_or_cited_by": [] }, { "id": "li-2024-eagle", "title": "EAGLE: Speculative Sampling Requires Rethinking Feature Uncertainty", "authors": [ "Yuhui Li", "Fangyun Wei", "Chao Zhang", "Hongyang Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2401.15077", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "second-to-top-layer feature extrapolation" ], "cost_reduction_reported": "2.7x-3.5x latency speedup and doubled throughput on LLaMA2-Chat 70B while preserving the output distribution", "benchmarks": [], "models_studied": [ "Vicuna", "LLaMA2-Chat", "Mixtral 8x7B Instruct" ], "production_evidence": false, "summary": "Drafts at the feature level (second-to-top layer) using a lightweight head that also sees the next-step token, reducing uncertainty and improving draft acceptance.", "key_idea": "Feature-level drafting for speculative sampling.", "relevance_score": 2, "relevance_rationale": "Leading speculative-decoding method used in serving engines.", "found_via": "task seed", "cites_or_cited_by": [ "cai-2024-medusa", "leviathan-2022-speculative-decoding", "li-2024-eagle-2", "li-2025-eagle-3", "wang-2026-agentspec", "xia-2024-speculative-decoding-survey" ] }, { "id": "li-2024-eagle-2", "title": "EAGLE-2: Faster Inference of Language Models with Dynamic Draft Trees", "authors": [ "Yuhui Li", "Fangyun Wei", "Chao Zhang", "Hongyang Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.16858", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "draft-model confidence" ], "cost_reduction_reported": "speedup ratios 3.05x-4.26x, 20%-40% faster than EAGLE-1, lossless", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Uses the calibrated confidence of the EAGLE draft model to build context-dependent dynamic draft trees.", "key_idea": "Dynamic draft trees from draft confidence.", "relevance_score": 2, "relevance_rationale": "Speculative decoding refinement.", "found_via": "task seed", "cites_or_cited_by": [ "li-2024-eagle", "li-2025-eagle-3" ] }, { "id": "li-2024-more-agents-is-all-you-need", "title": "More Agents Is All You Need", "authors": [ "Junyou Li", "Qin Zhang", "Yangbin Yu", "Qiang Fu", "Deheng Ye" ], "year": 2024, "venue_or_source": "TMLR", "source_type": "paper", "url": "https://arxiv.org/abs/2402.05120", "code_url": "https://github.com/MoreAgentsIsAllYouNeed/AgentForest", "topic_clusters": [ "C9", "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "majority vote" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows that simply sampling many agent outputs and taking a majority vote (Agent Forest) scales LLM performance with the number of agents, and that the gain grows with task difficulty. Serves as a strong, cheap-to-build baseline against which elaborate multi-agent designs should be compared.", "key_idea": "Performance scales with the number of sampled agents under plain voting.", "relevance_score": 2, "relevance_rationale": "Baseline for when multi-agent structure is worth its cost.", "found_via": "web search: More agents is all you need", "cites_or_cited_by": [ "kapoor-2024-ai-agents-that-matter" ] }, { "id": "li-2024-prompt-compression-survey", "title": "Prompt Compression for Large Language Models: A Survey", "authors": [ "Zongqian Li", "Yinhong Liu", "Yixuan Su", "Nigel Collier" ], "year": 2024, "venue_or_source": "NAACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2410.12388", "code_url": "https://github.com/ZongqianLi/Prompt-Compression-Survey", "topic_clusters": [ "C10", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of hard-prompt (filtering, distillation-enhanced, RL-enhanced) and soft-prompt compression methods, interpreting them through attention optimization, PEFT and modality-integration lenses, and listing open problems.", "key_idea": "Taxonomy of prompt compression methods.", "relevance_score": 2, "relevance_rationale": "Orientation survey for the compression sub-area.", "found_via": "prompt compression survey LLM 2024 2025", "cites_or_cited_by": [ "jiang-2023-llmlingua", "li-2023-selective-context", "pan-2024-llmlingua-2" ] }, { "id": "li-2024-serving-survey", "title": "LLM Inference Serving: Survey of Recent Advances and Opportunities", "authors": [ "Baolin Li", "Yankai Jiang", "Vijay Gadepally", "Devesh Tiwari" ], "year": 2024, "venue_or_source": "IEEE HPEC 2024 (per Semantic Scholar)", "source_type": "paper", "url": "https://arxiv.org/abs/2407.12391", "code_url": null, "topic_clusters": [ "C11", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of LLM serving system advances since 2023 that improve efficiency without changing decoding.", "key_idea": "Serving systems survey.", "relevance_score": 2, "relevance_rationale": "Background for C11.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "li-2024-snapkv", "title": "SnapKV: LLM Knows What You are Looking for Before Generation", "authors": [ "Yuhong Li", "Yingbing Huang", "Bowen Yang", "Bharat Venkitesh", "Acyr Locatelli", "Hanchen Ye", "Tianle Cai", "Patrick Lewis", "Deming Chen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.14469", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "observation-window attention" ], "cost_reduction_reported": null, "benchmarks": [ "16 long-sequence datasets", "Needle-in-a-Haystack" ], "models_studied": [], "production_evidence": false, "summary": "Selects important KV positions per attention head using attention from the end of the prompt, compressing long-prompt KV caches with comparable accuracy and processing up to 380K tokens on one A100-80GB.", "key_idea": "Compress prompt KV using attention from an observation window.", "relevance_score": 2, "relevance_rationale": "KV compression background.", "found_via": "KV cache compression survey LLM inference efficiency", "cites_or_cited_by": [ "zhang-2023-h2o" ] }, { "id": "lin-2024-hammer", "title": "Hammer: Robust Function-Calling for On-Device Language Models via Function Masking", "authors": [ "Qiqiang Lin", "Muning Wen", "Qiuying Peng", "Guanyu Nie", "Junwei Liao", "Jun Wang", "Xiaoyun Mo", "Jiamu Zhou", "Cheng Cheng", "Yin Zhao", "Jun Wang", "Weinan Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.04587", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "function masking" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "On-device function-calling models trained with function masking and irrelevance data to reduce reliance on names.", "key_idea": "Robust small function-calling models.", "relevance_score": 2, "relevance_rationale": "Specialization.", "found_via": "keyword: function calling small language model", "cites_or_cited_by": [] }, { "id": "lin-2024-scaffolded-lm-survey", "title": "Scaffolded Language Models with Language Supervision for Mixed-Autonomy: A Survey", "authors": [ "Matthieu Lin", "Jenny Sheng", "Andrew Zhao", "Shenzhi Wang", "Yang Yue", "Victor Shea Jay Huang", "Huan Liu", "Jun Liu", "Gao Huang", "Yong-Jin Liu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.16392", "code_url": null, "topic_clusters": [ "C4", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey (first released as 'LLM-based Optimization of Compound AI Systems: A Survey') of training non-parametric variables of scaffolded LMs, such as prompts, tools and scaffold code, using an LM as optimizer with language supervision.", "key_idea": "LLM-as-optimizer over prompts/tools/scaffold code, surveyed.", "relevance_score": 2, "relevance_rationale": "Background survey for C4.", "found_via": "search: LLM-based Optimization of Compound AI Systems survey arXiv", "cites_or_cited_by": [] }, { "id": "liu-2024-grab-llm-data-classification", "title": "LLM-powered data classification for data entities at scale", "authors": [ "Hualin Liu", "Stefan Jaro", "Harvey Li", "Jerome Tong", "Andrew Lam", "Chamal Sapumohotti", "Feng Cheng", "Aaqib Kufran" ], "year": 2024, "venue_or_source": "Grab engineering blog", "source_type": "blog", "url": "https://engineering.grab.com/llm-powered-data-classification", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "Company-reported: ~360 man-days per year saved; cost described as extremely affordable", "benchmarks": [], "models_studied": [ "GPT-3.5 (Azure OpenAI)" ], "production_evidence": true, "summary": "Grab's production pipeline tagging sensitive data columns with GPT-3.5, scanning 20K+ entities in its first month, with prompt techniques for schema enforcement.", "key_idea": "Cheap model sufficient for high-volume classification.", "relevance_score": 2, "relevance_rationale": "Production example of choosing a small model.", "found_via": "seed list: Grab case study", "cites_or_cited_by": [] }, { "id": "lu-2024-merge-ensemble-cooperate", "title": "Merge, Ensemble, and Cooperate! A Survey on Collaborative Strategies in the Era of Large Language Models", "authors": [ "Jinliang Lu", "Ziliang Pang", "Min Xiao", "Yaochen Zhu", "Rui Xia", "Jiajun Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.06089", "code_url": null, "topic_clusters": [ "C1", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of collaborative strategies among LLMs: merging in parameter space, ensembling outputs, and cooperation across models for specific tasks.", "key_idea": "Merge/ensemble/cooperate taxonomy.", "relevance_score": 2, "relevance_rationale": "Background survey.", "found_via": "snowball: cited-by ding-2024-hybrid-llm", "cites_or_cited_by": [ "ding-2024-hybrid-llm" ] }, { "id": "luo-2024-omegaprm", "title": "Improve Mathematical Reasoning in Language Models by Automated Process Supervision", "authors": [ "Liangchen Luo", "Yinxiao Liu", "Rosanne Liu", "Samrat Phatale", "Harsh Lara", "Yunxuan Li", "Lei Shu", "Yun Zhu", "Lei Meng", "Jiao Sun", "Abhinav Rastogi" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.06592", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "step", "learning_regime": "supervised-offline", "signals_used": [ "process reward", "MCTS labels" ], "cost_reduction_reported": null, "benchmarks": [ "MATH", "GSM8K" ], "models_studied": [ "Gemini Pro", "Gemma2 27B" ], "production_evidence": false, "summary": "Proposes OmegaPRM, a divide-and-conquer MCTS that efficiently finds the first error step to collect process supervision labels at scale, then trains PRMs that improve math reasoning.", "key_idea": "Binary-search MCTS for cheap process labels.", "relevance_score": 2, "relevance_rationale": "PRM data collection technique.", "found_via": "keyword: process reward model", "cites_or_cited_by": [] }, { "id": "mitra-2024-agentinstruct-orca3", "title": "AgentInstruct: Toward Generative Teaching with Agentic Flows", "authors": [ "Arindam Mitra", "Luciano Del Corro", "Guoqing Zheng", "Shweti Mahajan", "Dany Rouhana", "Andres Codas", "Yadong Lu", "Wei-ge Chen", "Olga Vrousgos", "Corby Rosset", "Fillipe Silva", "Hamed Khanpour", "Yash Lara", "Ahmed Awadallah" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.03502", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "agentic synthetic data flows" ], "cost_reduction_reported": null, "benchmarks": [ "AGIEval", "MMLU", "GSM8K", "BBH", "AlpacaEval" ], "models_studied": [ "Mistral-7B" ], "production_evidence": false, "summary": "Uses multi-agent flows to generate 25M synthetic post-training pairs teaching skills incl. tool use; Orca-3 improves substantially over Mistral-7B-Instruct.", "key_idea": "Agentic flows generate synthetic teaching data.", "relevance_score": 2, "relevance_rationale": "Synthetic data generation.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "murty-2024-nnetnav", "title": "NNetNav: Unsupervised Learning of Browser Agents Through Environment Interaction in the Wild", "authors": [ "Shikhar Murty", "Dzmitry Bahdanau", "Christopher D. Manning" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.02907", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "retroactive instruction labeling" ], "cost_reduction_reported": "SFT of smaller policy improves WebArena by 6 points and MiniWoB++ by over 20", "benchmarks": [ "WebArena", "MiniWoB++" ], "models_studied": [], "production_evidence": false, "summary": "Collects web-agent demonstrations by exploration and retroactively labels them with instructions, then fine-tunes a smaller policy.", "key_idea": "Unsupervised demonstration synthesis for web agents.", "relevance_score": 2, "relevance_rationale": "Synthetic trajectories without human demos.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "nguyen-2024-laser", "title": "LASeR: Learning to Adaptively Select Reward Models with Multi-Armed Bandits", "authors": [ "Duy Nguyen", "Archiki Prasad", "Elias Stengel-Eskin", "Mohit Bansal" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.01735", "code_url": null, "topic_clusters": [ "C5", "C7" ], "routing_granularity": "none", "learning_regime": "bandit-online", "signals_used": [ "reward model scores" ], "cost_reduction_reported": "about 2x speedup vs RM ensemble", "benchmarks": [ "commonsense and math reasoning", "WildChat", "long-context QA" ], "models_studied": [ "Llama-3-8B" ], "production_evidence": false, "summary": "LASeR picks which reward model to use per instance with a multi-armed bandit during iterative LLM training, beating RM ensembles in accuracy and efficiency.", "key_idea": "Bandit selection among reward models.", "relevance_score": 2, "relevance_rationale": "Bandit selection of evaluators; background.", "found_via": "snowball: cited-by dai-2024-c2mab-v", "cites_or_cited_by": [] }, { "id": "nikitin-2024-kernel-language-entropy", "title": "Kernel Language Entropy: Fine-grained Uncertainty Quantification for LLMs from Semantic Similarities", "authors": [ "Alexander V. Nikitin", "Jannik Kossen", "Yarin Gal", "Pekka Marttinen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.20003", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "semantic similarity kernel", "von Neumann entropy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Generalizes semantic entropy by encoding semantic similarities among sampled outputs in a kernel and computing von Neumann entropy for finer-grained uncertainty.", "key_idea": "Soft semantic clustering for UQ.", "relevance_score": 2, "relevance_rationale": "Improved semantic UQ.", "found_via": "keyword: semantic entropy", "cites_or_cited_by": [ "kuhn-2023-semantic-uncertainty" ] }, { "id": "orgad-2024-llms-know-more-than-show", "title": "LLMs Know More Than They Show: On the Intrinsic Representation of LLM Hallucinations", "authors": [ "Hadas Orgad", "Michael Toker", "Zorik Gekhman", "Roi Reichart", "Idan Szpektor", "Hadas Kotek", "Yonatan Belinkov" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.02707", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden-state probe", "exact answer tokens" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Finds truthfulness signals concentrate in specific answer tokens, improving probe-based error detection, but such detectors generalize poorly across datasets.", "key_idea": "Probe the exact-answer tokens; beware poor cross-task generalization.", "relevance_score": 2, "relevance_rationale": "Caveat for deploying probes across heterogeneous traffic.", "found_via": "keyword: probes hidden states", "cites_or_cited_by": [] }, { "id": "ou-2024-synatra", "title": "Synatra: Turning Indirect Knowledge into Direct Demonstrations for Digital Agents at Scale", "authors": [ "Tianyue Ou", "Frank F. Xu", "Aman Madaan", "Jiarui Liu", "Robert Lo", "Abishek Sridhar", "Sudipta Sengupta", "Dan Roth", "Graham Neubig", "Shuyan Zhou" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2409.15637", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "indirect-knowledge-to-demonstration synthesis" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "CodeLlama-7B" ], "production_evidence": false, "summary": "Converts indirect knowledge (tutorials) into 100k synthetic demonstrations to fine-tune a 7B web agent.", "key_idea": "Synthetic demos from indirect knowledge.", "relevance_score": 2, "relevance_rationale": "Synthetic agent data.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "patel-2024-self-improve-web-agents", "title": "Large Language Models Can Self-Improve At Web Agent Tasks", "authors": [ "Ajay Patel", "Markus Hofmarcher", "Claudiu Leoveanu-Condrei", "Marius-Constantin Dinu", "Chris Callison-Burch", "Sepp Hochreiter" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.20309", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "self-generated synthetic data" ], "cost_reduction_reported": "31% improvement in task completion over base model on WebArena", "benchmarks": [ "WebArena" ], "models_studied": [ "Qwen-1.5-72B" ], "production_evidence": false, "summary": "Fine-tunes an LLM on its own filtered synthetic trajectories to self-improve at web tasks.", "key_idea": "Self-improvement from own trajectories.", "relevance_score": 2, "relevance_rationale": "Self-training background.", "found_via": "hf-papers search: self-improving agents fine-tuning", "cites_or_cited_by": [] }, { "id": "phu-2024-discord-genai", "title": "Developing Rapidly with Generative AI", "authors": [ "Shannon Phu" ], "year": 2024, "venue_or_source": "Discord blog", "source_type": "blog", "url": "https://discord.com/blog/developing-rapidly-with-generative-ai", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "GPT-4 critic evaluation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-4", "self-hosted smaller models" ], "production_evidence": false, "summary": "Discord's process: prototype with frontier APIs, use GPT-4 as an automated critic, then consider smaller self-hosted models for cost once validated.", "key_idea": "Prototype big, then move to smaller self-hosted models.", "relevance_score": 2, "relevance_rationale": "Practitioner cost workflow.", "found_via": "seed list: engineering case studies", "cites_or_cited_by": [] }, { "id": "plaut-2024-msp-predicts-correctness", "title": "Probabilities of Chat LLMs Are Miscalibrated but Still Predict Correctness on Multiple-Choice Q&A", "authors": [ "Benjamin Plaut", "Nguyen X. Khanh", "Tu Trinh" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.13213", "code_url": null, "topic_clusters": [ "C6", "C7" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "max softmax probability" ], "cost_reduction_reported": null, "benchmarks": [ "multiple-choice QA" ], "models_studied": [ "15 chat LLMs" ], "production_evidence": false, "summary": "Finds chat LLM maximum softmax probabilities are miscalibrated but still predict correctness, and uses them for selective abstention with little labeled data.", "key_idea": "Softmax confidence predicts correctness despite miscalibration.", "relevance_score": 2, "relevance_rationale": "Cheap correctness signal.", "found_via": "OpenAlex title: predict LLM correctness", "cites_or_cited_by": [] }, { "id": "qian-2024-macnet-scaling-multi-agent", "title": "Scaling Large Language Model-based Multi-Agent Collaboration", "authors": [ "Chen Qian", "Zihao Xie", "YiFei Wang", "Wei Liu", "Kunlun Zhu", "Hanchen Xia", "Yufan Dang", "Zhuoyun Du", "Weize Chen", "Cheng Yang", "Zhiyuan Liu", "Maosong Sun" ], "year": 2024, "venue_or_source": "ICLR 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2406.07155", "code_url": "https://github.com/OpenBMB/ChatDev/tree/macnet", "topic_clusters": [ "C12" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "MacNet organizes up to a thousand agents on directed acyclic graphs and studies how performance changes as agent count grows, reporting a logistic collaborative scaling law and that irregular topologies beat regular ones. Provides evidence on the returns (and cost) of adding more agents.", "key_idea": "Collaborative scaling law: performance grows logistically with agent count.", "relevance_score": 2, "relevance_rationale": "Background on scaling agent count vs cost.", "found_via": "arXiv title search", "cites_or_cited_by": [] }, { "id": "qiu-2024-semantic-density", "title": "Semantic Density: Uncertainty Quantification for Large Language Models through Confidence Measurement in Semantic Space", "authors": [ "Xin Qiu", "Risto Miikkulainen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.13845", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "semantic density" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Llama 3", "Mixtral-8x22B" ], "production_evidence": false, "summary": "Response-wise confidence from semantic-space density of sampled outputs, beating prior UQ methods on four QA benchmarks.", "key_idea": "Semantic density UQ.", "relevance_score": 2, "relevance_rationale": "UQ variant.", "found_via": "hf-papers search: uncertainty quantification LLM agents", "cites_or_cited_by": [] }, { "id": "ravi-2024-lynx", "title": "Lynx: An Open Source Hallucination Evaluation Model", "authors": [ "Selvan Sunitha Ravi", "Bartosz Mielczarek", "Anand Kannappan", "Douwe Kiela", "Rebecca Qian" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2407.08488", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "hallucination judge" ], "cost_reduction_reported": null, "benchmarks": [ "HaluBench" ], "models_studied": [ "GPT-4o", "Claude-3-Sonnet" ], "production_evidence": false, "summary": "Open hallucination-detection judge for RAG that outperforms GPT-4o and Claude-3-Sonnet on the 15k-sample HaluBench.", "key_idea": "Specialized RAG hallucination judge.", "relevance_score": 2, "relevance_rationale": "Open alternative to frontier judges.", "found_via": "keyword: hallucination detection", "cites_or_cited_by": [] }, { "id": "regmi-2024-gpt-semantic-cache", "title": "GPT Semantic Cache: Reducing LLM Costs and Latency via Semantic Embedding Caching", "authors": [ "Sajal Regmi", "Chetan Phakami Pun" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.05276", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "query embedding similarity" ], "cost_reduction_reported": "API calls reduced by up to 68.8% with cache hit rates of 61.6%-68.8% and positive-hit accuracy above 97%", "benchmarks": [ "customer-service style query categories" ], "models_studied": [], "production_evidence": false, "summary": "Stores query embeddings in an in-memory store (Redis) and reuses responses for semantically similar queries. Reports hit rates and accuracy per query category.", "key_idea": "In-memory embedding cache for LLM responses.", "relevance_score": 2, "relevance_rationale": "Simple semantic cache with reported savings; weaker evaluation.", "found_via": "semantic cache LLM agents tool calls caching arXiv", "cites_or_cited_by": [ "bang-2023-gptcache" ] }, { "id": "renze-2024-concise-cot", "title": "The Benefits of a Concise Chain of Thought on Problem-Solving in Large Language Models", "authors": [ "Matthew Renze", "Erhan Guven" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2401.05618", "code_url": "https://github.com/matthewrenze/jhu-concise-cot", "topic_clusters": [ "C9", "C10" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "concise prompting" ], "cost_reduction_reported": "CCoT reduced average response length by 48.70% for GPT-3.5 and GPT-4 with negligible impact; average per-token cost reduction 22.67%; GPT-3.5 math penalty 27.69%", "benchmarks": [ "MCQA benchmark" ], "models_studied": [ "GPT-3.5", "GPT-4" ], "production_evidence": false, "summary": "Compares standard and concise CoT prompts and finds large length savings with small accuracy impact, except a notable penalty on math for GPT-3.5.", "key_idea": "Concise CoT prompting is nearly free except on hard math.", "relevance_score": 2, "relevance_rationale": "Background evidence on prompt-level savings.", "found_via": "GitHub list: Awesome-Efficient-Reasoning-LLMs (snowball: related work of sui-2025-stop-overthinking)", "cites_or_cited_by": [ "sui-2025-stop-overthinking" ] }, { "id": "richemond-2024-dro", "title": "Offline Regularised Reinforcement Learning for Large Language Models Alignment", "authors": [ "Pierre Harvey Richemond", "Yunhao Tang", "Daniel Guo", "Daniele Calandriello", "Mohammad Gheshlaghi Azar", "Rafael Rafailov", "Bernardo Ávila Pires", "Eugene Tarassov", "Lucas Spangher", "Will Ellsworth", "Aliaksei Severyn", "Jonathan Mallinson", "Lior Shani", "Gil I. Shamir", "Rishabh Joshi", "Tianqi Liu", "Rémi Munos", "Bilal Piot" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2405.19107", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "single-trajectory thumbs-up/down feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "T5" ], "production_evidence": false, "summary": "DRO optimizes an LLM from single-trajectory data (prompt, response, user thumbs up/down) with a mean-squared objective, avoiding the need for pairwise preferences; outperforms KTO.", "key_idea": "Learn from abundant single-response user feedback logs.", "relevance_score": 2, "relevance_rationale": "Relevant to using production feedback, though for alignment not routing.", "found_via": "OpenAlex title search", "cites_or_cited_by": [] }, { "id": "snell-2024-predicting-emergent", "title": "Predicting Emergent Capabilities by Finetuning", "authors": [ "Charlie Snell", "Eric Wallace", "Dan Klein", "Sergey Levine" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2411.16035", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "finetuning scaling curves" ], "cost_reduction_reported": null, "benchmarks": [ "MMLU", "GSM8K", "CommonsenseQA", "CoLA" ], "models_studied": [], "production_evidence": false, "summary": "Predicts whether future larger models will show emergent capability on a task by fine-tuning smaller models with varying data and fitting emergence laws.", "key_idea": "Emergence prediction via fine-tuning.", "relevance_score": 2, "relevance_rationale": "Background on capability prediction.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "ruan-2024-observational-scaling" ] }, { "id": "sun-2024-llumnix", "title": "Llumnix: Dynamic Scheduling for Large Language Model Serving", "authors": [ "Biao Sun", "Ziming Huang", "Hanyu Zhao", "Wencong Xiao", "Xinyi Zhang", "Yong Li", "Wei Lin" ], "year": 2024, "venue_or_source": "OSDI 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2406.03243", "code_url": "https://github.com/AlibabaPAI/llumnix", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "live request migration" ], "cost_reduction_reported": "tail latencies improved by an order of magnitude, high-priority requests up to 1.5x faster, and up to 36% cost savings at similar tail latency", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Reschedules requests across model instances at runtime via live KV migration to balance load, reduce fragmentation and honour priorities.", "key_idea": "Live migration of requests across instances.", "relevance_score": 2, "relevance_rationale": "Serving background with cost savings.", "found_via": "task seed", "cites_or_cited_by": [] }, { "id": "sun-2024-os-genesis", "title": "OS-Genesis: Automating GUI Agent Trajectory Construction via Reverse Task Synthesis", "authors": [ "Qiushi Sun", "Kanzhi Cheng", "Zichen Ding", "Chuanyang Jin", "Yian Wang", "Fangzhi Xu", "Zhenyu Wu", "Chengyou Jia", "Liheng Chen", "Zhoumianze Liu", "Ben Kao", "Guohao Li", "Junxian He", "Yu Qiao", "Zhiyong Wu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.19723", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "reverse task synthesis", "trajectory reward model" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Synthesizes GUI trajectories by exploring interfaces first and deriving tasks retrospectively, filtered by a reward model.", "key_idea": "Reverse task synthesis for GUI data.", "relevance_score": 2, "relevance_rationale": "Synthetic agent data.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "thakur-2024-judging-the-judges", "title": "Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges", "authors": [ "Aman Singh Thakur", "Kartik Choudhary", "Venkat Srinik Ramayapally", "Sankaran Vaidyanathan", "Dieuwke Hupkes" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.12624", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judge", "lexical judge" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Evaluates 9 judge models against humans; strong judges align well, but smaller judges or lexical matches can rank models comparably.", "key_idea": "Meta-evaluation of judge models.", "relevance_score": 2, "relevance_rationale": "Background on judge reliability and small-judge viability.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "tomani-2024-uncertainty-abstention", "title": "Uncertainty-Based Abstention in LLMs Improves Safety and Reduces Hallucinations", "authors": [ "Christian Tomani", "Kamalika Chaudhuri", "Ivan Evtimov", "Daniel Cremers", "Mark Ibrahim" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.10960", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "statistical uncertainty", "In-Dialogue Uncertainty" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies abstaining when uncertain for correctness, unanswerable questions and safety; the right uncertainty measure improves correctness by 2-8%, avoids 50% of hallucinations and raises safety by 70-99%.", "key_idea": "Uncertainty-based abstention for reliability.", "relevance_score": 2, "relevance_rationale": "Background on abstention signals.", "found_via": "arXiv title search: abstention large language models", "cites_or_cited_by": [] }, { "id": "wang-2024-airbnb-automation-platform-v2", "title": "Automation Platform v2: Improving conversational AI at Airbnb", "authors": [ "Chutian Wang", "Zhiheng Xu", "Paul Lou", "Ziyi Wang", "Jiayu Lou", "Liuming Zhang", "Jingwen Qiang", "Clint Kelly", "Lei Shi", "Dan Zhao", "Xu Hu", "Jianqi Liao", "Zecheng Xu", "Tong Chen" ], "year": 2024, "venue_or_source": "Airbnb Tech Blog (Medium)", "source_type": "blog", "url": "https://medium.com/airbnb-engineering/automation-platform-v2-improving-conversational-ai-at-airbnb-d86c9386e0cb", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Airbnb's platform for LLM-driven support workflows with chain-of-thought loops, context retrievers, tool execution and parallel guardrails; qualitative, notes latency and hallucination concerns.", "key_idea": "Production conversational-AI platform design.", "relevance_score": 2, "relevance_rationale": "Production context; no cost data.", "found_via": "seed list: Airbnb case study", "cites_or_cited_by": [] }, { "id": "wang-2024-codeact", "title": "Executable Code Actions Elicit Better LLM Agents", "authors": [ "Xingyao Wang", "Yangyi Chen", "Lifan Yuan", "Yizhe Zhang", "Yunzhu Li", "Hao Peng", "Heng Ji" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.01030", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "CodeActInstruct trajectories" ], "cost_reduction_reported": "Code actions up to 20% higher success rate than JSON/text", "benchmarks": [ "API-Bank" ], "models_studied": [ "Llama2", "Mistral" ], "production_evidence": false, "summary": "Uses executable Python code as the agent action space and releases CodeActInstruct to fine-tune CodeActAgent.", "key_idea": "Code as unified action space.", "relevance_score": 2, "relevance_rationale": "Architecture plus fine-tuning.", "found_via": "keyword: fine-tuning small agents", "cites_or_cited_by": [] }, { "id": "wen-2024-abstention-survey", "title": "Know Your Limits: A Survey of Abstention in Large Language Models", "authors": [ "Bingbing Wen", "Jihan Yao", "Shangbin Feng", "Chenjun Xu", "Yulia Tsvetkov", "Bill Howe", "Lucy Lu Wang" ], "year": 2024, "venue_or_source": "TACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2407.18418", "code_url": null, "topic_clusters": [ "C2", "C7", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Surveys abstention in LLMs from the perspectives of the query, the model and human values, organising methods, benchmarks and metrics and listing open problems.", "key_idea": "Framework and survey of LLM abstention.", "relevance_score": 2, "relevance_rationale": "Background on abstention as the terminal action of cascades.", "found_via": "arXiv title search: abstention large language models", "cites_or_cited_by": [] }, { "id": "xia-2024-speculative-decoding-survey", "title": "Unlocking Efficiency in Large Language Model Inference: A Comprehensive Survey of Speculative Decoding", "authors": [ "Heming Xia", "Zhe Yang", "Qingxiu Dong", "Peiyi Wang", "Yongqi Li", "Tao Ge", "Tianyu Liu", "Wenjie Li", "Zhifang Sui" ], "year": 2024, "venue_or_source": "Findings of ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2401.07851", "code_url": null, "topic_clusters": [ "C11", "C15" ], "routing_granularity": "token", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey formalizing speculative decoding, organizing drafter selection and verification strategies, and comparing leading methods under a common testbed.", "key_idea": "Taxonomy of speculative decoding.", "relevance_score": 2, "relevance_rationale": "Orientation survey.", "found_via": "speculative decoding survey Xia unlocking efficiency", "cites_or_cited_by": [ "cai-2024-medusa", "leviathan-2022-speculative-decoding", "li-2024-eagle" ] }, { "id": "xie-2024-osworld", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.07972", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "execution-based evaluation scripts" ], "cost_reduction_reported": null, "benchmarks": [ "OSWorld (369 tasks)" ], "models_studied": [ "LLM and VLM agents" ], "production_evidence": false, "summary": "Scalable real-computer environment across Ubuntu, Windows and macOS with 369 open-ended tasks and execution-based evaluation. Humans complete over 72% of tasks versus about 12% for the best model at release.", "key_idea": "Real OS environment for computer-use agents.", "relevance_score": 2, "relevance_rationale": "Background computer-use benchmark; used in reliability studies.", "found_via": "web search: OSWorld", "cites_or_cited_by": [] }, { "id": "xu-2024-agent-services-survey", "title": "Deploying Foundation Model Powered Agent Services: A Survey", "authors": [ "Wenchao Xu", "Jinyu Chen", "Peirong Zheng", "Xiaoquan Yi", "Tianyi Tian", "Wenhui Zhu", "Quan Wan", "Haozhao Wang", "Yunfeng Fan", "Qinliang Su", "Xuemin Shen" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.13437", "code_url": null, "topic_clusters": [ "C11", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey on deploying foundation-model-powered agent services, covering resource scaling, inference acceleration (compression, token reduction) and components for agent services with QoS.", "key_idea": "Survey of agent service deployment.", "relevance_score": 2, "relevance_rationale": "Orientation survey for agent serving.", "found_via": "InferCept intercept augmented LLM inference tool calls", "cites_or_cited_by": [] }, { "id": "xu-2024-agenttrek", "title": "AgentTrek: Agent Trajectory Synthesis via Guiding Replay with Web Tutorials", "authors": [ "Yiheng Xu", "Dunjie Lu", "Zhennan Shen", "Junli Wang", "Zekun Wang", "Yuchen Mao", "Caiming Xiong", "Tao Yu" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.09605", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "tutorial-guided replay" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Synthesizes GUI trajectories by replaying web tutorials with a VLM agent, cheaper than human annotation.", "key_idea": "Tutorial-guided trajectory synthesis.", "relevance_score": 2, "relevance_rationale": "Synthetic agent data.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "xu-2024-survey-kd-llms", "title": "A Survey on Knowledge Distillation of Large Language Models", "authors": [ "Xiaohan Xu", "Ming Li", "Chongyang Tao", "Tao Shen", "Reynold Cheng", "Jinyang Li", "Can Xu", "Dacheng Tao", "Tianyi Zhou" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.13116", "code_url": null, "topic_clusters": [ "C8", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of knowledge distillation for LLMs across algorithms, skills and verticals.", "key_idea": "KD survey.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "keyword: knowledge distillation LLM survey", "cites_or_cited_by": [] }, { "id": "yadav-2024-moerging-survey", "title": "A Survey on Model MoErging: Recycling and Routing Among Specialized Experts for Collaborative Learning", "authors": [ "Prateek Yadav", "Colin Raffel", "Mohammed Muqeeth", "Lucas Caccia", "Haokun Liu", "Tianlong Chen", "Mohit Bansal", "Leshem Choshen", "Alessandro Sordoni" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2408.07057", "code_url": null, "topic_clusters": [ "C1", "C15" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of MoErging methods that recycle specialized expert models and route inputs among them, with a taxonomy of design choices.", "key_idea": "Routing among specialized experts.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "snowball: cited-by ong-2024-routellm", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "ye-2024-calm-judge-biases", "title": "Justice or Prejudice? Quantifying Biases in LLM-as-a-Judge", "authors": [ "Jiayi Ye", "Yanbo Wang", "Yue Huang", "Dongping Chen", "Qihui Zhang", "Nuno Moniz", "Tian Gao", "Werner Geyer", "Chao Huang", "Pin-Yu Chen", "Nitesh V Chawla", "Xiangliang Zhang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.02736", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "LLM judge" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Identifies 12 biases in LLM-as-a-Judge and an automated framework (CALM) to quantify them.", "key_idea": "Catalog of judge biases.", "relevance_score": 2, "relevance_rationale": "Caveat for judge-based routing.", "found_via": "keyword: LLM-as-a-judge bias", "cites_or_cited_by": [] }, { "id": "ye-2024-chunkattention", "title": "ChunkAttention: Efficient Self-Attention with Prefix-Aware KV Cache and Two-Phase Partition", "authors": [ "Lu Ye", "Ze Tao", "Yong Huang", "Yang Li" ], "year": 2024, "venue_or_source": "ACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2402.15220", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "prefix-tree KV chunks" ], "cost_reduction_reported": "self-attention kernel 3.2-4.8x faster than state of the art for system prompts of 1024-4096 tokens", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Prefix-aware KV cache organized as a prefix tree of chunks with a two-phase attention kernel exploiting shared system prompts.", "key_idea": "Prefix-tree KV with shared-prompt-aware kernel.", "relevance_score": 2, "relevance_rationale": "Shared system-prompt efficiency.", "found_via": "snowball: related work of zheng-2023-sglang", "cites_or_cited_by": [ "zheng-2023-sglang" ] }, { "id": "yin-2024-godel-agent", "title": "Gödel Agent: A Self-Referential Agent Framework for Recursive Self-Improvement", "authors": [ "Xunjian Yin", "Xinyi Wang", "Liangming Pan", "Li Lin", "Xiaojun Wan", "William Yang Wang" ], "year": 2024, "venue_or_source": "ACL 2025", "source_type": "paper", "url": "https://arxiv.org/abs/2410.04444", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "task feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Gödel Agent is a self-referential agent that inspects and modifies its own logic (via monkey patching) to recursively self-improve without predefined optimization routines.", "key_idea": "Self-referential recursive self-improvement of agent code.", "relevance_score": 2, "relevance_rationale": "Background on self-optimizing agents.", "found_via": "search: Darwin Gödel Machine self-improving agent design arXiv", "cites_or_cited_by": [] }, { "id": "zhang-2024-agentohana", "title": "AgentOhana: Design Unified Data and Training Pipeline for Effective Agent Learning", "authors": [ "Jianguo Zhang", "Tian Lan", "Rithesh Murthy", "Zhiwei Liu", "Weiran Yao", "Juntao Tan", "Thai Hoang", "Liangwei Yang", "Yihao Feng", "Zuxin Liu", "Tulika Awalgaonkar", "Juan Carlos Niebles", "Silvio Savarese", "Shelby Heinecke", "Huan Wang", "Caiming Xiong" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.15506", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "unified multi-source trajectories" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Unifies heterogeneous agent trajectories into a consistent format with a balanced data pipeline and trains xLAM-v0.1.", "key_idea": "Data unification for agent training.", "relevance_score": 2, "relevance_rationale": "Pipeline background.", "found_via": "keyword: fine-tuning small agents", "cites_or_cited_by": [] }, { "id": "zhao-2024-predict-lvlm", "title": "Can We Predict Performance of Large Models across Vision-Language Tasks?", "authors": [ "Qinyu Zhao", "Ming Xu", "Kartik Gupta", "Akshay Asthana", "Liang Zheng", "Stephen Jay Gould" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2410.10112", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "sparse performance matrix" ], "cost_reduction_reported": null, "benchmarks": [ "LVLM benchmarks" ], "models_studied": [ "large vision-language models" ], "production_evidence": false, "summary": "Predicts unknown model-by-dataset scores for vision-language models via probabilistic matrix factorization with MCMC uncertainty, using uncertainty to prioritize which evaluations to run.", "key_idea": "Matrix completion with uncertainty for performance prediction.", "relevance_score": 2, "relevance_rationale": "Background; vision-language focus.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "zhang-2024-collaborative-performance-prediction" ] }, { "id": "zhao-2024-steve-2-distill", "title": "Do We Really Need a Complex Agent System? Distill Embodied Agent into a Single Model", "authors": [ "Zhonghan Zhao", "Ke Ma", "Wenhao Chai", "Xuan Wang", "Kewei Chen", "Dongxu Guo", "Yanting Zhang", "Hongwei Wang", "Gaoang Wang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.04619", "code_url": null, "topic_clusters": [ "C8", "C12" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "hierarchical distillation" ], "cost_reduction_reported": null, "benchmarks": [ "Minecraft tasks" ], "models_studied": [], "production_evidence": false, "summary": "Distills a complex multi-LLM embodied agent system into a single model through hierarchical knowledge distillation.", "key_idea": "Distill an agent system into one model.", "relevance_score": 2, "relevance_rationale": "Architecture simplification via distillation.", "found_via": "hf-papers search: distill multi-agent system into single model", "cites_or_cited_by": [] }, { "id": "zheng-2024-processbench", "title": "ProcessBench: Identifying Process Errors in Mathematical Reasoning", "authors": [ "Chujie Zheng", "Zhenru Zhang", "Beichen Zhang", "Runji Lin", "Keming Lu", "Bowen Yu", "Dayiheng Liu", "Jingren Zhou", "Junyang Lin" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2412.06559", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "step", "learning_regime": "n/a", "signals_used": [ "step error identification" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "3,400 competition-level math solutions with expert-annotated first-error steps; existing PRMs generalize poorly and underperform prompted critic models.", "key_idea": "Benchmark for step-error detection.", "relevance_score": 2, "relevance_rationale": "Evaluates PRM reliability.", "found_via": "keyword: process reward model benchmark", "cites_or_cited_by": [] }, { "id": "zhong-2024-distserve", "title": "DistServe: Disaggregating Prefill and Decoding for Goodput-optimized Large Language Model Serving", "authors": [ "Yinmin Zhong", "Shengyu Liu", "Junda Chen", "Jianbo Hu", "Yibo Zhu", "Xuanzhe Liu", "Xin Jin", "Hao Zhang" ], "year": 2024, "venue_or_source": "OSDI 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2401.09670", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "TTFT/TPOT SLOs" ], "cost_reduction_reported": "serves 7.4x more requests or meets 12.6x tighter SLOs vs state-of-the-art systems while keeping >90% of requests within latency constraints", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Disaggregates prefill and decoding onto different GPUs, co-optimizing resources and parallelism for each phase and placing them by cluster bandwidth to maximize goodput.", "key_idea": "Prefill/decode disaggregation for goodput.", "relevance_score": 2, "relevance_rationale": "Serving background; relevant for prefill-heavy agent traffic.", "found_via": "task seed", "cites_or_cited_by": [ "ding-2026-conserve", "patel-2023-splitwise", "peng-2026-hexagent", "qin-2024-mooncake" ] }, { "id": "zhou-2024-agent-symbolic-learning", "title": "Symbolic Learning Enables Self-Evolving Agents", "authors": [ "Wangchunshu Zhou", "Yixin Ou", "Shengwei Ding", "Long Li", "Jialong Wu", "Tiannan Wang", "Jiamin Chen", "Shuai Wang", "Xiaohua Xu", "Ningyu Zhang", "Huajun Chen", "Yuchen Eleanor Jiang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2406.18532", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "language loss and gradients" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Agent symbolic learning treats agent pipelines as symbolic networks and optimizes prompts, tools and pipeline structure using natural-language analogues of loss, back-propagation and gradient descent, enabling agents to self-evolve after deployment.", "key_idea": "Language-based backprop over agent pipelines.", "relevance_score": 2, "relevance_rationale": "Background optimizer for agent pipelines.", "found_via": "search: Agent Symbolic Learning self-evolving arXiv", "cites_or_cited_by": [] }, { "id": "zhou-2024-archer", "title": "ArCHer: Training Language Model Agents via Hierarchical Multi-Turn RL", "authors": [ "Yifei Zhou", "Andrea Zanette", "Jiayi Pan", "Sergey Levine", "Aviral Kumar" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2402.19446", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "RL-offline", "signals_used": [ "hierarchical value function" ], "cost_reduction_reported": "About 100x sample efficiency over existing methods", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Hierarchical multi-turn RL with an utterance-level off-policy critic and token-level policy, greatly improving sample efficiency for LLM agents.", "key_idea": "Hierarchical actor-critic for multi-turn LLM RL.", "relevance_score": 2, "relevance_rationale": "Foundational multi-turn RL method.", "found_via": "seed", "cites_or_cited_by": [ "zhou-2025-sweet-rl" ] }, { "id": "zhou-2024-efficient-inference-survey", "title": "A Survey on Efficient Inference for Large Language Models", "authors": [ "Zixuan Zhou", "Xuefei Ning", "Ke Hong", "Tianyu Fu", "Jiaming Xu", "Shiyao Li", "Yuming Lou", "Luning Wang", "Zhihang Yuan", "Xiuhong Li", "Shengen Yan", "Guohao Dai", "Xiao-Ping Zhang", "Yuhan Dong", "Yu Wang" ], "year": 2024, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2404.14294", "code_url": null, "topic_clusters": [ "C11", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Comprehensive survey of efficient LLM inference organized into data-level, model-level and system-level optimizations, with comparative experiments.", "key_idea": "Efficient inference taxonomy.", "relevance_score": 2, "relevance_rationale": "Background for serving-level levers.", "found_via": "search: efficient LLM inference survey", "cites_or_cited_by": [] }, { "id": "azaria-2023-internal-state-lying", "title": "The Internal State of an LLM Knows When its Lying", "authors": [ "Amos Azaria", "Tom Mitchell" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2304.13734", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "hidden states", "classifier" ], "cost_reduction_reported": null, "benchmarks": [ "True-False dataset" ], "models_studied": [ "OPT", "LLaMA" ], "production_evidence": false, "summary": "Trains a classifier on LLM hidden activations to judge truthfulness of statements, outperforming few-shot prompting.", "key_idea": "Hidden-state probes reveal statement veracity.", "relevance_score": 2, "relevance_rationale": "Early evidence for probe-based quality signals.", "found_via": "keyword: probes hidden states", "cites_or_cited_by": [] }, { "id": "boubdir-2023-elo-uncovered", "title": "Elo Uncovered: Robustness and Best Practices in Language Model Evaluation", "authors": [ "M. Boubdir", "Edward Kim", "Beyza Ermiş", "Sara Hooker", "Marzieh Fadaee" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.17295", "code_url": null, "topic_clusters": [ "C5", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "pairwise comparisons" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Tests whether Elo ratings for LLMs satisfy reliability and transitivity and finds they often do not, giving guidance on hyperparameters and practice.", "key_idea": "Elo-based LLM evaluation is volatile; best practices.", "relevance_score": 2, "relevance_rationale": "Background for preference-based online evaluation.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "chiang-2024-chatbot-arena" ] }, { "id": "chen-2023-speculative-sampling", "title": "Accelerating Large Language Model Decoding with Speculative Sampling", "authors": [ "Charlie Chen", "Sebastian Borgeaud", "Geoffrey Irving", "Jean-Baptiste Lespiau", "Laurent Sifre", "John Jumper" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2302.01318", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "draft-model proposals", "modified rejection sampling" ], "cost_reduction_reported": "2-2.5x decoding speedup on Chinchilla 70B in a distributed setup without compromising sample quality", "benchmarks": [], "models_studied": [ "Chinchilla 70B" ], "production_evidence": false, "summary": "Concurrent DeepMind formulation of speculative sampling with a modified rejection scheme that preserves the target distribution.", "key_idea": "Speculative sampling for large distributed models.", "relevance_score": 2, "relevance_rationale": "Speculative decoding foundation.", "found_via": "task seed", "cites_or_cited_by": [ "leviathan-2022-speculative-decoding" ] }, { "id": "chen-2023-universal-self-consistency", "title": "Universal Self-Consistency for Large Language Model Generation", "authors": [ "Xinyun Chen", "Renat Aksitov", "Uri Alon", "Jie Ren", "Kefan Xiao", "Pengcheng Yin", "Sushant Prakash", "Charles Sutton", "Xuezhi Wang", "Denny Zhou" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.17311", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "LLM-selected consistency" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Extends self-consistency to free-form outputs by asking the LLM to pick the most consistent among candidates.", "key_idea": "LLM-judged consistency for open-ended outputs.", "relevance_score": 2, "relevance_rationale": "Agreement signal for free-form agent outputs.", "found_via": "keyword: self-consistency", "cites_or_cited_by": [] }, { "id": "chevalier-2023-autocompressors", "title": "Adapting Language Models to Compress Contexts", "authors": [ "Alexis Chevalier", "Alexander Wettig", "Anirudh Ajith", "Danqi Chen" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.14788", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "learned summary vectors" ], "cost_reduction_reported": null, "benchmarks": [ "in-context learning tasks", "passage re-ranking", "retrieval-augmented LM" ], "models_studied": [ "Llama-2-7B" ], "production_evidence": false, "summary": "Fine-tunes LMs to compress long contexts into compact soft \"summary vectors\" that can be precomputed and reused. Summary vectors can substitute for plain-text demonstrations, increasing accuracy while reducing inference cost.", "key_idea": "Learn soft summary vectors that replace long context segments.", "relevance_score": 2, "relevance_rationale": "Soft-prompt compression background; requires model access, so less applicable to closed frontier APIs.", "found_via": "prompt compression survey LLM 2024 2025", "cites_or_cited_by": [ "mu-2023-gist" ] }, { "id": "duan-2023-shifting-attention-relevance", "title": "Shifting Attention to Relevance: Towards the Uncertainty Estimation of Large Language Models", "authors": [ "Jinhao Duan", "Hao Cheng", "Shiqi Wang", "Alex Zavalny", "Chenan Wang", "Renjing Xu", "Bhavya Kailkhura", "Kaidi Xu" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2307.01379", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "token relevance-weighted entropy" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Proposes SAR, reweighting token- and sentence-level uncertainty by semantic relevance, improving uncertainty estimation over treating all tokens equally.", "key_idea": "Relevance-weighted uncertainty.", "relevance_score": 2, "relevance_rationale": "Improved UQ variant.", "found_via": "keyword: uncertainty quantification LLM", "cites_or_cited_by": [] }, { "id": "fadeeva-2023-lm-polygraph", "title": "LM-Polygraph: Uncertainty Estimation for Language Models", "authors": [ "Ekaterina Fadeeva", "Roman Vashurin", "Akim Tsvigun", "Artem Vazhentsev", "Sergey Petrakov", "Kirill Fedyanin", "Daniil Vasilev", "Elizaveta Goncharova", "Alexander Panchenko", "Panov, Maxim", "Timothy J. Baldwin", "Artem Shelmanov" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.07383", "code_url": null, "topic_clusters": [ "C7", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "multiple UE methods" ], "cost_reduction_reported": null, "benchmarks": [ "LM-Polygraph benchmark" ], "models_studied": [], "production_evidence": false, "summary": "Open-source framework implementing many uncertainty estimation methods for LLM generation with a unified API, plus a benchmark and demo app.", "key_idea": "Toolkit for uncertainty estimation.", "relevance_score": 2, "relevance_rationale": "Practical library for evaluating confidence signals.", "found_via": "keyword: uncertainty quantification LLM", "cites_or_cited_by": [] }, { "id": "fernando-2023-promptbreeder", "title": "Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution", "authors": [ "Chrisantha Fernando", "Dylan Banarse", "Henryk Michalewski", "Simon Osindero", "Tim Rocktäschel" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2309.16797", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "fitness on training set" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Promptbreeder evolves task prompts and the mutation prompts that modify them in a self-referential evolutionary loop.", "key_idea": "Self-referential evolution of task and mutation prompts.", "relevance_score": 2, "relevance_rationale": "Foundational prompt optimization technique.", "found_via": "search: Promptbreeder ProTeGi Automatic Prompt Engineer arXiv", "cites_or_cited_by": [] }, { "id": "fu-2023-specializing-smaller-lms", "title": "Specializing Smaller Language Models towards Multi-Step Reasoning", "authors": [ "Yao Fu", "Hao Peng", "Litu Ou", "Ashish Sabharwal", "Tushar Khot" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2301.12726", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "teacher CoT" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Specializes small FlanT5 models to multi-step math reasoning via distillation, trading generic ability for target-task gains.", "key_idea": "Concentrate small model capacity on target task.", "relevance_score": 2, "relevance_rationale": "Specialization tradeoff evidence.", "found_via": "hf-papers search: specialized small model outperforms GPT-4", "cites_or_cited_by": [] }, { "id": "gao-2023-continual-qa-feedback", "title": "Continually Improving Extractive QA via Human Feedback", "authors": [ "Ge Gao", "Hung‐Ting Chen", "Yoav Artzi", "Eunsol Choi" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.12473", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "user feedback" ], "cost_reduction_reported": null, "benchmarks": [ "deployed extractive QA" ], "models_studied": [], "production_evidence": true, "summary": "Deploys an extractive QA system and improves it iteratively from thousands of real user feedback interactions, showing gains over time including domain adaptation.", "key_idea": "Continual improvement from real deployed-user feedback.", "relevance_score": 2, "relevance_rationale": "Real feedback loop evidence; not cost-focused.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "gao-2022-bandit-user-feedback-qa" ] }, { "id": "ge-2023-icae", "title": "In-context Autoencoder for Context Compression in a Large Language Model", "authors": [ "Tao Ge", "Jing Hu", "Lei Wang", "Xun Wang", "Si-Qing Chen", "Furu Wei" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2307.06945", "code_url": "https://github.com/getao/icae", "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "memory slots from an autoencoder" ], "cost_reduction_reported": "4x context compression on Llama with ~1% additional parameters, improving latency and GPU memory", "benchmarks": [], "models_studied": [ "Llama" ], "production_evidence": false, "summary": "The In-context Autoencoder learns to encode long contexts into a small number of memory slots that the frozen LLM can condition on, pretrained with autoencoding and LM objectives then fine-tuned on instruction data.", "key_idea": "Autoencode context into memory slots readable by the base LLM.", "relevance_score": 2, "relevance_rationale": "Soft context compression background.", "found_via": "prompt compression survey LLM 2024 2025", "cites_or_cited_by": [] }, { "id": "geng-2023-survey-confidence-calibration", "title": "A Survey of Confidence Estimation and Calibration in Large Language Models", "authors": [ "Jiahui Geng", "Fengyu Cai", "Yuxia Wang", "Heinz Koeppl", "Preslav Nakov", "Iryna Gurevych" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.08298", "code_url": null, "topic_clusters": [ "C7", "C15" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "confidence" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of confidence estimation and calibration methods for LLMs and their applications.", "key_idea": "Survey of LLM confidence estimation.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "keyword: confidence estimation survey", "cites_or_cited_by": [] }, { "id": "gou-2023-critic", "title": "CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing", "authors": [ "Zhibin Gou", "Zhihong Shao", "Yeyun Gong", "Yelong Shen", "Yujiu Yang", "Nan Duan", "Weizhu Chen" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.11738", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "tool-based verification" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "CRITIC lets LLMs verify and revise outputs using external tools (search, code interpreter), since intrinsic self-verification is unreliable.", "key_idea": "Tool-grounded critique beats self-critique.", "relevance_score": 2, "relevance_rationale": "External verification signal.", "found_via": "keyword: verifier", "cites_or_cited_by": [] }, { "id": "gu-2023-minillm", "title": "MiniLLM: On-Policy Distillation of Large Language Models", "authors": [ "Yuxian Gu", "Li Dong", "Furu Wei", "Minlie Huang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2306.08543", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "reverse KL distillation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "White-box KD for generative LLMs using reverse KL to avoid overestimating low-probability regions.", "key_idea": "Reverse-KL distillation for LLMs.", "relevance_score": 2, "relevance_rationale": "KD background.", "found_via": "keyword: knowledge distillation LLM", "cites_or_cited_by": [] }, { "id": "honeycomb-2023-hard-stuff-llm", "title": "All the Hard Stuff Nobody Talks About when Building Products with LLMs", "authors": [ "Honeycomb" ], "year": 2023, "venue_or_source": "Honeycomb blog", "source_type": "blog", "url": "https://www.honeycomb.io/blog/hard-stuff-nobody-talks-about-llm", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Practitioner write-up of building Honeycomb's Query Assistant, covering context-window limits, latency, prompt injection and evaluation difficulties. Author listed on the page by first name only.", "key_idea": "Early production LLM pitfalls.", "relevance_score": 2, "relevance_rationale": "Background production experience.", "found_via": "seed list: engineering case studies", "cites_or_cited_by": [] }, { "id": "huang-2023-cannot-self-correct", "title": "Large Language Models Cannot Self-Correct Reasoning Yet", "authors": [ "Jie Huang", "Xinyun Chen", "Swaroop Mishra", "Huaixiu Steven Zheng", "Adams Wei Yu", "Xinying Song", "Denny Zhou" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.01798", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "self-correction" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows LLMs struggle to self-correct reasoning without external feedback and sometimes degrade.", "key_idea": "Intrinsic self-correction is not a reliable quality signal.", "relevance_score": 2, "relevance_rationale": "Caveat for self-judging.", "found_via": "keyword: self-verification", "cites_or_cited_by": [] }, { "id": "jimenez-2023-swe-bench", "title": "SWE-bench: Can Language Models Resolve Real-World GitHub Issues?", "authors": [ "Carlos E. Jimenez", "John Yang", "Alexander Wettig", "Shunyu Yao", "Kexin Pei", "Ofir Press", "Karthik Narasimhan" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.06770", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "unit tests" ], "cost_reduction_reported": null, "benchmarks": [ "SWE-bench (2,294 issues, 12 repos)" ], "models_studied": [ "Claude 2", "other LLMs" ], "production_evidence": false, "summary": "Benchmark of 2,294 real GitHub issues and pull requests across 12 Python repositories, requiring multi-file edits validated by tests. At release the best model solved 1.96% of issues.", "key_idea": "Real repository issue resolution as a coding-agent benchmark.", "relevance_score": 2, "relevance_rationale": "Background; basis for many later cost-per-task analyses of coding agents.", "found_via": "web search: SWE-bench", "cites_or_cited_by": [ "zhu-2025-agentic-benchmark-checklist" ] }, { "id": "kim-2023-prometheus", "title": "Prometheus: Inducing Fine-grained Evaluation Capability in Language Models", "authors": [ "Seungone Kim", "Jamin Shin", "Yejin Cho", "Joel Jang", "Shayne Longpre", "Hwaran Lee", "Sangdoo Yun", "Seongjin Shin", "Sungdong Kim", "James Thorne", "Minjoon Seo" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.08491", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "rubric-based judge" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "13B open evaluator trained on GPT-4 feedback with custom rubrics that approaches GPT-4 evaluation when references and rubrics are given.", "key_idea": "Open rubric-based judge.", "relevance_score": 2, "relevance_rationale": "Cheap judge background.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "li-2023-alpaserve", "title": "AlpaServe: Statistical Multiplexing with Model Parallelism for Deep Learning Serving", "authors": [ "Zhuohan Li", "Lianmin Zheng", "Yinmin Zhong", "Vincent Liu", "Ying Sheng", "Xin Jin", "Yanping Huang", "Zhifeng Chen", "Hao Zhang", "Joseph E. Gonzalez", "Ion Stoica" ], "year": 2023, "venue_or_source": "OSDI 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2302.11665", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "bursty workload statistics", "model parallelism placement" ], "cost_reduction_reported": "up to 10x higher request rates or 6x more burstiness within latency constraints for >99% of requests on production workloads", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows model parallelism can statistically multiplex several models across devices under bursty traffic, and searches placement and parallelization strategies for model collections.", "key_idea": "Statistical multiplexing of many models via model parallelism.", "relevance_score": 2, "relevance_rationale": "Multi-model serving background.", "found_via": "task seed", "cites_or_cited_by": [ "duan-2024-muxserve" ] }, { "id": "li-2023-auto-j", "title": "Generative Judge for Evaluating Alignment", "authors": [ "Junlong Li", "Shichao Sun", "Weizhe Yuan", "Run-Ze Fan", "Hai Zhao", "Pengfei Liu" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2310.05470", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "generative judge" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "13B generative judge supporting pairwise and single-response critiques across 58 scenarios, outperforming strong baselines.", "key_idea": "Generative judge model.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "keyword: small judge", "cites_or_cited_by": [] }, { "id": "lin-2023-awq", "title": "AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration", "authors": [ "Ji Lin", "Jiaming Tang", "Haotian Tang", "Shang Yang", "Wei-Ming Chen", "Wei-Chen Wang", "Guangxuan Xiao", "Xingyu Dang", "Chuang Gan", "Song Han" ], "year": 2023, "venue_or_source": "MLSys 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2306.00978", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "activation-aware salient channel scaling" ], "cost_reduction_reported": "TinyChat 4-bit inference over 3x faster than HuggingFace FP16 on desktop and mobile GPUs", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Protects the small fraction of salient weight channels (identified from activations) via per-channel scaling before low-bit weight quantization.", "key_idea": "Activation-aware weight-only quantization.", "relevance_score": 2, "relevance_rationale": "Quantization background.", "found_via": "task seed", "cites_or_cited_by": [ "frantar-2022-gptq", "kurtic-2024-bf16-or-death" ] }, { "id": "lin-2023-instinct", "title": "Use Your INSTINCT: INSTruction optimization for LLMs usIng Neural bandits Coupled with Transformers", "authors": [ "Xiaoqiang Lin", "Zhaoxuan Wu", "Zhongxiang Dai", "Wenyang Hu", "Yao Shu", "See-Kiong Ng", "Patrick Jaillet", "Bryan Kian Hsiang Low" ], "year": 2023, "venue_or_source": "ICML 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.02905", "code_url": "https://github.com/xqlin98/INSTINCT", "topic_clusters": [ "C4", "C5" ], "routing_granularity": "none", "learning_regime": "bandit-online", "signals_used": [ "LLM hidden representations", "task score" ], "cost_reduction_reported": null, "benchmarks": [ "instruction induction", "zero-shot CoT" ], "models_studied": [ "ChatGPT" ], "production_evidence": false, "summary": "INSTINCT optimizes instructions for black-box LLMs with a neural bandit whose surrogate uses a pre-trained transformer's representations.", "key_idea": "Neural-bandit instruction optimization.", "relevance_score": 2, "relevance_rationale": "Bandit configuration search background.", "found_via": "OpenAlex title: bandit LLM", "cites_or_cited_by": [ "shi-2024-triple-bai-prompt" ] }, { "id": "liu-2023-agentbench", "title": "AgentBench: Evaluating LLMs as Agents", "authors": [ "Xiao Liu", "Hao Yu", "Hanchen Zhang", "Yifan Xu", "Xuanyu Lei", "Hanyu Lai", "Yu Gu", "Hangliang Ding", "Kaiwen Men", "Kejuan Yang", "Shudan Zhang", "Xiang Deng", "Aohan Zeng", "Zhengxiao Du", "Chenhui Zhang", "Sheng Shen", "Tianjun Zhang", "Yu Su", "Huan Sun", "Minlie Huang", "Yuxiao Dong", "Jie Tang" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2308.03688", "code_url": "https://github.com/THUDM/AgentBench", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "AgentBench (8 environments)" ], "models_studied": [ "29+ API and open LLMs" ], "production_evidence": false, "summary": "Multi-environment benchmark of LLMs acting as agents in eight interactive settings, testing reasoning and decision-making. Commercial models lead, with a large gap to open models, often due to weak long-horizon planning and instruction following.", "key_idea": "Broad multi-environment agent capability benchmark.", "relevance_score": 2, "relevance_rationale": "Seed; background capability benchmark without cost focus.", "found_via": "seed: AgentBench", "cites_or_cited_by": [] }, { "id": "liu-2023-cachegen", "title": "CacheGen: KV Cache Compression and Streaming for Fast Large Language Model Serving", "authors": [ "Yuhan Liu", "Hanchen Li", "Yihua Cheng", "Siddhant Ray", "Yuyang Huang", "Qizheng Zhang", "Kuntai Du", "Jiayi Yao", "Shan Lu", "Ganesh Ananthanarayanan", "Michael Maire", "Henry Hoffmann", "Ari Holtzman", "Junchen Jiang" ], "year": 2023, "venue_or_source": "SIGCOMM 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.07240", "code_url": "https://github.com/UChi-JCL/CacheGen", "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "available bandwidth" ], "cost_reduction_reported": "KV cache size reduced 3.5-4.3x and context fetch+processing delay reduced 3.2-3.7x with negligible quality impact", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Encodes KV caches into compact bitstreams and adapts compression to network bandwidth so long contexts can be loaded quickly from remote storage instead of recomputed.", "key_idea": "Compress and stream KV caches to reuse long contexts across machines.", "relevance_score": 2, "relevance_rationale": "Systems background for storing and reusing agent context KV.", "found_via": "CacheBlend fast LLM serving RAG cached knowledge fusion", "cites_or_cited_by": [ "yao-2024-cacheblend" ] }, { "id": "liu-2023-calibrating-llm-evaluator", "title": "Calibrating LLM-Based Evaluator", "authors": [ "Yuxuan Liu", "Tianchi Yang", "Shaohan Huang", "Zihan Zhang", "Haizhen Huang", "Furu Wei", "Weiwei Deng", "Feng Sun", "Qi Zhang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2309.13308", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "evaluator calibration" ], "cost_reduction_reported": null, "benchmarks": [ "NLG evaluation datasets" ], "models_studied": [], "production_evidence": false, "summary": "AutoCalibrate aligns LLM evaluators with human preferences by inducing and refining scoring criteria from a few human labels, without gradient training.", "key_idea": "Calibrate judge criteria with few labels.", "relevance_score": 2, "relevance_rationale": "Judge calibration technique.", "found_via": "keyword: LLM-as-a-judge", "cites_or_cited_by": [] }, { "id": "liu-2023-g-eval", "title": "G-Eval: NLG Evaluation using GPT-4 with Better Human Alignment", "authors": [ "Yang Liu", "Dan Iter", "Yichong Xu", "Shuohang Wang", "Ruochen Xu", "Chenguang Zhu" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2303.16634", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "LLM judge with CoT" ], "cost_reduction_reported": null, "benchmarks": [ "summarization" ], "models_studied": [ "GPT-4" ], "production_evidence": false, "summary": "GPT-4 with chain-of-thought and form-filling for NLG evaluation, reaching 0.514 Spearman with humans on summarization.", "key_idea": "CoT-based LLM judge.", "relevance_score": 2, "relevance_rationale": "Background judge method.", "found_via": "keyword: LLM-as-a-judge", "cites_or_cited_by": [] }, { "id": "liu-2023-lost-in-the-middle", "title": "Lost in the Middle: How Language Models Use Long Contexts", "authors": [ "Nelson F. Liu", "Kevin Lin", "John Hewitt", "Ashwin Paranjape", "Michele Bevilacqua", "Fabio Petroni", "Percy Liang" ], "year": 2023, "venue_or_source": "TACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2307.03172", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "position of relevant information" ], "cost_reduction_reported": null, "benchmarks": [ "multi-document QA", "key-value retrieval" ], "models_studied": [], "production_evidence": false, "summary": "Shows that LLM accuracy is highest when relevant information is at the start or end of the context and degrades substantially when it is in the middle, even for long-context models.", "key_idea": "More context is not free: position-dependent degradation motivates pruning and reordering.", "relevance_score": 2, "relevance_rationale": "Background motivating context compression and curation in long agent trajectories.", "found_via": "task seed", "cites_or_cited_by": [ "jiang-2023-longllmlingua" ] }, { "id": "mao-2023-l2d-multiple-experts", "title": "Principled Approaches for Learning to Defer with Multiple Experts", "authors": [ "Anqi Mao", "Mehryar Mohri", "Yutao Zhong" ], "year": 2023, "venue_or_source": "ISAIM 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2310.14774", "code_url": null, "topic_clusters": [ "C2", "C15" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "SVHN", "CIFAR-10" ], "models_studied": [], "production_evidence": false, "summary": "Introduces surrogate losses for learning to defer among multiple experts, with H-consistency bounds, jointly learning prediction and deferral functions.", "key_idea": "H-consistent surrogates for multi-expert deferral.", "relevance_score": 2, "relevance_rationale": "Theory for deferral among many models, i.e. multi-model routing.", "found_via": "arXiv title search: learning to defer", "cites_or_cited_by": [] }, { "id": "mialon-2023-gaia", "title": "GAIA: a benchmark for General AI Assistants", "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.12983", "code_url": null, "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [ "GAIA (466 questions, 3 levels)" ], "models_studied": [ "GPT-4 with plugins" ], "production_evidence": false, "summary": "Benchmark of real-world assistant questions that are simple for humans but need reasoning, multimodality, browsing and tool use; humans score 92% versus 15% for GPT-4 with plugins at release.", "key_idea": "Human-easy, AI-hard assistant tasks.", "relevance_score": 2, "relevance_rationale": "Common agent benchmark used for cost-of-pass efficiency studies.", "found_via": "web search: GAIA benchmark", "cites_or_cited_by": [ "wang-2025-efficient-agents" ] }, { "id": "miao-2023-specinfer", "title": "SpecInfer: Accelerating Generative Large Language Model Serving with Tree-based Speculative Inference and Verification", "authors": [ "Xupeng Miao", "Gabriele Oliaro", "Zhihao Zhang", "Xinhao Cheng", "Zeyu Wang", "Zhengxin Zhang", "Rae Ying Yee Wong", "Alan Zhu", "Lijie Yang", "Xiaoxiang Shi", "Chunan Shi", "Zhuoming Chen", "Daiyaan Arfeen", "Reyna Abhyankar", "Zhihao Jia" ], "year": 2023, "venue_or_source": "ASPLOS 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2305.09781", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "token", "learning_regime": "training-free", "signals_used": [ "token-tree speculation" ], "cost_reduction_reported": "1.5-2.8x faster for distributed inference and 2.6-3.5x for offloading-based inference while preserving generation", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Small speculative models propose token trees verified by the LLM in parallel with tree-based decoding.", "key_idea": "Tree-based speculative inference.", "relevance_score": 2, "relevance_rationale": "Speculative serving background.", "found_via": "speculative decoding survey Xia unlocking efficiency", "cites_or_cited_by": [ "leviathan-2022-speculative-decoding" ] }, { "id": "mu-2023-gist", "title": "Learning to Compress Prompts with Gist Tokens", "authors": [ "Jesse Mu", "Xiang Lisa Li", "Noah Goodman" ], "year": 2023, "venue_or_source": "NeurIPS 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2304.08467", "code_url": null, "topic_clusters": [ "C10" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "gist tokens" ], "cost_reduction_reported": "up to 26x prompt compression, up to 40% FLOPs reduction and 4.2% wall-time speedup with minimal quality loss", "benchmarks": [ "instruction-following (Alpaca+)" ], "models_studied": [ "LLaMA-7B", "FLAN-T5-XXL" ], "production_evidence": false, "summary": "Trains models to compress a prompt into a few \"gist\" tokens via modified attention masks during instruction tuning; gist activations can be cached and reused across calls.", "key_idea": "Compress a recurring prompt into cacheable gist tokens.", "relevance_score": 2, "relevance_rationale": "Foundational soft-compression work linking compression and caching.", "found_via": "prompt compression survey LLM 2024 2025", "cites_or_cited_by": [ "chevalier-2023-autocompressors" ] }, { "id": "mukherjee-2023-orca", "title": "Orca: Progressive Learning from Complex Explanation Traces of GPT-4", "authors": [ "Subhabrata Mukherjee", "Arindam Mitra", "Ganesh Jawahar", "Sahaj Agarwal", "Hamid Palangi", "Ahmed Awadallah" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2306.02707", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "GPT-4 explanation traces" ], "cost_reduction_reported": null, "benchmarks": [ "BBH", "AGIEval" ], "models_studied": [ "LLaMA-13B", "GPT-4" ], "production_evidence": false, "summary": "Imitation learning from GPT-4 explanation traces and step-by-step thought processes to train a 13B model with stronger reasoning.", "key_idea": "Learn from rich teacher explanations.", "relevance_score": 2, "relevance_rationale": "Background distillation.", "found_via": "keyword: teacher-student distillation", "cites_or_cited_by": [] }, { "id": "patel-2023-splitwise", "title": "Splitwise: Efficient generative LLM inference using phase splitting", "authors": [ "Pratyush Patel", "Esha Choukse", "Chaojie Zhang", "Aashaka Shah", "Íñigo Goiri", "Saeed Maleki", "Ricardo Bianchini" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2311.18677", "code_url": null, "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "phase-specific hardware" ], "cost_reduction_reported": "1.4x higher throughput at 20% lower cost, or 2.35x more throughput at the same cost and power", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Splits prompt computation and token generation onto separate machines (possibly different hardware), transferring KV state over fast interconnects, and designs clusters for throughput, cost and power.", "key_idea": "Phase splitting onto cost-appropriate hardware.", "relevance_score": 2, "relevance_rationale": "Cost-oriented serving background.", "found_via": "task seed", "cites_or_cited_by": [ "zhong-2024-distserve" ] }, { "id": "pryzant-2023-protegi", "title": "Automatic Prompt Optimization with \"Gradient Descent\" and Beam Search", "authors": [ "Reid Pryzant", "Dan Iter", "Jerry Li", "Yin Tat Lee", "Chenguang Zhu", "Michael Zeng" ], "year": 2023, "venue_or_source": "EMNLP 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2305.03495", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "textual gradients", "bandit beam selection" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "ProTeGi (APO) forms natural-language 'gradients' from minibatch errors, edits prompts in the opposite semantic direction, and uses beam search with bandit selection to retain top prompts.", "key_idea": "Textual-gradient prompt optimization with bandit beam search.", "relevance_score": 2, "relevance_rationale": "Foundational prompt optimization technique.", "found_via": "search: Promptbreeder ProTeGi Automatic Prompt Engineer arXiv", "cites_or_cited_by": [] }, { "id": "quach-2023-conformal-language-modeling", "title": "Conformal Language Modeling", "authors": [ "Victor Quach", "Adam Fisch", "Tal Schuster", "Adam Yala", "Jae Ho Sohn", "Tommi S. Jaakkola", "Regina Barzilay" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2306.10193", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "conformal calibration", "rejection rule" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Adapts conformal prediction to generative LMs by sampling until a calibrated stopping rule guarantees a correct answer in the set with high probability.", "key_idea": "Conformal guarantees for generation.", "relevance_score": 2, "relevance_rationale": "Statistical guarantees for escalation thresholds.", "found_via": "keyword: uncertainty quantification LLM", "cites_or_cited_by": [] }, { "id": "schick-2023-toolformer", "title": "Toolformer: Language Models Can Teach Themselves to Use Tools", "authors": [ "Timo Schick", "Jane Dwivedi-Yu", "Roberto Dessì", "Roberta Raileanu", "Maria Lomeli", "Luke Zettlemoyer", "Nicola Cancedda", "Thomas Scialom" ], "year": 2023, "venue_or_source": "NeurIPS 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2302.04761", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "self-supervised API-call filtering by loss reduction" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-J 6.7B" ], "production_evidence": false, "summary": "LM teaches itself to insert API calls by sampling candidate calls and keeping those that reduce language-modeling loss, then fine-tuning on the result.", "key_idea": "Self-supervised tool-use data from loss reduction.", "relevance_score": 2, "relevance_rationale": "Foundational self-supervision recipe.", "found_via": "seed", "cites_or_cited_by": [] }, { "id": "shen-2023-hugginggpt", "title": "HuggingGPT: Solving AI Tasks with ChatGPT and its Friends in Hugging Face", "authors": [ "Yongliang Shen", "Kaitao Song", "Xu Tan", "Dongsheng Li", "Weiming Lu", "Yueting Zhuang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2303.17580", "code_url": null, "topic_clusters": [ "C3" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "model descriptions" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "ChatGPT", "Hugging Face models" ], "production_evidence": false, "summary": "An LLM controller plans a user request into subtasks, picks a specialist model from Hugging Face for each subtask using model descriptions, runs them and summarizes the results. An early example of per-subtask model selection by an orchestrating LLM, though aimed at capability rather than cost.", "key_idea": "LLM planner assigns each decomposed subtask to a specialist model chosen by its description.", "relevance_score": 2, "relevance_rationale": "Background precedent for subtask-level model selection; no cost optimization.", "found_via": "web search: HuggingGPT", "cites_or_cited_by": [] }, { "id": "tang-2023-toolalpaca", "title": "ToolAlpaca: Generalized Tool Learning for Language Models with 3000 Simulated Cases", "authors": [ "Qiaoyu Tang", "Ziliang Deng", "Hongyu Lin", "Xianpei Han", "Qiao Liang", "Boxi Cao", "Le Sun" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2306.05301", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "simulated tool-use cases" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Simulates 3,938 tool-use instances over 400+ APIs with multi-agent simulation to fine-tune compact models with generalized tool use.", "key_idea": "Multi-agent simulated tool data.", "relevance_score": 2, "relevance_rationale": "Synthetic data background.", "found_via": "keyword: synthetic agent data", "cites_or_cited_by": [] }, { "id": "vivek-2023-anchor-points", "title": "Anchor Points: Benchmarking Models with Much Fewer Examples", "authors": [ "Rajan Vivek", "Kawin Ethayarajh", "Diyi Yang", "Douwe Kiela" ], "year": 2023, "venue_or_source": "EACL 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2309.08638", "code_url": null, "topic_clusters": [ "C6", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "cross-model confidence correlation" ], "cost_reduction_reported": null, "benchmarks": [ "six classification benchmarks" ], "models_studied": [ "87 model-prompt pairs" ], "production_evidence": false, "summary": "Anchor Point Selection picks tiny subsets whose model confidences represent the whole dataset, reliably ranking models and estimating per-point predictions.", "key_idea": "Rank models from 1-30 anchor examples.", "relevance_score": 2, "relevance_rationale": "Background on cheap evaluation.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "polo-2024-tinybenchmarks" ] }, { "id": "wan-2023-efficient-llms-survey", "title": "Efficient Large Language Models: A Survey", "authors": [ "Zhongwei Wan", "Xin Wang", "Che Liu", "Samiul Alam", "Yu Zheng", "Jiachen Liu", "Zhongnan Qu", "Shen Yan", "Yi Zhu", "Quanlu Zhang", "Mosharaf Chowdhury", "Mi Zhang" ], "year": 2023, "venue_or_source": "TMLR", "source_type": "paper", "url": "https://arxiv.org/abs/2312.03863", "code_url": "https://github.com/AIoT-MLSys-Lab/Efficient-LLMs-Survey", "topic_clusters": [ "C11", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Survey of efficient LLM research from model-centric, data-centric and framework-centric perspectives.", "key_idea": "Broad efficiency survey.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "search: efficient LLM inference survey", "cites_or_cited_by": [] }, { "id": "wang-2023-llms-not-fair-evaluators", "title": "Large Language Models are not Fair Evaluators", "authors": [ "Peiyi Wang", "Lei Li", "Liang Chen", "Zefan Cai", "Dawei Zhu", "Binghuai Lin", "Yunbo Cao", "Qi Liu", "Tianyu Liu", "Zhifang Sui" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.17926", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "judge score", "position calibration" ], "cost_reduction_reported": null, "benchmarks": [ "Vicuna Bench" ], "models_studied": [ "GPT-4", "ChatGPT" ], "production_evidence": false, "summary": "Shows LLM judges are sensitive to answer order (e.g., Vicuna-13B beats ChatGPT on 66/80 queries by reordering) and proposes calibration strategies including human-in-the-loop for hard cases.", "key_idea": "Position bias in LLM judges and its mitigation.", "relevance_score": 2, "relevance_rationale": "Caveat for judge-driven routing.", "found_via": "keyword: LLM-as-a-judge bias", "cites_or_cited_by": [] }, { "id": "xiao-2023-streamingllm", "title": "Efficient Streaming Language Models with Attention Sinks", "authors": [ "Guangxuan Xiao", "Yuandong Tian", "Beidi Chen", "Song Han", "Mike Lewis" ], "year": 2023, "venue_or_source": "ICLR 2024", "source_type": "paper", "url": "https://arxiv.org/abs/2309.17453", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "attention sinks" ], "cost_reduction_reported": "up to 22.2x speedup over sliding-window recomputation in streaming settings", "benchmarks": [], "models_studied": [ "Llama-2", "MPT", "Falcon", "Pythia" ], "production_evidence": false, "summary": "Observes that initial tokens act as attention sinks; keeping them plus a recent window lets models stream over millions of tokens stably.", "key_idea": "Keep attention-sink tokens plus a recent window.", "relevance_score": 2, "relevance_rationale": "Background on bounded-KV long-running sessions.", "found_via": "KV cache compression survey LLM inference efficiency", "cites_or_cited_by": [ "zhang-2023-h2o" ] }, { "id": "xie-2023-self-eval-guided-decoding", "title": "Self-Evaluation Guided Beam Search for Reasoning", "authors": [ "Yuxi Xie", "Kenji Kawaguchi", "Yiran Zhao", "Xu Zhao", "Min-Yen Kan", "Junxian He", "Qizhe Xie" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.00633", "code_url": null, "topic_clusters": [ "C7", "C9" ], "routing_granularity": "step", "learning_regime": "training-free", "signals_used": [ "self-evaluation" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Codex" ], "production_evidence": false, "summary": "Uses step-level self-evaluation to guide stochastic beam search over reasoning chains, improving accuracy.", "key_idea": "Self-evaluation as step-level guidance.", "relevance_score": 2, "relevance_rationale": "Step-level signal background.", "found_via": "keyword: step-level reward", "cites_or_cited_by": [] }, { "id": "yang-2023-gpt4tools", "title": "GPT4Tools: Teaching Large Language Model to Use Tools via Self-instruction", "authors": [ "Rui Yang", "Lin Song", "Yanwei Li", "Sijie Zhao", "Yixiao Ge", "Xiu Li", "Ying Shan" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.18752", "code_url": "https://github.com/StevenGrove/GPT4Tools", "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "self-instruct from ChatGPT" ], "cost_reduction_reported": null, "benchmarks": [ "GPT4Tools eval" ], "models_studied": [ "LLaMA", "OPT" ], "production_evidence": false, "summary": "Self-instruct data from ChatGPT teaches open LLMs to use multimodal tools via LoRA.", "key_idea": "Self-instruct tool-use data.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "keyword: teacher-student for tool use", "cites_or_cited_by": [] }, { "id": "yang-2023-opro", "title": "Large Language Models as Optimizers", "authors": [ "Chengrun Yang", "Xuezhi Wang", "Yifeng Lu", "Hanxiao Liu", "Quoc V. Le", "Denny Zhou", "Xinyun Chen" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2309.03409", "code_url": "https://github.com/google-deepmind/opro", "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "score of previous solutions in meta-prompt" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K", "Big-Bench Hard" ], "models_studied": [], "production_evidence": false, "summary": "OPRO uses an LLM as an optimizer: the meta-prompt lists earlier solutions and their scores, and the LLM proposes new candidates each step. Optimized prompts beat human prompts by up to 8% on GSM8K and up to 50% on Big-Bench Hard.", "key_idea": "Optimization by prompting: LLM proposes new solutions conditioned on scored history.", "relevance_score": 2, "relevance_rationale": "Foundational LLM-as-optimizer method used inside many config optimizers; no cost objective.", "found_via": "seed: OPRO", "cites_or_cited_by": [] }, { "id": "yin-2023-selfaware", "title": "Do Large Language Models Know What They Don't Know?", "authors": [ "Zhangyue Yin", "Qiushi Sun", "Qipeng Guo", "Jiawen Wu", "Xipeng Qiu", "Xuanjing Huang" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2305.18153", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [ "self-knowledge" ], "cost_reduction_reported": null, "benchmarks": [ "SelfAware" ], "models_studied": [ "GPT-3", "InstructGPT", "LLaMA" ], "production_evidence": false, "summary": "Evaluates whether 20 LLMs recognize unanswerable questions, finding intrinsic but limited self-knowledge.", "key_idea": "Measures LLM self-knowledge.", "relevance_score": 2, "relevance_rationale": "Background.", "found_via": "keyword: confidence estimation", "cites_or_cited_by": [] }, { "id": "zhang-2023-h2o", "title": "H2O: Heavy-Hitter Oracle for Efficient Generative Inference of Large Language Models", "authors": [ "Zhenyu Zhang", "Ying Sheng", "Tianyi Zhou", "Tianlong Chen", "Lianmin Zheng", "Ruisi Cai", "Zhao Song", "Yuandong Tian", "Christopher Ré", "Clark Barrett", "Zhangyang Wang", "Beidi Chen" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2306.14048", "code_url": null, "topic_clusters": [ "C10", "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "accumulated attention scores" ], "cost_reduction_reported": "with 20% heavy hitters, throughput improves up to 29x over DeepSpeed Zero-Inference and HF Accelerate and 3x over FlexGen; latency reduced up to 1.9x", "benchmarks": [], "models_studied": [ "OPT", "LLaMA", "GPT-NeoX" ], "production_evidence": false, "summary": "Keeps only \"heavy hitter\" tokens with high accumulated attention plus recent tokens in the KV cache, greatly reducing memory with little accuracy loss.", "key_idea": "Evict KV entries by accumulated attention.", "relevance_score": 2, "relevance_rationale": "KV-compression background for long agent contexts in self-hosted serving.", "found_via": "KV cache compression survey LLM inference efficiency", "cites_or_cited_by": [ "li-2024-snapkv", "wang-2026-actkv", "xiao-2023-streamingllm" ] }, { "id": "zhou-2023-predictable-ai", "title": "Predictable Artificial Intelligence", "authors": [ "Lexin Zhou", "Pablo A. Moreno-Casares", "Fernando Martínez‐Plumed", "John Burden", "Ryan Burnell", "Lucy G. Cheke", "Cèsar Ferri", "Alexandru Marcoci", "Behzad Mehrbakhsh", "Yael Moros-Daval", "Seán Ó hÉigeartaigh", "Danaja Rutar", "Wout Schellaert", "Konstantinos Voudouris", "José Hernández‐Orallo" ], "year": 2023, "venue_or_source": "Artificial Intelligence (journal)", "source_type": "paper", "url": "https://arxiv.org/abs/2310.06167", "code_url": null, "topic_clusters": [ "C6", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Position paper defining Predictable AI: anticipating validity indicators such as performance and safety of AI systems, and the candidate predictors and trade-offs involved.", "key_idea": "Framework for predicting AI performance.", "relevance_score": 2, "relevance_rationale": "Conceptual background for assessor-based routing.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "pacchiardi-2025-predictaboard" ] }, { "id": "zhou-2023-webarena", "title": "WebArena: A Realistic Web Environment for Building Autonomous Agents", "authors": [ "Shuyan Zhou", "Frank F. Xu", "Hao Zhu", "Xuhui Zhou", "Robert Lo", "Abishek Sridhar", "Xianyi Cheng", "Tianyue Ou", "Yonatan Bisk", "Daniel Fried", "Uri Alon", "Graham Neubig" ], "year": 2023, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2307.13854", "code_url": "https://github.com/web-arena-x/webarena", "topic_clusters": [ "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "programmatic functional-correctness checks" ], "cost_reduction_reported": null, "benchmarks": [ "WebArena (812 tasks)" ], "models_studied": [ "GPT-4", "other LLMs" ], "production_evidence": false, "summary": "Self-hostable realistic web environment with sites for e-commerce, forums, code collaboration and CMS, plus tools and knowledge resources, and 812 long-horizon tasks checked for functional correctness.", "key_idea": "Reproducible realistic websites for agent evaluation.", "relevance_score": 2, "relevance_rationale": "Background; used in routing evaluation studies.", "found_via": "web search: WebArena", "cites_or_cited_by": [ "chezelles-2024-browsergym", "kumar-2026-task-session-routing-eval" ] }, { "id": "frantar-2022-gptq", "title": "GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers", "authors": [ "Elias Frantar", "Saleh Ashkboos", "Torsten Hoefler", "Dan Alistarh" ], "year": 2022, "venue_or_source": "ICLR 2023", "source_type": "paper", "url": "https://arxiv.org/abs/2210.17323", "code_url": "https://github.com/IST-DASLab/gptq", "topic_clusters": [ "C11" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "approximate second-order weight updates" ], "cost_reduction_reported": "end-to-end inference speedups over FP16 of ~3.25x on A100 and ~4.5x on A6000", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "One-shot post-training weight quantization to 3-4 bits using approximate second-order information, scaling to 175B-parameter models.", "key_idea": "Accurate one-shot 3-4 bit weight quantization.", "relevance_score": 2, "relevance_rationale": "Quantization background for cheaper self-hosted serving.", "found_via": "task seed", "cites_or_cited_by": [ "kurtic-2024-bf16-or-death", "lin-2023-awq" ] }, { "id": "gao-2022-bandit-user-feedback-qa", "title": "Simulating Bandit Learning from User Feedback for Extractive Question Answering", "authors": [ "Ge Gao", "Eunsol Choi", "Yoav Artzi" ], "year": 2022, "venue_or_source": "ACL 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2203.10079", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "user feedback" ], "cost_reduction_reported": null, "benchmarks": [ "extractive QA datasets" ], "models_studied": [], "production_evidence": false, "summary": "Simulates contextual-bandit learning from user feedback on extractive QA, showing models trained on few examples improve substantially from feedback and can adapt to new domains without annotation.", "key_idea": "Bandit learning from simulated user feedback in QA.", "relevance_score": 2, "relevance_rationale": "Background on learning from deployment feedback.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "gao-2023-continual-qa-feedback" ] }, { "id": "ho-2022-reasoning-teachers", "title": "Large Language Models Are Reasoning Teachers", "authors": [ "Namgyu Ho", "Laura Schmid", "Se-Young Yun" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2212.10071", "code_url": null, "topic_clusters": [ "C8" ], "routing_granularity": "none", "learning_regime": "supervised-offline", "signals_used": [ "teacher CoT samples" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Fine-tune-CoT uses large models to generate reasoning samples that fine-tune much smaller students, often outperforming the teacher.", "key_idea": "CoT distillation into small students.", "relevance_score": 2, "relevance_rationale": "Background distillation.", "found_via": "keyword: teacher-student distillation", "cites_or_cited_by": [] }, { "id": "khalili-2022-babybear", "title": "BabyBear: Cheap inference triage for expensive language models", "authors": [ "Leila Khalili", "Yao You", "John Bohannon" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2205.11747", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "cheap-model confidence" ], "cost_reduction_reported": "Reduces compute of large-scale classification jobs by more than 50% while retaining accuracy; saves 33% of compute for NER while keeping F1 above 95% on CoNLL", "benchmarks": [ "CoNLL", "document classification datasets" ], "models_studied": [ "transformer LMs" ], "production_evidence": false, "summary": "Inference triage for NLP: a cheap model trained from a deep model's outputs handles confident cases and exits early; the expensive model handles the rest.", "key_idea": "Early-exit triage cascade for NLP classification.", "relevance_score": 2, "relevance_rationale": "Pre-LLM cascade background.", "found_via": "arXiv search: model cascade language model", "cites_or_cited_by": [] }, { "id": "kiyohara-2022-cascade-dr", "title": "Doubly Robust Off-Policy Evaluation for Ranking Policies under the Cascade Behavior Model", "authors": [ "Haruka Kiyohara", "Yuta Saito", "Tatsuya Matsuhiro", "Yusuke Narita", "Nobuyuki Shimizu", "Yasuo Yamamoto" ], "year": 2022, "venue_or_source": "WSDM 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2202.01562", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "cascade behavior assumption" ], "cost_reduction_reported": null, "benchmarks": [ "synthetic and real e-commerce data" ], "models_studied": [], "production_evidence": false, "summary": "Cascade doubly robust estimator for off-policy evaluation of ranking policies, reducing variance under a top-down user behavior assumption.", "key_idea": "Cascade-DR OPE for rankings.", "relevance_score": 2, "relevance_rationale": "OPE for sequential decisions, analogous to cascades.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "dudik-2011-doubly-robust" ] }, { "id": "lin-2022-verbalized-probability", "title": "Teaching Models to Express Their Uncertainty in Words", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2205.14334", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "verbalized confidence" ], "cost_reduction_reported": null, "benchmarks": [ "CalibratedMath" ], "models_studied": [ "GPT-3" ], "production_evidence": false, "summary": "Fine-tunes GPT-3 to output calibrated confidence in words alongside answers, remaining moderately calibrated under distribution shift.", "key_idea": "Models can be taught verbalized calibrated uncertainty.", "relevance_score": 2, "relevance_rationale": "Background for verbalized confidence signals.", "found_via": "keyword: verbalized confidence", "cites_or_cited_by": [] }, { "id": "liu-2022-nonstationary-bwk", "title": "Non-stationary Bandits with Knapsacks", "authors": [ "Shang Liu", "Jiashuo Jiang", "Xiaocheng Li" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2205.12427", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Studies bandits with knapsacks in non-stationary environments, proposing a global non-stationarity measure and primal-dual bounds.", "key_idea": "Non-stationary BwK.", "relevance_score": 2, "relevance_rationale": "Theory for budgeted routing under drift.", "found_via": "OpenAlex title: bandits with knapsacks", "cites_or_cited_by": [ "badanidiyuru-2013-bwk", "taberner-miller-2026-paretobandit" ] }, { "id": "schuster-2022-calm", "title": "Confident Adaptive Language Modeling", "authors": [ "Tal Schuster", "Adam Fisch", "Jai Gupta", "Mostafa Dehghani", "Dara Bahri", "Vinh Q. Tran", "Yi Tay", "Donald Metzler" ], "year": 2022, "venue_or_source": "NeurIPS 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2207.07061", "code_url": null, "topic_clusters": [ "C2", "C9", "C11" ], "routing_granularity": "token", "learning_regime": "supervised-offline", "signals_used": [ "per-token confidence measures" ], "cost_reduction_reported": "Potential speedup of up to 3x while provably maintaining high performance on three generation tasks", "benchmarks": [ "three text generation tasks" ], "models_studied": [], "production_evidence": false, "summary": "Confident Adaptive Language Modeling exits early per decoding step when a confidence measure is high, with calibration linking sequence-level quality constraints to per-token exit decisions.", "key_idea": "Per-token early exit with calibrated quality guarantees.", "relevance_score": 2, "relevance_rationale": "Foundational early-exit adaptive compute.", "found_via": "keyword: early exit", "cites_or_cited_by": [] }, { "id": "varshney-2022-model-cascading-nlp", "title": "Model Cascading: Towards Jointly Improving Efficiency and Accuracy of NLP Systems", "authors": [ "Neeraj Varshney", "Chitta Baral" ], "year": 2022, "venue_or_source": "EMNLP 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2210.05528", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "model confidence" ], "cost_reduction_reported": "In K=3 setting, saves up to 88.93% computation cost with up to 2.18% accuracy improvement", "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Explorative study of cascading models of varying capacity for NLP classification; shows cascades can improve both efficiency and accuracy and more models increase gains.", "key_idea": "Cascades improve both efficiency and accuracy for NLP.", "relevance_score": 2, "relevance_rationale": "Pre-LLM cascade background.", "found_via": "snowball: related work of cascade papers (verified on arXiv)", "cites_or_cited_by": [] }, { "id": "verma-2022-calibrated-l2d-ova", "title": "Calibrated Learning to Defer with One-vs-All Classifiers", "authors": [ "Rajeev Verma", "Eric Nalisnick" ], "year": 2022, "venue_or_source": "ICML 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2202.03673", "code_url": null, "topic_clusters": [ "C2", "C7" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "calibrated expert-correctness probability" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Shows the Mozannar-Sontag L2D model is not calibrated for expert correctness and proposes a one-vs-all formulation that yields calibrated probabilities and remains a consistent surrogate.", "key_idea": "Calibrated L2D via one-vs-all classifiers.", "relevance_score": 2, "relevance_rationale": "Foundational; calibration matters for deferral thresholds.", "found_via": "arXiv title search: learning to defer", "cites_or_cited_by": [] }, { "id": "xu-2022-learning-after-deployment", "title": "Learning New Skills after Deployment: Improving open-domain internet-driven dialogue with human feedback", "authors": [ "Jing Xu", "Megan Ung", "Mojtaba Komeili", "Kushal Arora", "Y-Lan Boureau", "Jason Weston" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2208.03270", "code_url": null, "topic_clusters": [ "C5", "C6" ], "routing_granularity": "n/a", "learning_regime": "hybrid", "signals_used": [ "binary feedback", "free-form feedback", "failure reasons" ], "cost_reduction_reported": null, "benchmarks": [ "internet-driven dialogue deployment data" ], "models_studied": [ "BlenderBot-style models" ], "production_evidence": true, "summary": "Collects deployment interactions with several kinds of human feedback and compares learning algorithms (supervised, rejection sampling, Director, reward-based) for improving dialogue models; releases the data.", "key_idea": "Which feedback types and algorithms work for post-deployment improvement.", "relevance_score": 2, "relevance_rationale": "Background on learning from production feedback.", "found_via": "OpenAlex title search", "cites_or_cited_by": [] }, { "id": "zhou-2022-ape", "title": "Large Language Models Are Human-Level Prompt Engineers", "authors": [ "Yongchao Zhou", "Andrei Ioan Muresanu", "Ziwen Han", "Keiran Paster", "Silviu Pitis", "Harris Chan", "Jimmy Ba" ], "year": 2022, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2211.01910", "code_url": null, "topic_clusters": [ "C4" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [ "score of candidate instructions" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Automatic Prompt Engineer generates candidate instructions with an LLM from demonstrations, scores them, and iteratively refines the best, reaching human-level prompt engineering on many tasks.", "key_idea": "LLM-generated and scored instruction search.", "relevance_score": 2, "relevance_rationale": "Foundational prompt optimization technique.", "found_via": "search: Promptbreeder ProTeGi Automatic Prompt Engineer arXiv", "cites_or_cited_by": [] }, { "id": "cobbe-2021-gsm8k-verifiers", "title": "Training Verifiers to Solve Math Word Problems", "authors": [ "Karl Cobbe", "Vineet Kosaraju", "Mohammad Bavarian", "Mark Chen", "Heewoo Jun", "Lukasz Kaiser", "Matthias Plappert", "Jerry Tworek", "Jacob Hilton", "Reiichiro Nakano", "Christopher Hesse", "John Schulman" ], "year": 2021, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2110.14168", "code_url": null, "topic_clusters": [ "C7" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "verifier score" ], "cost_reduction_reported": null, "benchmarks": [ "GSM8K" ], "models_studied": [ "GPT-3" ], "production_evidence": false, "summary": "Introduces GSM8K and trains verifiers that score candidate solutions; at test time many samples are generated and the top-ranked one is chosen. Verification improves accuracy and scales better with data than fine-tuning alone.", "key_idea": "Train an outcome verifier and rerank samples.", "relevance_score": 2, "relevance_rationale": "Foundational outcome reward model / verifier.", "found_via": "keyword: verifier", "cites_or_cited_by": [] }, { "id": "gibbs-2021-adaptive-conformal", "title": "Adaptive Conformal Inference Under Distribution Shift", "authors": [ "Isaac Gibbs", "Emmanuel J. Candès" ], "year": 2021, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2106.00170", "code_url": null, "topic_clusters": [ "C5", "C7" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "prediction residuals" ], "cost_reduction_reported": null, "benchmarks": [ "two real-world datasets" ], "models_studied": [], "production_evidence": false, "summary": "Adaptive conformal inference keeps long-run coverage of prediction sets online under arbitrary distribution shift by updating one miscoverage parameter over time.", "key_idea": "Online conformal calibration under drift.", "relevance_score": 2, "relevance_rationale": "Tool for keeping router/cascade thresholds calibrated under drift.", "found_via": "OpenAlex title search", "cites_or_cited_by": [] }, { "id": "kreutzer-2020-offline-rl-seq2seq", "title": "Offline Reinforcement Learning from Human Feedback in Real-World Sequence-to-Sequence Tasks", "authors": [ "Julia Kreutzer", "Stefan Riezler", "Carolin Lawrence" ], "year": 2020, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/2011.02511", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "RL-offline", "signals_used": [ "interaction logs", "human feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": true, "summary": "Position paper on offline RL from interaction logs of deployed NLP systems, covering practical challenges (e.g. deterministic logging, reliability of feedback) and possible solutions.", "key_idea": "Offline RL from deployed NLP system logs: challenges.", "relevance_score": 2, "relevance_rationale": "Background on learning from production logs.", "found_via": "OpenAlex title search", "cites_or_cited_by": [] }, { "id": "mozannar-2020-consistent-l2d", "title": "Consistent Estimators for Learning to Defer to an Expert", "authors": [ "Hussein Mozannar", "David Sontag" ], "year": 2020, "venue_or_source": "ICML 2020", "source_type": "paper", "url": "https://arxiv.org/abs/2006.01862", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "classifier and rejector outputs" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Learns a classifier and a rejector jointly from samples of expert decisions via a reduction to cost-sensitive learning, with a consistent surrogate loss that generalises cross-entropy.", "key_idea": "First consistent surrogate loss for learning to defer.", "relevance_score": 2, "relevance_rationale": "Foundational L2D theory underlying deferral rules.", "found_via": "web search: Mozannar Sontag consistent estimators", "cites_or_cited_by": [] }, { "id": "wang-2020-wisdom-of-committees", "title": "Wisdom of Committees: An Overlooked Approach To Faster and More Accurate Models", "authors": [ "Xiaofang Wang", "Dan Kondratyuk", "Eric Christiansen", "Kris M. Kitani", "Yair Alon", "Elad Eban" ], "year": 2020, "venue_or_source": "ICLR 2022", "source_type": "paper", "url": "https://arxiv.org/abs/2012.01988", "code_url": null, "topic_clusters": [ "C2" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "confidence thresholds" ], "cost_reduction_reported": "EfficientNet cascade 5.4x speedup over B7; ViT cascade 2.3x speedup over ViT-L-384 at equal accuracy", "benchmarks": [ "image classification", "video classification", "semantic segmentation" ], "models_studied": [ "EfficientNet", "ViT", "ResNet", "MobileNetV2", "X3D" ], "production_evidence": false, "summary": "Shows simple ensembles and cascades of pre-trained models can match state-of-the-art accuracy far more efficiently, even beating NAS methods.", "key_idea": "Committees of existing models are an overlooked efficiency baseline.", "relevance_score": 2, "relevance_rationale": "Foundational cascade result from vision.", "found_via": "snowball: related work of cascade papers (verified on arXiv)", "cites_or_cited_by": [] }, { "id": "zhang-2020-neural-thompson", "title": "Neural Thompson Sampling", "authors": [ "Weitong Zhang", "Dongruo Zhou", "Lihong C. Li", "Quanquan Gu" ], "year": 2020, "venue_or_source": "ICLR 2021", "source_type": "paper", "url": "https://arxiv.org/abs/2010.00827", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Neural Thompson Sampling uses a neural-network posterior mean with neural-tangent-feature variance for contextual bandits.", "key_idea": "Neural Thompson sampling.", "relevance_score": 2, "relevance_rationale": "Background algorithm.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "zhou-2019-neuralucb" ] }, { "id": "zhou-2019-neuralucb", "title": "Neural Contextual Bandits with UCB-based Exploration", "authors": [ "Dongruo Zhou", "Lihong C. Li", "Quanquan Gu" ], "year": 2019, "venue_or_source": "ICML 2020", "source_type": "paper", "url": "https://arxiv.org/abs/1911.04462", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "NeuralUCB uses neural-network features to build upper confidence bounds with near-optimal regret for contextual bandits.", "key_idea": "Neural contextual bandits with UCB.", "relevance_score": 2, "relevance_rationale": "Algorithm used by neural bandit routers.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "tsai-2026-neuralucb-routing", "zhang-2020-neural-thompson" ] }, { "id": "gilotte-2018-offline-ab", "title": "Offline A/B Testing for Recommender Systems", "authors": [ "Alexandre Gilotte", "Clément Calauzènes", "Thomas Nedelec", "Alexandre Abraham", "Simon Dollé" ], "year": 2018, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/1801.07030", "code_url": null, "topic_clusters": [ "C5", "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "logged recommendations", "business metrics" ], "cost_reduction_reported": null, "benchmarks": [ "large-scale commercial recommender" ], "models_studied": [], "production_evidence": true, "summary": "Proposes a counterfactual estimator with a modeled bias for offline A/B testing of recommender systems and benchmarks estimators against real online A/B results.", "key_idea": "Offline A/B testing validated against online A/B tests.", "relevance_score": 2, "relevance_rationale": "Shows how OPE correlates with production A/B outcomes.", "found_via": "OpenAlex title search", "cites_or_cited_by": [ "dudik-2011-doubly-robust" ] }, { "id": "joachims-2018-banditnet", "title": "Deep Learning with Logged Bandit Feedback", "authors": [ "Thorsten Joachims", "Adith Swaminathan", "Maarten de Rijke" ], "year": 2018, "venue_or_source": "ICLR 2018", "source_type": "paper", "url": "https://openalex.org/W2785875001", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "logged bandit feedback" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "BanditNet trains deep networks from logged contextual bandit feedback using a CRM objective with an equivariant, self-normalized estimator.", "key_idea": "Deep learning from logged bandit feedback.", "relevance_score": 2, "relevance_rationale": "Template for training neural routers on logged traffic.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "swaminathan-2015-crm", "swaminathan-2015-snips" ] }, { "id": "madras-2017-predict-responsibly", "title": "Predict Responsibly: Improving Fairness and Accuracy by Learning to Defer", "authors": [ "David Madras", "Toniann Pitassi", "Richard Zemel" ], "year": 2017, "venue_or_source": "NeurIPS 2018", "source_type": "paper", "url": "https://arxiv.org/abs/1711.06664", "code_url": "https://github.com/dmadras/predict-responsibly", "topic_clusters": [ "C2" ], "routing_granularity": "n/a", "learning_regime": "supervised-offline", "signals_used": [ "model confidence", "expert decision behaviour" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Introduces learning to defer, generalising rejection learning to account for a downstream decision-maker. Shows deferring models can improve accuracy and fairness of the combined system even with biased experts.", "key_idea": "Original learning-to-defer formulation.", "relevance_score": 2, "relevance_rationale": "Foundational L2D work used by LLM cascade papers.", "found_via": "web search: Madras 2018 Predict Responsibly", "cites_or_cited_by": [] }, { "id": "russo-2017-thompson-tutorial", "title": "A Tutorial on Thompson Sampling", "authors": [ "Daniel Joseph Russo", "Benjamin Van Roy", "Abbas Kazerouni", "Ian Osband", "Zheng Wen" ], "year": 2017, "venue_or_source": "Foundations and Trends in ML 2018", "source_type": "paper", "url": "https://arxiv.org/abs/1707.02038", "code_url": null, "topic_clusters": [ "C5", "C15" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Tutorial on Thompson sampling covering Bernoulli bandits, recommendation, assortment and RL, with guidance on when it works.", "key_idea": "Thompson sampling tutorial.", "relevance_score": 2, "relevance_rationale": "Background for TS-based routers.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "agrawal-2011-thompson-analysis" ] }, { "id": "agrawal-2015-linear-cbwk", "title": "Linear Contextual Bandits with Knapsacks", "authors": [ "Shipra Agrawal", "Nikhil R. Devanur" ], "year": 2015, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/1507.06738", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "linear context" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Linear contextual bandits with knapsack constraints, generalizing LinUCB and BwK, with near-optimal regret.", "key_idea": "Linear contextual BwK.", "relevance_score": 2, "relevance_rationale": "Direct template for budgeted LinUCB routing.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "badanidiyuru-2013-bwk", "li-2010-linucb" ] }, { "id": "swaminathan-2015-crm", "title": "Counterfactual Risk Minimization: Learning from Logged Bandit Feedback", "authors": [ "Adith Swaminathan", "Thorsten Joachims" ], "year": 2015, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/1502.02362", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "propensity scores" ], "cost_reduction_reported": null, "benchmarks": [ "multi-label classification" ], "models_studied": [], "production_evidence": false, "summary": "Introduces counterfactual risk minimization for batch learning from logged bandit feedback, with variance-aware generalization bounds and the POEM algorithm.", "key_idea": "Counterfactual risk minimization (POEM).", "relevance_score": 2, "relevance_rationale": "Foundation for learning a router from logged partial feedback.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "dudik-2011-doubly-robust", "joachims-2018-banditnet", "kiyohara-2025-prompt-opt-logged-bandit", "swaminathan-2015-snips" ] }, { "id": "swaminathan-2015-snips", "title": "The self-normalized estimator for counterfactual learning", "authors": [ "Adith Swaminathan", "Thorsten Joachims" ], "year": 2015, "venue_or_source": "Conference paper (2015; venue per OpenAlex not listed)", "source_type": "paper", "url": "https://openalex.org/W2188353343", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "propensity scores" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Identifies a propensity-overfitting problem in the standard counterfactual risk estimator and proposes the self-normalized estimator (SNIPS) to avoid it in batch learning from logged bandit feedback.", "key_idea": "Self-normalized IPS estimator.", "relevance_score": 2, "relevance_rationale": "Standard low-variance OPE estimator.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "joachims-2018-banditnet", "saito-2020-open-bandit-pipeline", "swaminathan-2015-crm" ] }, { "id": "badanidiyuru-2014-resourceful-cb", "title": "Resourceful Contextual Bandits", "authors": [ "Ashwinkumar Badanidiyuru", "John C. Langford", "Aleksandrs Slivkins" ], "year": 2014, "venue_or_source": "COLT 2014", "source_type": "paper", "url": "https://arxiv.org/abs/1402.6779", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "context", "resource consumption" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Extends bandits with knapsacks to contextual bandits over general policy classes with near-optimal regret.", "key_idea": "Contextual bandits with resource constraints.", "relevance_score": 2, "relevance_rationale": "Background for budgeted contextual routing.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "badanidiyuru-2013-bwk" ] }, { "id": "badanidiyuru-2013-bwk", "title": "Bandits with Knapsacks", "authors": [ "Ashwinkumar Badanidiyuru", "Robert Kleinberg", "Aleksandrs Slivkins" ], "year": 2013, "venue_or_source": "FOCS 2013", "source_type": "paper", "url": "https://arxiv.org/abs/1305.2545", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "resource consumption" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Defines bandits with knapsacks, where each pull consumes budgeted resources, and gives near-optimal algorithms (balanced exploration and primal-dual).", "key_idea": "Bandits under budget constraints.", "relevance_score": 2, "relevance_rationale": "Foundation for budget-constrained routing.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "agrawal-2015-linear-cbwk", "badanidiyuru-2014-resourceful-cb", "liu-2022-nonstationary-bwk", "panda-2025-pilot" ] }, { "id": "kohavi-2013-online-experiments", "title": "Online controlled experiments at large scale", "authors": [ "Ron Kohavi", "Alex Deng", "Brian Frasca", "Toby Walker", "Ya Xu", "Nils Pohlmann" ], "year": 2013, "venue_or_source": "KDD 2013", "source_type": "paper", "url": "https://doi.org/10.1145/2487575.2488217", "code_url": null, "topic_clusters": [ "C5", "C14" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "online metrics" ], "cost_reduction_reported": null, "benchmarks": [ "Bing" ], "models_studied": [], "production_evidence": true, "summary": "Describes running hundreds of concurrent online controlled experiments at Bing and the cultural, engineering and trustworthiness challenges involved.", "key_idea": "A/B testing practice at scale.", "relevance_score": 2, "relevance_rationale": "Reference practice for A/B testing LLM routing changes.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [] }, { "id": "bottou-2012-counterfactual-reasoning", "title": "Counterfactual Reasoning and Learning Systems", "authors": [ "Léon Bottou", "Jonas Peters", "Joaquin Quiñonero-Candela", "Denis Xavier Charles", "David M. Chickering", "Elon Portugaly", "Dipankar Ray", "Patrice Simard", "Ed Snelson" ], "year": 2012, "venue_or_source": "JMLR", "source_type": "paper", "url": "https://arxiv.org/abs/1209.2355", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "causal inference on logs" ], "cost_reduction_reported": null, "benchmarks": [ "Bing ad placement" ], "models_studied": [], "production_evidence": true, "summary": "Uses causal inference on logged data to predict how changes to a complex learning system (Bing ad placement) would affect performance, before deploying them.", "key_idea": "Counterfactual reasoning on production logs.", "relevance_score": 2, "relevance_rationale": "Classic production example of counterfactual system tuning.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "dudik-2011-doubly-robust" ] }, { "id": "chapelle-2012-interleaving", "title": "Large-scale validation and analysis of interleaved search evaluation", "authors": [ "Olivier Chapelle", "Thorsten Joachims", "Filip Radlinski", "Yisong Yue" ], "year": 2012, "venue_or_source": "ACM Transactions on Information Systems", "source_type": "paper", "url": "https://doi.org/10.1145/2094072.2094078", "code_url": null, "topic_clusters": [ "C5", "C13" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "implicit click feedback" ], "cost_reduction_reported": null, "benchmarks": [ "two commercial search engines", "scientific literature search" ], "models_studied": [], "production_evidence": true, "summary": "Comprehensive empirical analysis of interleaving, which compares two rankers by mixing their results and counting user clicks, using data from two commercial search engines; shows it is efficient and agrees with other evaluations.", "key_idea": "Interleaving as sensitive online comparison from implicit feedback.", "relevance_score": 2, "relevance_rationale": "Background for online comparison of LLM configurations with implicit feedback.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [] }, { "id": "agrawal-2011-thompson-analysis", "title": "Analysis of Thompson Sampling for the multi-armed bandit problem", "authors": [ "Shipra Agrawal", "Navin Goyal" ], "year": 2011, "venue_or_source": "arXiv", "source_type": "preprint", "url": "https://arxiv.org/abs/1111.1797", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "First logarithmic regret analysis of Thompson Sampling for the multi-armed bandit.", "key_idea": "Regret guarantees for Thompson sampling.", "relevance_score": 2, "relevance_rationale": "Foundational TS reference.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "russo-2017-thompson-tutorial" ] }, { "id": "dudik-2011-doubly-robust", "title": "Doubly Robust Policy Evaluation and Learning", "authors": [ "Miroslav Dudı́k", "John C. Langford", "Lihong Li" ], "year": 2011, "venue_or_source": "ICML 2011", "source_type": "paper", "url": "https://arxiv.org/abs/1103.4601", "code_url": null, "topic_clusters": [ "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "reward model", "propensity model" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Applies doubly robust estimation to contextual-bandit policy evaluation and optimization, giving accurate estimates when either the reward model or the logging-policy model is good.", "key_idea": "Doubly robust off-policy evaluation.", "relevance_score": 2, "relevance_rationale": "Core estimator for counterfactual evaluation of routers from logs.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "bhargava-2024-ope-logged-human-feedback", "bottou-2012-counterfactual-reasoning", "gilotte-2018-offline-ab", "kiyohara-2022-cascade-dr", "saito-2020-open-bandit-pipeline", "saito-2022-ope-large-action-embeddings", "swaminathan-2015-crm", "xiang-2026-otrope" ] }, { "id": "li-2010-linucb", "title": "A contextual-bandit approach to personalized news article recommendation", "authors": [ "Lihong Li", "Wei Chu", "John C. Langford", "Robert E. Schapire" ], "year": 2010, "venue_or_source": "WWW 2010", "source_type": "paper", "url": "https://arxiv.org/abs/1003.0146", "code_url": null, "topic_clusters": [ "C5" ], "routing_granularity": "n/a", "learning_regime": "bandit-online", "signals_used": [ "user and item features", "click feedback" ], "cost_reduction_reported": null, "benchmarks": [ "Yahoo! Front Page Today Module" ], "models_studied": [], "production_evidence": true, "summary": "Introduces LinUCB for personalized news recommendation on Yahoo! traffic, a contextual bandit with linear payoff and upper-confidence exploration.", "key_idea": "LinUCB contextual bandit.", "relevance_score": 2, "relevance_rationale": "Foundational algorithm used by PILOT, OrcaRouter and others.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "agrawal-2015-linear-cbwk", "bao-2026-orcarouter", "elumar-2025-camvo", "li-2010-unbiased-offline-eval", "panda-2025-pilot", "poon-2025-online-multi-llm-context-evolution", "zhang-2026-cclub-prompt-routing" ] }, { "id": "li-2010-unbiased-offline-eval", "title": "Unbiased Offline Evaluation of Contextual-bandit-based News Article Recommendation Algorithms", "authors": [ "Lihong Li", "Wei Chu", "John C. Langford", "Xuanhui Wang" ], "year": 2010, "venue_or_source": "WSDM 2011", "source_type": "paper", "url": "https://arxiv.org/abs/1003.5956", "code_url": null, "topic_clusters": [ "C5", "C6" ], "routing_granularity": "n/a", "learning_regime": "n/a", "signals_used": [ "randomized logging" ], "cost_reduction_reported": null, "benchmarks": [ "Yahoo! Front Page news data" ], "models_studied": [], "production_evidence": true, "summary": "Introduces a replay method for provably unbiased offline evaluation of contextual-bandit algorithms from randomly logged traffic, validated against online bucket tests at Yahoo!.", "key_idea": "Replay evaluation of bandit policies from logs.", "relevance_score": 2, "relevance_rationale": "Simple method to evaluate online routers offline if some traffic is randomized.", "found_via": "OpenAlex title search (foundational)", "cites_or_cited_by": [ "li-2010-linucb" ] }, { "id": "anthropic-choosing-a-model", "title": "Choosing the right model", "authors": [ "Anthropic" ], "year": null, "venue_or_source": "Claude Platform docs", "source_type": "docs", "url": "https://docs.anthropic.com/en/docs/about-claude/models/choosing-a-model", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude models" ], "production_evidence": false, "summary": "Provider guidance on balancing capability, speed and cost when picking a Claude model, including starting points and how to test the choice.", "key_idea": "Model choice guidance.", "relevance_score": 2, "relevance_rationale": "Background provider guidance.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [] }, { "id": "artificial-analysis", "title": "Artificial Analysis: AI Model and API Providers Analysis", "authors": [ "Artificial Analysis" ], "year": null, "venue_or_source": "Artificial Analysis website", "source_type": "benchmark", "url": "https://artificialanalysis.ai", "code_url": null, "topic_clusters": [ "C13", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "quality", "price", "speed", "latency" ], "cost_reduction_reported": null, "benchmarks": [ "independent benchmarks" ], "models_studied": [ "commercial and open models" ], "production_evidence": false, "summary": "Independent benchmarking site comparing model quality, price, output speed and latency across API providers.", "key_idea": "Price-quality-speed comparisons across providers.", "relevance_score": 2, "relevance_rationale": "Useful for building cost-quality frontiers.", "found_via": "seed list: efficiency leaderboards", "cites_or_cited_by": [] }, { "id": "aws-bedrock-prompt-caching", "title": "Prompt caching for faster model inference (Amazon Bedrock)", "authors": [ "Amazon Web Services" ], "year": null, "venue_or_source": "Amazon Bedrock User Guide", "source_type": "docs", "url": "https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Claude and Nova models on Bedrock" ], "production_evidence": false, "summary": "Bedrock documentation for prompt caching to cut latency and inference cost for repeated prefixes.", "key_idea": "Cloud-platform prompt caching.", "relevance_score": 2, "relevance_rationale": "Provider caching lever.", "found_via": "snowball: related work of aws-2025-bedrock-intelligent-prompt-routing", "cites_or_cited_by": [] }, { "id": "cloudflare-ai-gateway", "title": "Cloudflare AI Gateway", "authors": [ "Cloudflare" ], "year": null, "venue_or_source": "Cloudflare docs", "source_type": "docs", "url": "https://developers.cloudflare.com/ai-gateway/", "code_url": null, "topic_clusters": [ "C10", "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Managed gateway providing analytics, response caching, rate limiting and model fallback for AI applications.", "key_idea": "Edge gateway with caching and fallback.", "relevance_score": 2, "relevance_rationale": "Gateway plumbing for cost control.", "found_via": "seed list: AI gateway", "cites_or_cited_by": [] }, { "id": "databricks-ai-gateway", "title": "AI governance with Unity Gateway (Mosaic AI Gateway)", "authors": [ "Databricks" ], "year": null, "venue_or_source": "Databricks docs", "source_type": "docs", "url": "https://docs.databricks.com/aws/en/ai-gateway/", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Databricks gateway documentation for governing model endpoints with usage tracking, rate limits, guardrails and traffic management.", "key_idea": "Platform-level governance gateway.", "relevance_score": 2, "relevance_rationale": "Gateway with usage logging for cost attribution.", "found_via": "seed list: Databricks blogs", "cites_or_cited_by": [] }, { "id": "google-gemini-batch-api", "title": "Batch API (Gemini API)", "authors": [ "Google" ], "year": null, "venue_or_source": "Google AI for Developers docs", "source_type": "docs", "url": "https://ai.google.dev/gemini-api/docs/batch-mode", "code_url": null, "topic_clusters": [ "C11", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": "(vendor-reported) 50% of standard cost, 24-hour target turnaround", "benchmarks": [], "models_studied": [ "Gemini models" ], "production_evidence": false, "summary": "Gemini batch mode for large non-urgent jobs at half price with a 24-hour turnaround target.", "key_idea": "Async batch at 50% price.", "relevance_score": 2, "relevance_rationale": "Seed; batch API cost lever.", "found_via": "seed: provider batch APIs", "cites_or_cited_by": [] }, { "id": "helicone-observability", "title": "Helicone: open-source LLM observability platform", "authors": [ "Helicone" ], "year": null, "venue_or_source": "GitHub", "source_type": "repo", "url": "https://github.com/Helicone/helicone", "code_url": null, "topic_clusters": [ "C6", "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "request logs", "cost" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Open-source proxy-based platform for logging, monitoring, evaluating and experimenting on LLM calls, including cost tracking.", "key_idea": "Logging layer that yields data for offline routing analysis.", "relevance_score": 2, "relevance_rationale": "Source of production logs for offline optimization.", "found_via": "seed list: LLMOps cost", "cites_or_cited_by": [] }, { "id": "huggingface-llm-perf-leaderboard", "title": "LLM-Perf Leaderboard", "authors": [ "Hugging Face (optimum)" ], "year": null, "venue_or_source": "Hugging Face Spaces", "source_type": "benchmark", "url": "https://huggingface.co/spaces/optimum/llm-perf-leaderboard", "code_url": "https://github.com/huggingface/optimum-benchmark", "topic_clusters": [ "C11", "C13" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [ "latency", "throughput", "memory", "energy" ], "cost_reduction_reported": null, "benchmarks": [ "single-GPU generation benchmark" ], "models_studied": [ "open LLMs" ], "production_evidence": false, "summary": "Leaderboard of LLM latency, throughput, memory and energy across hardware, backends and quantization, built on optimum-benchmark.", "key_idea": "Efficiency leaderboard for open models.", "relevance_score": 2, "relevance_rationale": "Efficiency benchmark background.", "found_via": "seed list: LLM-Perf/efficiency leaderboards", "cites_or_cited_by": [] }, { "id": "katanemo-plano-archgw", "title": "Plano (formerly archgw): AI-native proxy and data plane for agentic apps", "authors": [ "Katanemo" ], "year": null, "venue_or_source": "GitHub", "source_type": "repo", "url": "https://github.com/katanemo/archgw", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "supervised-offline", "signals_used": [ "domain-action preference" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "Arch-Router-1.5B" ], "production_evidence": false, "summary": "Open-source proxy for agent apps providing LLM routing (via Arch-Router), observability, agent orchestration and guardrails.", "key_idea": "Gateway with preference-aligned router model.", "relevance_score": 2, "relevance_rationale": "OSS gateway embedding a router model.", "found_via": "snowball: related work of tran-2025-arch-router", "cites_or_cited_by": [ "tran-2025-arch-router" ] }, { "id": "kong-ai-gateway", "title": "Kong AI Gateway", "authors": [ "Kong Inc." ], "year": null, "venue_or_source": "Kong product page", "source_type": "product", "url": "https://konghq.com/products/kong-ai-gateway", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "API-gateway extension for LLM and MCP traffic offering centralized security, routing, observability and cost control.", "key_idea": "Enterprise API gateway for LLM traffic.", "relevance_score": 2, "relevance_rationale": "Gateway plumbing.", "found_via": "seed list: AI gateway", "cites_or_cited_by": [] }, { "id": "litellm-router", "title": "LiteLLM Router and AI Gateway", "authors": [ "BerriAI" ], "year": null, "venue_or_source": "LiteLLM docs / GitHub", "source_type": "repo", "url": "https://docs.litellm.ai/docs/routing", "code_url": "https://github.com/BerriAI/litellm", "topic_clusters": [ "C10", "C14" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "rpm/tpm", "latency", "usage", "cost", "in-flight calls" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "100+ LLM APIs" ], "production_evidence": false, "summary": "Widely used open-source SDK and proxy that unifies LLM APIs with cost tracking, and a router offering simple-shuffle, latency-, usage-, least-busy- and cost-based strategies, cooldowns, retries, fallbacks, response caching and session affinity for provider prompt caching.", "key_idea": "Load-balancing router with cost-based strategy.", "relevance_score": 2, "relevance_rationale": "Seed; standard OSS gateway.", "found_via": "seed: LiteLLM", "cites_or_cited_by": [] }, { "id": "martian-docs", "title": "Martian Documentation (Gateway API)", "authors": [ "Martian" ], "year": null, "venue_or_source": "Martian docs", "source_type": "docs", "url": "https://docs.withmartian.com", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "200+ models" ], "production_evidence": false, "summary": "Martian, the company behind RouterBench and an early commercial model router, now documents a gateway API for 200+ models alongside research tools (ARES, K-Steering); its homepage emphasizes interpretability research rather than routing, and no current router-specific page was found.", "key_idea": "Early router vendor, now gateway plus research.", "relevance_score": 2, "relevance_rationale": "Seed; RouterBench authors; limited current router detail.", "found_via": "seed: Martian", "cites_or_cited_by": [ "hu-2024-routerbench" ] }, { "id": "nvidia-llm-router-blueprint", "title": "NVIDIA AI Blueprint: LLM Router", "authors": [ "NVIDIA" ], "year": null, "venue_or_source": "GitHub (NVIDIA-AI-Blueprints)", "source_type": "repo", "url": "https://github.com/NVIDIA-AI-Blueprints/llm-router", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "hybrid", "signals_used": [ "intent classification by small LLM", "CLIP embeddings + trained NN" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "GPT-5-Chat", "Nemotron VLM 12B", "Nemotron 9B", "Qwen 1.7B (router)" ], "production_evidence": false, "summary": "Reference router offering training-free intent routing with a small LLM and a learned auto-router using CLIP embeddings and a neural network, returning model recommendations through an OpenAI-compatible endpoint.", "key_idea": "Reference architecture for intent and learned routing.", "relevance_score": 2, "relevance_rationale": "Vendor reference implementation.", "found_via": "seed list: model router product", "cites_or_cited_by": [] }, { "id": "openai-cost-optimization", "title": "Cost optimization", "authors": [ "OpenAI" ], "year": null, "venue_or_source": "OpenAI API docs", "source_type": "docs", "url": "https://platform.openai.com/docs/guides/cost-optimization", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "OpenAI models" ], "production_evidence": false, "summary": "Provider guide listing levers: fewer requests, fewer input/output tokens, smaller models where accuracy allows, Batch API and flex processing. No quantitative claims.", "key_idea": "Provider checklist of cost levers.", "relevance_score": 2, "relevance_rationale": "Seed; baseline provider guidance.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [ "openai-batch-api", "openai-flex-processing" ] }, { "id": "openai-model-selection", "title": "Model selection", "authors": [ "OpenAI" ], "year": null, "venue_or_source": "OpenAI API docs", "source_type": "docs", "url": "https://platform.openai.com/docs/guides/model-selection", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "OpenAI models" ], "production_evidence": false, "summary": "Provider guidance on choosing models by balancing accuracy, latency and cost, starting from accuracy targets and then optimizing.", "key_idea": "Accuracy-first then optimize cost.", "relevance_score": 2, "relevance_rationale": "Background provider guidance.", "found_via": "seed: provider cost-optimization guides", "cites_or_cited_by": [] }, { "id": "portkey-ai-gateway", "title": "Portkey AI Gateway", "authors": [ "Portkey" ], "year": null, "venue_or_source": "Portkey docs / GitHub", "source_type": "product", "url": "https://portkey.ai/docs/product/ai-gateway", "code_url": "https://github.com/Portkey-AI/gateway", "topic_clusters": [ "C10", "C14" ], "routing_granularity": "query", "learning_regime": "training-free", "signals_used": [ "custom conditions", "metadata" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "1,600+ LLMs (per repo)" ], "production_evidence": false, "summary": "Open-source and hosted gateway with conditional routing, fallbacks, load balancing, simple and semantic caching, budget and rate limits, retries and canary testing. No quantitative benchmarks provided.", "key_idea": "Rule-based routing plus semantic caching and budgets.", "relevance_score": 2, "relevance_rationale": "Seed; common gateway plumbing for cost control.", "found_via": "seed: Portkey", "cites_or_cited_by": [] }, { "id": "requesty-router", "title": "Requesty: AI Gateway and LLM Router", "authors": [ "Requesty" ], "year": null, "venue_or_source": "Requesty website", "source_type": "product", "url": "https://www.requesty.ai", "code_url": null, "topic_clusters": [ "C1", "C14" ], "routing_granularity": "query", "learning_regime": "n/a", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [ "600+ models" ], "production_evidence": false, "summary": "Commercial OpenAI-compatible gateway with smart routing, caching, failover, observability and EU data residency.", "key_idea": "Commercial router-gateway.", "relevance_score": 2, "relevance_rationale": "Gateway product landscape.", "found_via": "seed list: LLM gateway", "cites_or_cited_by": [] }, { "id": "unify-product", "title": "unify", "authors": [ "Unify" ], "year": null, "venue_or_source": "Unify website", "source_type": "product", "url": "https://unify.ai", "code_url": null, "topic_clusters": [ "C8", "C14" ], "routing_granularity": "n/a", "learning_regime": "RL-online", "signals_used": [ "production corrections", "retries", "outcomes" ], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Unify, previously a model-routing platform (compared against RouteLLM in the LMSYS blog), now presents itself as post-training an agent's model on its own production work, gated by evals. Routing-era details were not found on the current site.", "key_idea": "Pivot from routing to continual post-training on production data.", "relevance_score": 2, "relevance_rationale": "Seed; illustrates vendor pivot from routing to specialization.", "found_via": "seed: Unify", "cites_or_cited_by": [ "ong-2024-routellm" ] }, { "id": "vercel-ai-gateway", "title": "Vercel AI Gateway", "authors": [ "Vercel" ], "year": null, "venue_or_source": "Vercel docs", "source_type": "docs", "url": "https://vercel.com/docs/ai-gateway", "code_url": null, "topic_clusters": [ "C14" ], "routing_granularity": "none", "learning_regime": "training-free", "signals_used": [], "cost_reduction_reported": null, "benchmarks": [], "models_studied": [], "production_evidence": false, "summary": "Managed gateway offering access to many models with routing, fallbacks and observability.", "key_idea": "Managed multi-model gateway.", "relevance_score": 2, "relevance_rationale": "Gateway plumbing.", "found_via": "seed list: AI gateway", "cites_or_cited_by": [] } ]