{ "meta": { "schema": "jev-survey/papers@1", "cutoff": "2026-09-24", "timezone": "Asia/Singapore", "note": "Counts are derived by scripts/build.py. Venues are stated only as given in arXiv metadata (journal_ref or author comment); proceedings were not checked (venue_verified_at is null)." }, "papers": [ { "id": "arxiv:2609.25845", "arxiv_id": "2609.25845", "versioned_id": "2609.25845v1", "doi": null, "title": "Visual Jev: Accurate and Efficient Decisions from Shared Visual Context", "short_title": "Visual Jev", "authors": [ "Guanxu Yu", "Yuhang Yao" ], "published_at": "2026-09-22T08:12:29Z", "updated_at": "2026-09-22T08:12:29Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CV", "cs.LG" ], "abstract": "Many vision applications ask several independent, forced-choice questions about the same image. Visual Jev encodes the image and public context once, executes isolated question suffixes as a batch, and reads candidate probabilities from the backbone's language-model head. Across four benchmarks, answer-supervised post-training raises equal-weight macro accuracy from 70.6% to 76.1%, with the gain concentrated on the two task families represented in training. At N=32 questions per image, shared batched execution is 8.9x faster in warm amortized time than independent serial execution and remains 3.4x faster than an already-batched baseline that recomputes the prefix, at the cost of higher peak memory. A matched typed-head control offers no consistent accuracy advantage over the language-model-head readout. The supported design is therefore simple: adapt the backbone for quality, retain the existing readout, and share execution for efficiency.", "url": "https://arxiv.org/abs/2609.25845v1", "pdf_url": "https://arxiv.org/pdf/2609.25845v1", "bibtex_key": "arxiv2609_25845", "included_by": [ "Q01_jev" ], "inclusion_reason": "独立 Visual Jev；并非 TypeSafe 视觉版", "relationship": "An independent vision–language decision model. It is not a TypeSafe product and is not evidence that hosted Jev accepts images.", "summary": "Encodes an image and public context once, runs isolated question suffixes as a batch, and reads candidate probabilities from the backbone’s LM head. The design separates four effects: model quality, readout, prefix sharing and batching.", "task": "Forced-choice visual questions on GQA, SNLI-VE, TextVQA and TallyQA with Qwen3-VL-4B (8B extension) on one RTX 5090.", "datasets": [ "GQA", "SNLI-VE", "TextVQA", "TallyQA" ], "baselines": [ "untuned backbone read through its LM head", "matched typed-head control", "independent serial execution", "batched execution without prefix reuse" ], "models": [ "Visual Jev (Qwen3-VL-4B post-trained)" ], "main_finding": "Post-training raised equal-weight macro accuracy from 0.706 to 0.761; with 32 questions per image, shared batched execution was 8.9× faster (warm, amortized) than independent serial calls and 3.4× faster than batching that recomputes the prefix.", "caveats": [ "5.7 ms is amortized: a 32-question batch completes in about 182 ms. It is not the latency of an independent request.", "Accuracy gains concentrate on the two task families represented in training.", "A matched typed head showed no consistent accuracy advantage over the LM-head readout.", "Single-machine study on a consumer GPU; queueing latency is excluded." ], "survey_use": "Attribute speed to prefix sharing and batching and quality to adaptation — not to a special head. Classify separately from text-feature approaches such as JEVQA.", "topics": [ "multimodal", "efficiency", "open-models" ], "stages": [ "readout", "use" ], "method_families": [ "finetuned_decoder" ], "applications": [ "visual-qa" ], "model_relationship": [ "independent_jev_like" ], "test_level": "model_test", "model_versions": [ { "model": "Visual Jev (Qwen3-VL-4B based)", "version": "paper v1" } ], "study_family_id": null, "openness": { "code": { "status": "project_page_only", "note": "The repository carries the project page and result JSON; its README states that the full evaluation tree is not in the repository.", "url": "https://github.com/guanxuyu-sv/Visual-Jev/blob/2c3451b20d282545aa7372aa78b06fc795804738/README.md" }, "weights": { "status": "not_located" }, "data": { "status": "available", "note": "Public benchmarks." }, "predictions": { "status": "partial", "note": "Result files behind the paper tables are published on the project page." }, "recomputable": { "status": "partial" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§4–5; Tables 2–3; Limitations; linked repository README", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-25845-quality", "c-25845-speed", "c-25845-typed-head" ] }, { "id": "arxiv:2609.26532", "arxiv_id": "2609.26532", "versioned_id": "2609.26532v1", "doi": null, "title": "REFLEX with Jev for Efficient Selective Control in LLM Agents", "short_title": "REFLEX with Jev", "authors": [ "Tiantong Wu", "Wei Yang Bryan Lim" ], "published_at": "2026-09-22T14:54:02Z", "updated_at": "2026-09-22T14:54:02Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "LLM agents often use generative models for bounded decisions, raising the question of when these decisions can be handled more efficiently without reducing task success. We study REFLEX, an agent architecture that uses Jev as a fast, typed decision layer and calls a strong LLM when confidence is low, or generation is required. On a frozen 100-task benchmark, REFLEX achieves 95% success with 72.7% fewer strong-model calls than a strong-only agent, with reductions persisting across three fallback families. Controlled interventions show that reliability depends on action-set size and near-valid alternatives near authorization boundaries. External BFCL and $τ$-style evaluations reveal limited advantages over a cheap generative cascade when ordinary routing is already highly accurate. These findings identify when selective control with Jev can reduce computation and where its benefits are limited.", "url": "https://arxiv.org/abs/2609.26532v1", "pdf_url": "https://arxiv.org/pdf/2609.26532v1", "bibtex_key": "arxiv2609_26532", "included_by": [ "Q01_jev", "Q04_typed_decision", "Q10_recent_decision_models" ], "inclusion_reason": "商业 Jev 控制层 + 强 LLM fallback", "relationship": "Uses hosted jev-1.13.0 as a typed decision layer inside an LLM agent, calling a strong LLM when confidence is low or generation is required.", "summary": "Compares strong-only, cheap-generative-cascade and Jev-gated agents on a frozen 100-task benchmark, adds controlled interventions on action-set size and near-valid alternatives, and checks external BFCL and τ-style tasks.", "task": "Tool-using agent decisions on REFLEX-Sim (100 frozen tasks); BFCL function selection; 300 τ-style episodes over 60 paired tasks.", "datasets": [ "REFLEX-Sim (100 tasks)", "BFCL", "τ-style held-out episodes (300)" ], "baselines": [ "strong-only agent (B0)", "cheap generative cascade with self-escalation", "three strong-fallback families" ], "models": [ "TypeSafe Jev (jev-1.13.0)", "strong LLM fallbacks" ], "main_finding": "At threshold τ = 0.5, REFLEX reached 95% success with 1.12 strong-model calls per task, against 88% and 4.10 calls for the strong-only agent (72.7% fewer strong calls).", "caveats": [ "Only one proprietary decision model is tested.", "On BFCL ordinary routing is near-saturated; on τ-style episodes a cheap generative cascade remains competitive.", "The τ evaluation is a 300-episode held-out validation, not a full leaderboard; API costs are dated snapshots.", "Action-set size and ambiguity effects concentrate in authority and risk families." ], "survey_use": "Compare Jev-only, strong-only, cheap-cascade and Jev-cascade arms on end-task success and the actual number of strong-model calls.", "topics": [ "agents", "selective-control", "efficiency", "routing" ], "stages": [ "control", "use" ], "method_families": [ "hosted_service" ], "applications": [ "agent-control" ], "model_relationship": [ "commercial_jev", "downstream_system" ], "test_level": "system_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13.0" } ], "study_family_id": null, "openness": { "code": { "status": "not_located", "note": "No uniquely matching author repository located; community projects named “reflex” are unrelated." }, "weights": { "status": "not_applicable" }, "data": { "status": "partial", "note": "Uses public BFCL / τ-style tasks; REFLEX-Sim release not located." }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §5.1; Table 1; external evaluation; Limitations", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-26532-calls", "c-26532-external", "c-26532-action-set" ] }, { "id": "arxiv:2609.26550", "arxiv_id": "2609.26550", "versioned_id": "2609.26550v1", "doi": null, "title": "JEV-as-a-Judge: Accept When Confident, Escalate When Unsure", "short_title": "JEV-as-a-Judge", "authors": [ "Yubo Li", "Yidi Miao", "Ramayya Krishnan", "Rema Padman" ], "published_at": "2026-09-22T15:05:56Z", "updated_at": "2026-09-22T15:05:56Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "LLM-as-a-judge enables evaluation across diverse tasks, but inference cost and confidence reliability become critical at scale. We study whether a decision-only judge can provide an economical first pass and identify when stronger evaluation is needed. Comparing jev-as-a-judge with sixteen generative and reward-model judges, with blinded human adjudication, we find it within three percentage points of a state-of-the-art LLM judge, our strongest comparator, on ordinary preference and evidence-grounded factuality at 0.36% of the comparator's fee. Larger gaps arise when judgments require checking a derivation or resisting an elaborately written wrong answer. On several benchmarks, JEV's gap to this comparator is concentrated in low-confidence decisions. A frozen cascade that accepts confident verdicts and escalates uncertain ones retains 99% of the comparator's accuracy at lower cost.", "url": "https://arxiv.org/abs/2609.26550v1", "pdf_url": "https://arxiv.org/pdf/2609.26550v1", "bibtex_key": "arxiv2609_26550", "included_by": [ "Q01_jev", "Q10_recent_decision_models" ], "inclusion_reason": "商业 Jev 与生成式/奖励模型 judge 比较", "relationship": "Evaluates hosted jev-1.13.0 as a model judge against sixteen generative and reward-model judges.", "summary": "Asks whether a decision-only judge can provide an economical first pass and identify when stronger evaluation is needed. Native label probabilities are compared with verbalised probabilities, invalid outputs count as errors, and frozen accept-or-escalate cascades are tested.", "task": "Pairwise preference, evidence-grounded factuality and final-answer adjudication (RewardBench, JudgeBench, HaluEval; 1,312-item extension), with RewardBench 2 and RM-Bench diagnostics.", "datasets": [ "RewardBench", "JudgeBench", "HaluEval", "RewardBench 2 (sample)", "RM-Bench (sample)", "150 saved final-answer replies", "120-decision latency panel" ], "baselines": [ "13 hosted judges incl. GPT-6 Astra, Claude Sonnet 5, Gemini 3.1 Pro", "local Qwen3-32B and Qwen3.5-27B", "PairRM-hf", "Skywork-Reward-V2-Qwen3-8B" ], "models": [ "TypeSafe Jev (jev-1.13.0)" ], "main_finding": "On ordinary preference and evidence-grounded factuality Jev stayed within three points of the strongest judge (GPT-6 Astra) at 0.36% of its fee; a frozen two-order cascade kept 99% of GPT-6’s accuracy at 57% of its fee.", "caveats": [ "Gaps widen when a judgment requires checking a derivation or resisting an elaborately written wrong answer.", "The 1,312-item extension was added after preliminary results were known; multi-family analyses are exploratory.", "Human adjudication covers 183 disagreement-selected items, completed by one author.", "Latency comes from a 120-decision panel at one client location; simulated cascade fees say nothing about real cascade latency.", "Single-order cascades are post hoc; only the frozen two-order policies test a pre-specified rule." ], "survey_use": "Separate native from verbalised probabilities; judge pairs in both orders; count invalid outputs as errors; report fees and real cascade latency separately.", "topics": [ "judging", "selective-control", "calibration", "efficiency", "evaluation-methodology" ], "stages": [ "probability", "control", "use" ], "method_families": [ "hosted_service" ], "applications": [ "judging" ], "model_relationship": [ "commercial_jev" ], "test_level": "model_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13.0", "note": "Price at collection: $0.042 per million input tokens." } ], "study_family_id": null, "openness": { "code": { "status": "claimed_not_located", "note": "The paper points to an anonymous supplementary package; no public package was confirmed. A same-named GitHub project cannot be matched automatically." }, "weights": { "status": "not_applicable" }, "data": { "status": "partial", "note": "Public benchmarks; licensing of the saved replies needs review before redistribution (per the paper)." }, "predictions": { "status": "claimed_not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§4–5, 7; Conclusion; Limitations; Appendices I, L", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-26550-quality", "c-26550-fee", "c-26550-cascade", "c-26550-confidence" ] }, { "id": "arxiv:2609.26758", "arxiv_id": "2609.26758", "versioned_id": "2609.26758v1", "doi": null, "title": "Type-Safe Is Not Error-Free: A Constrained Decision Head Follows the Option Name, Not the Rubric Bound to It", "short_title": "Type-Safe Is Not Error-Free", "authors": [ "Yu Sun", "Junhao Xu" ], "published_at": "2026-09-22T17:38:12Z", "updated_at": "2026-09-22T17:38:12Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "Typed decision models are built for settings where model outputs are consumed directly by software. Instead of generating free-form text, they return a decision over a predefined set of options. By construction, every output conforms to the required schema. Yet this guarantee does not tell us whether the model interprets the options as intended. We study Jev and two Jev-like models with open weights by changing how option names are assigned to rubrics. Each option consists of an option name and a textual rubric that defines what the option means. We change only which option name is assigned to each rubric; the question, state, rubric wording, and set of option names remain exactly the same. On 1200 workflow decisions with task-specific rubrics, renaming the two options from 0/1 to no/yes changes 70.4 more answers per hundred (95% CI: [67.6, 73.1]) and shifts AUC from .94 to .23, revealing a systematic reversal in the decision ranking rather than simple uncertainty. The same operation has little effect with neutral option names. This pattern holds across all 4 predicates, where the effect is at least 7.4x larger than under the neutral control, and becomes stronger as the number of options increases. The effect also depends on the read-out geometry: a second model family that mean-pools over the full option span flips 4.1x less often. The hosted model exhibits the same behavior: the swap changes AUC from .8146 to .5806 and produces 24x as many answer flips as its test-retest floor. In contrast, replacing the option names with random character strings returns all model families to the neutral-control regime without reducing accuracy. The failure therefore depends on the semantic polarity of the option names rather than on the renaming operation itself. Across all conditions, the type-error rate remains 0%, even when decision accuracy degrades substantially.", "url": "https://arxiv.org/abs/2609.26758v1", "pdf_url": "https://arxiv.org/pdf/2609.26758v1", "bibtex_key": "arxiv2609_26758", "included_by": [ "Q01_jev", "Q04_typed_decision", "Q10_recent_decision_models" ], "inclusion_reason": "商业 Jev 黑箱评测 + 两种开放决策头", "relationship": "Black-box evaluation of hosted Jev alongside two open-weight Jev-like encoder decision heads.", "summary": "Holds question, state, rubric wording and the set of option names fixed and changes only which name is bound to which rubric. The study separates type validity (0% type errors by construction) from name invariance, ranking quality and test–retest noise across three read-out families.", "task": "Binary workflow decisions with task-specific rubrics (1,200 items over 4 predicates), plus generic-rubric, multi-way (683 questions) and random-name arms.", "datasets": [ "1,200 binary workflow decisions (4 predicates)", "683 multi-way questions", "1,800-item pooled corpus (span-mean head)" ], "baselines": [ "aligned (as-shipped) name–rubric binding", "neutral names (0/1, A/B)", "random character-string names", "hosted-model test–retest floor" ], "models": [ "Open ModernBERT-large marker-readout head", "Open DeBERTa-v3-large span-mean head", "Hosted TypeSafe model (version not stated)" ], "main_finding": "Rebinding the rubrics behind no/yes lowered hosted Jev’s AUROC from .8146 to .5806 and flipped 32.50% of its answers (neutral names: ~2%; 24× its test–retest floor) while type errors stayed at 0%. The steeper .94→.23 inversion comes from an open marker head, not from Jev.", "caveats": [ "The .9376→.2315 AUROC inversion belongs to an open marker-readout checkpoint, not to hosted Jev.", "One released checkpoint per open read-out geometry; English only.", "The hosted model is a non-deterministic black box observed at one point in time; no model version is pinned.", "The multi-way arm rests on a pool with shipped accuracy .5637 and is directional.", "Mitigations (neutral identifiers; randomising names in training) are proposed, not evaluated." ], "survey_use": "Report type validity, name invariance, ranking quality and test–retest noise as separate quantities; order permutation and name–rubric rebinding are different interventions.", "topics": [ "interface-semantics", "robustness", "open-models", "evaluation-methodology" ], "stages": [ "contract", "readout" ], "method_families": [ "hosted_service", "encoder_head" ], "applications": [ "workflow-decisions" ], "model_relationship": [ "commercial_jev", "independent_jev_like" ], "test_level": "model_test", "model_versions": [ { "model": "TypeSafe hosted model", "version": null, "reason": "not_reported", "note": "Queried over HTTP; vendor pages fetched 2026-09-22." } ], "study_family_id": null, "openness": { "code": { "status": "claimed_not_located", "note": "The paper states that evaluation code, per-item probabilities and raw hosted responses are released with it; no uniquely matching public repository was located." }, "weights": { "status": "not_located", "note": "The two open checkpoints are said to be on the Hugging Face Hub, with names deferred to the unlocated harness." }, "data": { "status": "claimed_not_located" }, "predictions": { "status": "claimed_not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3, 4.1–4.7, 6–7; Tables 6–7", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-26758-hosted-swap", "c-26758-retest", "c-26758-open-inversion", "c-26758-type-errors", "c-26758-random-names", "c-26758-multiway" ] }, { "id": "arxiv:2609.23959", "arxiv_id": "2609.23959", "versioned_id": "2609.23959v1", "doi": null, "title": "Open-Jev Judgments on CallScreenBench: Calibrated One-Pass Scam Screening with a Small Language Model", "short_title": "Open-Jev on CallScreenBench", "authors": [ "Simiao Ren", "Kidus Zewde", "Xingyu Shen", "Yuchen Zhou", "Dennis Ng", "Ankit Raj", "Tommy Duong", "Yuxin Zhang", "Neo Tiangratanakul" ], "published_at": "2026-09-21T00:09:24Z", "updated_at": "2026-09-21T00:09:24Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Screening a phone call for fraud needs a trustworthy probability after every caller turn, in milliseconds. Jev-style typed decisions promise exactly that: declared options go in, one calibrated probability per option comes out of a single forward pass, with no generated text. We test an open implementation of this readout, JevLite, on scam-call screening: Qwen3-4B is LoRA-tuned so that the temperature-scaled softmax over two answer-label logits is P(scam). On 41 held-out CallScreenBench scenarios (577 per-turn decisions) a three-seed ensemble reaches AUROC .974 with calibration error .052, non-inferior to an LLM judge (MiniMax-M3) at a pre-registered .02 margin, with no false alarms on legitimate calls, decisions 1.14 turns earlier under the same hang-up rule, and 64.5 ms per decision on one consumer GPU, 4.9x lower than the same backbone fine-tuned to generate its answer. The gain is in the readout and calibration, not accuracy: a fine-tuned ModernBERT encoder is not significantly worse, the recipe was selected with test-set exposure, and all callers are synthetic. We claim no architectural novelty; the contribution is the application and an evaluation reporting calibration, false alarms and decision timing alongside AUROC.", "url": "https://arxiv.org/abs/2609.23959v1", "pdf_url": "https://arxiv.org/pdf/2609.23959v1", "bibtex_key": "arxiv2609_23959", "included_by": [ "Q01_jev", "Q04_typed_decision", "Q10_recent_decision_models", "Q12c_jevlite" ], "inclusion_reason": "JevLite，独立 Qwen3-4B LoRA 实现", "relationship": "JevLite: the authors’ own open Jev-style readout on Qwen3-4B with LoRA; not TypeSafe’s model.", "summary": "Fine-tunes Qwen3-4B so that a temperature-scaled softmax over two answer-label logits gives P(scam) after every caller turn, and compares three ways of reading the same fine-tuned decoder.", "task": "41 held-out CallScreenBench scenarios (82 calls, 577 turn-level decisions); synthetic callers; three training seeds.", "datasets": [ "CallScreenBench (synthetic calls)" ], "baselines": [ "LLM judge (MiniMax-M3)", "fine-tuned ModernBERT-large", "same Qwen3-4B fine-tuned to generate", "zero-shot Qwen3-4B" ], "models": [ "JevLite (Qwen3-4B + LoRA)" ], "main_finding": "A three-seed ensemble reached AUROC .974 with calibration error .052, non-inferior to an LLM judge at a pre-registered .02 margin, with no false alarms on legitimate calls and decisions 1.14 turns earlier; 64.5 ms per decision on one consumer GPU, 4.9× faster than the same backbone fine-tuned to generate.", "caveats": [ "Recipe selection had test-set exposure (about ten recipes were scored before the selection rule was fixed).", "A fine-tuned ModernBERT encoder is not significantly worse; the gain is in readout and calibration, not accuracy.", "All callers are synthetic.", "Single-model latency and the three-model ensemble quality are measured on different scopes." ], "survey_use": "A same-backbone ablation isolates the readout; judge by false alarms and decision timing, not AUROC alone.", "topics": [ "open-models", "calibration", "efficiency", "robustness" ], "stages": [ "readout", "probability", "use" ], "method_families": [ "finetuned_decoder" ], "applications": [ "fraud-screening" ], "model_relationship": [ "independent_jev_like" ], "test_level": "model_test", "model_versions": [ { "model": "JevLite", "version": "paper v1 (three seeds)" } ], "study_family_id": null, "openness": { "code": { "status": "not_located", "note": "No uniquely matching public package located; similarly named OpenJev projects are unrelated." }, "weights": { "status": "not_located" }, "data": { "status": "not_located" }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; experimental protocol; Table 1; results", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-23959-quality", "c-23959-latency" ] }, { "id": "arxiv:2609.23986", "arxiv_id": "2609.23986", "versioned_id": "2609.23986v1", "doi": null, "title": "Jev-Mem: System-One-Controlled Agentic Memory for Efficient AI Agents", "short_title": "Jev-Mem", "authors": [ "Dongming Jiang", "Yi Li", "Bingzhe Li" ], "published_at": "2026-09-21T01:43:12Z", "updated_at": "2026-09-21T01:43:12Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI", "cs.LG" ], "abstract": "Agentic memory is becoming essential for long-horizon AI agents, yet many existing systems rely on autoregressive LLMs to control how memories are organized, retrieved, and used, placing expensive generation on the critical path of memory operations. We introduce \\textbf{\\method}, a new agentic memory architecture inspired by System-One/System-Two cognition. System One captures fast, lightweight decision-making, whereas System Two performs slower, deliberative reasoning. Jev-Mem brings this division of labor to agentic memory through a dedicated System-One control plane, a structured multi-relational memory plane, and a System-Two reasoning plane. The System-One controller governs memory typing and relational organization during construction, and dynamically performs query routing, retrieval-budget allocation, graph traversal, candidate scoring, and adaptive stopping during retrieval. System Two is invoked only for complex reasoning and answer synthesis. This design improves both memory effectiveness and system efficiency: on LoCoMo Jev-Mem achieves an overall LLM-as-a-Judge score of 0.777, an 11.0\\% relative improvement over the strongest baseline, while reducing memory construction time to 158\\,s, a 6.6$\\times$ speedup over the fastest competing memory system, and lowering average query latency to 0.93\\,s, a 36.7\\% reduction.", "url": "https://arxiv.org/abs/2609.23986v1", "pdf_url": "https://arxiv.org/pdf/2609.23986v1", "bibtex_key": "arxiv2609_23986", "included_by": [ "Q01_jev", "Q03_system_one", "Q11_recent_systemone" ], "inclusion_reason": "商业 Jev 控制平面 + 图记忆 + System Two", "relationship": "Uses hosted Jev as the System-One control plane of an agent memory system; a System-Two LLM writes the answers.", "summary": "Jev decides memory typing and organisation during construction, and query routing, retrieval budget, graph traversal, candidate scoring and stopping during retrieval; an LLM is called only for reasoning and answer synthesis.", "task": "Long-horizon conversational memory on LoCoMo.", "datasets": [ "LoCoMo" ], "baselines": [ "existing agentic memory systems" ], "models": [ "TypeSafe Jev (version not stated)", "System-Two LLM" ], "main_finding": "LLM-as-a-Judge 0.777 on LoCoMo (+11.0% relative to the strongest baseline), memory construction in 158 s (6.6× faster than the fastest competing system) and 0.93 s average query latency (36.7% lower).", "caveats": [ "Each headline number is relative to a different baseline.", "Gains belong to the full memory system, not to Jev alone; module-level attribution is needed.", "The quality metric is itself an LLM judge; transfer to other datasets is untested.", "No Jev model version is pinned." ], "survey_use": "System One decides resources and access; System Two answers. Attribute gains per module rather than reading them as general model superiority.", "topics": [ "agents", "efficiency" ], "stages": [ "control", "use" ], "method_families": [ "hosted_service" ], "applications": [ "agent-memory" ], "model_relationship": [ "downstream_system", "commercial_jev" ], "test_level": "system_test", "model_versions": [ { "model": "Jev", "version": null, "reason": "not_reported" } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "Repository confirmed; memory/jev_client.py calls system_one (read, not executed).", "url": "https://github.com/libingzheren/Jev-Mem/blob/81574eb23f3fd8d1a6c4d54a1e7d6f2dd539e9bb/README.md" }, "weights": { "status": "not_applicable" }, "data": { "status": "available", "note": "Public benchmark." }, "predictions": { "status": "not_located" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; Experiments; Table 1; memory/jev_client.py", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-23986-system" ] }, { "id": "arxiv:2609.24052", "arxiv_id": "2609.24052", "versioned_id": "2609.24052v1", "doi": null, "title": "Calibrated Decisions at Scale: Converting Police Crash Narratives into Probabilistic Crash Variables with a System One Model (Jev)", "short_title": "Calibrated Decisions at Scale", "authors": [ "Amir Rafe", "Subasish Das" ], "published_at": "2026-09-21T03:24:47Z", "updated_at": "2026-09-21T03:24:47Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Crash datasets that carry an investigator narrative hold information the coded fields omit. Coding those narratives at scale has been blocked by three obstacles. Frontier large language models are costly at that scale, their generated text cannot be verified, and no rule says how much output a human must check. This paper formulates narrative coding as gated, typed decisions answered by Jev, a System One model that returns probabilities over analyst-defined options and generates no text. A screen covered 499,500 Texas narratives and 195,857 were coded with a 27-question schema. Cost is governed by schema size rather than narrative length. The probabilities are audited against coded fields and against 2,416 blinded human judgments drawn under a stated sampling design. Two frontier large language models are benchmarked on the same records. Against human labels the typed model attains an F1 of 0.908. One frontier model gains 0.059 and the other is indistinguishable from it. Calibration varies by model rather than by paradigm, so each model must be audited. Recalibration on the same labels reduces calibration error by a factor of 3.3. Agreement with coded fields understates fidelity to the narrative by a median of 0.26 in kappa. A resolution-floor bound covers any model that reports probabilities on a discrete grid. A review budget over flagged records gives the records a human must read per variable and per year. Adding the calibrated variables to the coded fields raises the injury and fatal crashes attributed to nine factors by 10,747 per year.", "url": "https://arxiv.org/abs/2609.24052v1", "pdf_url": "https://arxiv.org/pdf/2609.24052v1", "bibtex_key": "arxiv2609_24052", "included_by": [ "Q01_jev", "Q03_system_one", "Q04_typed_decision", "Q05_calibrated_decisions", "Q10_recent_decision_models", "Q11_recent_systemone" ], "inclusion_reason": "商业 Jev 编码交通事故叙述", "relationship": "Uses hosted jev-1.13.0 to code police crash narratives as gated, typed decisions.", "summary": "Formulates narrative coding as gated typed decisions, audits probabilities against coded fields and blinded human judgments, benchmarks two frontier LLMs, and turns calibration into a human-review budget.", "task": "Screen of 499,500 Texas crash narratives (10-question schema); full 27-question coding of 195,857; 2,416 usable blinded human judgments (effective sample ≈ 1,265).", "datasets": [ "Texas CRIS crash narratives 2017–2025 (restricted)" ], "baselines": [ "two frontier LLMs on the same records", "administrative coded fields" ], "models": [ "TypeSafe Jev (jev-1.13.0)" ], "main_finding": "Against human labels the typed model reached F1 0.908; one frontier model was 0.059 higher and the other indistinguishable. Probabilities ranked well but overstated prevalence; recalibration on the same labels cut calibration error about 3.3×.", "caveats": [ "Single region, single model version and single schema.", "Recalibration was fitted on labels from the same distribution; it is not a guarantee for new regions or schemas.", "Two-decimal output probabilities put a floor on calibration error for rare variables.", "Narratives and record-level files are not redistributable; a de-identified audit table is available on request once the data owner confirms." ], "survey_use": "Distinguish administrative codes from narrative-grounded truth, and convert probability reliability into a human-review workload.", "topics": [ "calibration", "selective-control", "evaluation-methodology", "efficiency" ], "stages": [ "probability", "control", "use" ], "method_families": [ "hosted_service" ], "applications": [ "annotation" ], "model_relationship": [ "commercial_jev", "downstream_system" ], "test_level": "system_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13.0", "note": "Pinned identifier recorded per call." } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "Schema, asynchronous runner, metric implementations with tests and figure generators are public.", "url": "https://github.com/pozapas/jev-calibrated-narrative-coding/blob/850fe4f0db35f165b56247a53d786cd1dfc12302/README.md" }, "weights": { "status": "not_applicable" }, "data": { "status": "restricted", "note": "Underlying narratives are not redistributable under the data agreement." }, "predictions": { "status": "partial", "note": "Aggregated outputs are public; the pair-level audit table is available on request pending approval." }, "recomputable": { "status": "partial", "note": "Population-scale results depend on restricted Stage-2 files." }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3–5; Table 6; Data and code availability", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-24052-scale", "c-24052-f1", "c-24052-recalibration", "c-24052-quantisation", "c-24052-reference" ] }, { "id": "arxiv:2609.24395", "arxiv_id": "2609.24395", "versioned_id": "2609.24395v1", "doi": null, "title": "JEVQA - Video Quality from Metadata, Bitstream, and Pixel Features with a General-Purpose Decision Model", "short_title": "JEVQA", "authors": [ "Werner Robitza" ], "published_at": "2026-09-21T10:45:00Z", "updated_at": "2026-09-21T10:45:00Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "eess.IV" ], "abstract": "Instrumental quality models for video quality prediction are usually trained for a fixed set of codecs or other input features, and every new input variable requires retraining. Novel, general-purpose decision models can answer questions without task-specific training, but it is unclear whether they can judge video quality. We evaluate Jev, a commercial ``System One'' model that returns probability distributions over a provided answer scale, as a zero-shot video quality model. We call the resulting method JEVQA. In a first study on 1,936 AV1, H.264, HEVC, and VP9 encodes of 22 sources, scored against VMAF as ground truth, using encoding metadata only, JEVQA reached a Pearson correlation of 0.737, on par with the standardized ITU-T P.1204.1 model (0.733). Giving the model bitstream data raised the accuracy to 0.797, and combined pixel-based and bitstream features raised it to 0.824. A pixel-only variant failed in our tests. In a second study, using H.264, HEVC, and VP9 encodes in the AVT-VQDB-UHD-1 database, the metadata-only model reached a correlation of 0.879 with MOS, close to P.1204.1 (0.898). Bitstream statistics did not help there. Our results show that trained models on the same features remain clearly ahead in both studies, but that zero-shot classifiers are promising.", "url": "https://arxiv.org/abs/2609.24395v1", "pdf_url": "https://arxiv.org/pdf/2609.24395v1", "bibtex_key": "arxiv2609_24395", "included_by": [ "Q01_jev", "Q03_system_one", "Q06_jevqa", "Q10_recent_decision_models", "Q11_recent_systemone" ], "inclusion_reason": "商业文本 Jev 读取视频特征，非原生视频输入", "relationship": "Hosted text-only Jev reads video metadata, bitstream statistics and pixel-derived features; the video itself is never an input.", "summary": "Tests a general decision model as a zero-shot video-quality predictor from structured features and compares it with the standardised ITU-T P.1204 models and with regressors trained on the same features.", "task": "Study A: 1,936 AV1/H.264/HEVC/VP9 encodes of 22 sources scored against VMAF. Study B: 562 encodes from AVT-VQDB-UHD-1 with subjective MOS.", "datasets": [ "CRF ladders of 22 sources (Study A)", "AVT-VQDB-UHD-1 (Study B)" ], "baselines": [ "ITU-T P.1204.1", "models trained on the same features" ], "models": [ "TypeSafe Jev (jev-1.13)" ], "main_finding": "With metadata only, JEVQA reached PLCC 0.737 against VMAF (P.1204.1: 0.733), rising to 0.797 with bitstream data and 0.824 with bitstream plus pixel features; against MOS it reached 0.879 (P.1204.1: 0.898).", "caveats": [ "Study A’s target is VMAF, not human MOS.", "Trained models on the same features remain clearly ahead in both studies; the pixel-only variant failed.", "Standard models have seen part of the Study B database.", "Feature extraction does the perceptual work; this does not show that Jev can watch video." ], "survey_use": "Separate perceptual preprocessing, feature design and the decision model; distinguish zero-shot from supervised baselines.", "topics": [ "multimodal", "evaluation-methodology" ], "stages": [ "contract", "use" ], "method_families": [ "hosted_service" ], "applications": [ "video-quality" ], "model_relationship": [ "commercial_jev", "downstream_system" ], "test_level": "model_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13", "note": "Patch version not stated." } ], "study_family_id": null, "openness": { "code": { "status": "not_located", "note": "Video tooling repositories cited in the paper are not the study’s full reproduction package." }, "weights": { "status": "not_applicable" }, "data": { "status": "partial", "note": "Study B uses a public database; Study A encodes are described." }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§III–VI; Limitations", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-24395-study-a", "c-24395-study-b" ] }, { "id": "arxiv:2609.24574", "arxiv_id": "2609.24574", "versioned_id": "2609.24574v1", "doi": null, "title": "Evaluating Decision Models for Text Annotation in Computational Social Science", "short_title": "Decision Models for CSS Annotation", "authors": [ "Hazem Ibrahim", "Yasir Zaki" ], "published_at": "2026-09-21T13:41:54Z", "updated_at": "2026-09-23T08:36:11Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.CY" ], "abstract": "Computational social science increasingly relies on large language models for text annotation, and the validity of published findings now rests on the labels generated by such models. Decision models, a new model class built for categorical question answering, answer typed questions with a choice, a probability distribution over the label set, and a confidence score rather than free text, at a small fraction of frontier inference prices. Whether their answers are accurate, and whether that stated confidence can be trusted on social science constructs, are unknown. Here, we mirror the evaluation of Ziems et al. (2024) on 18 computational social science classification tasks (7,977 items), comparing the first commercial decision model and two open-weight counterparts against 19 frontier and open-weight language models under the same zero-shot protocol. The decision model trails the per-task best LLM on 14 of 15 evaluation tasks, with a median deficit of 11.6 macro-F1 points, at a median 44 times lower measured cost. Its confidence is better calibrated than the verbalized confidence of 16 of the 19 LLMs, yet three frontier models show lower median calibration error (0.157 against 0.066). While items above 0.9 confidence are typically labeled accurately (median accuracy 0.815), on one task, empathy in peer-support dialogues, the model reports high confidence while performing near chance. Nonetheless, our results suggest that decision models are useful as a first step in the annotation pipeline: routing low-confidence items to an LLM matches or exceeds the LLM alone at a quarter to half of its cost.", "url": "https://arxiv.org/abs/2609.24574v1", "pdf_url": "https://arxiv.org/pdf/2609.24574v1", "bibtex_key": "arxiv2609_24574", "included_by": [ "Q10_recent_decision_models" ], "inclusion_reason": "商业 Jev + 开放决策模型 + 19 个 LLM", "relationship": "Evaluates hosted Jev and open-weight decision models against 19 LLMs under one zero-shot protocol.", "summary": "Replicates an 18-task computational-social-science suite to ask whether decision models can serve as annotators. Accuracy, measured cost and calibration are reported together, and low-confidence routing to an LLM is tested.", "task": "18 CSS classification tasks (7,977 items: 3 for selection, 15 for evaluation); an appendix extends to open decision models.", "datasets": [ "Ziems et al. (2024) CSS suite" ], "baselines": [ "19 frontier and open-weight LLMs (verbalised confidence)", "open decision models: Laya, decider-0.8b/2b, SemIf, Bespoke-Nimble-9B, Qwen3-0.6B RLCD" ], "models": [ "TypeSafe Jev (typesafe/jev1.13 via OpenRouter)", "open-weight decision models" ], "main_finding": "Jev trailed the per-task best LLM on 14 of 15 tasks (median −11.6 macro-F1) at a median 44× lower measured cost. Its median ECE (0.157) beat the verbalised confidence of 16 of 19 LLMs but not three frontier Claude models (best 0.066). Routing low-confidence items to an LLM matched or exceeded the LLM alone at a quarter to half of its cost.", "caveats": [ "High confidence is not reliable across constructs: on empathy the model is confident while performing near chance (task calibration error 0.538).", "Verbalised LLM confidence and native probabilities are different signals; baselines that decline to estimate are excluded from calibration.", "Older public corpora may overlap with training data; the overlap is unknown." ], "survey_use": "A core paper that brand-name searches miss; the clearest accuracy–cost–calibration trade-off in the set.", "topics": [ "calibration", "selective-control", "efficiency", "open-models", "classification" ], "stages": [ "probability", "control", "use" ], "method_families": [ "hosted_service", "encoder_head", "frozen_decoder_readout", "finetuned_decoder" ], "applications": [ "annotation" ], "model_relationship": [ "commercial_jev", "independent_jev_like" ], "test_level": "model_test", "model_versions": [ { "model": "Jev", "version": "typesafe/jev1.13", "note": "Accessed via the OpenRouter decisions API (reference [26] of the paper)." } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "Author replication package with analysis scripts; pilot_jev.py was read (not executed).", "url": "https://github.com/hazemibrahim97/decision-models-css/blob/45b71ce8403eababae18cc51dda5ecb6d67e05e4/README.md" }, "weights": { "status": "not_applicable" }, "data": { "status": "available", "note": "Public releases of the Ziems et al. suite and upstream sources." }, "predictions": { "status": "available", "note": "Complete per-call records are released." }, "recomputable": { "status": "available", "note": "Aggregates can be recomputed from released records." }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3–5; Appendix B; Ethics and Limitations; repository", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-24574-accuracy-cost", "c-24574-ece", "c-24574-empathy", "c-24574-cascade" ] }, { "id": "arxiv:2609.24965", "arxiv_id": "2609.24965", "versioned_id": "2609.24965v1", "doi": null, "title": "Jev for Scientific Decisions: Evaluating Semantic Choices and Their Consequences", "short_title": "Jev for Scientific Decisions", "authors": [ "Boyuan Deng", "Shuyi Fan", "Hongyang Zhang", "Xinhong Xie" ], "published_at": "2026-09-21T17:51:50Z", "updated_at": "2026-09-23T04:04:31Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Scientific workflows often require choosing among known relations before a deterministic calculation can proceed. Whether observations share a culture, treatment or reference standard can change the scientific meaning of the resulting count or comparison. We evaluate Jev as a semantic decision component using a harness that follows its documented guidance and assigns arithmetic to code. The study compares twelve model configurations on twenty source-grounded Choices across ten scientific cases, each repeated five times. We measure semantic selections, downstream outputs and final claim labels separately. Jev matched five other configurations at complete semantic correctness and achieved the lowest observed median latency among successful responses. Across three comparison models, seven wrong selections on one culture-history question changed downstream counts while preserving the correct final label. These results identify a useful role for Jev in prepared scientific decision tasks and show why evaluating that role requires checking the relations and quantities that a workflow will reuse.", "url": "https://arxiv.org/abs/2609.24965v1", "pdf_url": "https://arxiv.org/pdf/2609.24965v1", "bibtex_key": "arxiv2609_24965", "included_by": [ "Q01_jev" ], "inclusion_reason": "商业 Jev 语义选择，代码计算", "relationship": "Evaluates hosted Jev 1.13 as a semantic decision component; arithmetic stays in code, following the vendor’s documented division of labour.", "summary": "Twenty source-grounded Choices across ten scientific cases decide which relation governs a downstream calculation. Semantic selections, derived quantities and final claim labels are scored separately.", "task": "20 Choices in 10 scientific cases; 12 model configurations; 5 repetitions each.", "datasets": [ "10 curated scientific cases (20 Choices)" ], "baselines": [ "eleven other model configurations (frontier and open-weight LLMs)" ], "models": [ "TypeSafe Jev 1.13" ], "main_finding": "Jev matched five other configurations at complete semantic correctness and had the lowest observed median latency among successful responses (0.335 s; p95 0.442 s). Seven wrong selections by comparison models changed downstream counts while leaving the final label correct.", "caveats": [ "A tiny development collection; repetitions do not add independent cases.", "Every model-routed reference label is “inconsistent”, so balanced claim decisions are untested.", "All observed semantic errors concern a single Choice." ], "survey_use": "Score intermediate relations, derived quantities and final labels together: a correct label can hide a wrong process.", "topics": [ "evaluation-methodology", "efficiency" ], "stages": [ "contract", "use" ], "method_families": [ "hosted_service" ], "applications": [ "scientific-workflow" ], "model_relationship": [ "commercial_jev" ], "test_level": "system_test", "model_versions": [ { "model": "Jev", "version": "Jev 1.13", "note": "Patch version not stated." } ], "study_family_id": null, "openness": { "code": { "status": "not_located", "note": "Protocol and appendices are detailed; no public code repository confirmed." }, "weights": { "status": "not_applicable" }, "data": { "status": "partial", "note": "Case materials are described in appendices." }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §3.2; Table 1; Limitations", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-24965-correct-latency", "c-24965-hidden-errors" ] }, { "id": "arxiv:2609.23886", "arxiv_id": "2609.23886", "versioned_id": "2609.23886v1", "doi": null, "title": "this-that-model-1.0: A typed decision model that decides in 30 ms, for a millionth of a cent", "short_title": "this-that-model-1.0", "authors": [ "Zehua Cheng", "Wei Dai", "Jiahao Sun" ], "published_at": "2026-09-20T21:43:03Z", "updated_at": "2026-09-20T21:43:03Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Software delegates more of its branches to models every year: which queue a ticket enters, whether a command is safe to run, whether a claim clears without a person. What the program needs back is not prose. It is one of n declared options and a number it can threshold. Today that costs a round trip to a frontier model -- hundreds of milliseconds, a per-token bill, and a parser -- for a question that is usually a conjunction of three clauses. this-that-model-1.0 is a 2B-parameter typed decision model. Its answer is read directly from the hidden state at a designated position and restricted to the option set the caller declared, so no text is generated, nothing can be malformed, and every question in a request is answered in the same forward pass. It decides in 30.9 ms on one laptop GPU and generates zero output tokens doing it, where a frontier API call costs 8758 ms and the hosted systems that answer these questions well spend between 21 and 212 generated tokens per question thinking first, billed for every one. It sustains 32 decisions per second on one consumer GPU and never lets the state leave the machine. On a third party's recorded cohort of 68 decision questions, on their inputs and their wording, it scores 0.941 with a Brier score of 0.042, against 0.765 and 0.133 for the hosted service Jev on the same items. One pass of our 42-family internal suite takes 32 seconds and 0.000217 USD of electricity; the most accurate hosted model we measured needs 155.2 minutes and 10.636 USD. We also report where it loses. On multi-step arithmetic, which a single forward pass cannot carry intermediate results through, it scores 0.560 against their 0.98 to 1.00, and a targeted second training round improved the five task families it was written for and transferred to none of the other 13. The model is open-sourced in https://huggingface.co/flock-io/this-that-model-1.0", "url": "https://arxiv.org/abs/2609.23886v1", "pdf_url": "https://arxiv.org/pdf/2609.23886v1", "bibtex_key": "arxiv2609_23886", "included_by": [ "Q01_jev", "Q04_typed_decision", "Q10_recent_decision_models" ], "inclusion_reason": "this-that-model-1.0；开放替代及 Jev 比较", "relationship": "Open-weight ~2B typed decision model adapted from decider-2b, compared with hosted Jev.", "summary": "Reads the answer from the hidden state at a designated position, restricted to the declared options, and trains with a proper scoring rule; reports local latency, electricity cost and accuracy against hosted systems.", "task": "A 42-family internal suite; a third party’s recorded 68-question cohort (17 states); a released 7,305-question benchmark, of which a 2,250-question subset was answered by every hosted system.", "datasets": [ "this-that-spatial-bench (7,305 questions)", "NanoJev recorded cohort (68 questions)", "internal 42-family suite (not released)" ], "baselines": [ "hosted Jev", "frontier hosted LLMs", "NanoJev-0.6B" ], "models": [ "this-that-model-1.0 (~2B)" ], "main_finding": "30.9 ms per decision on one laptop GPU; on the 68-question cohort 0.941 accuracy (Brier 0.042) against Jev’s 0.765 (0.133); on the 2,250-question subset 0.844 against Jev’s 0.803.", "caveats": [ "The 68-question accuracy gap rests on 12 questions.", "The model was trained on the 2,250-question subset’s question shapes; Jev was not.", "Electricity-only marginal cost excludes hardware and operations and is not comparable to an API price.", "Weaker than strong reasoning models on multi-step arithmetic; on branch-shaped families 0.818 against 0.837–0.968 for hosted models." ], "survey_use": "Local deployment with native restricted readout; compare hardware, training coverage, cost scope and weight availability separately.", "topics": [ "open-models", "efficiency", "calibration" ], "stages": [ "readout", "probability", "use" ], "method_families": [ "finetuned_decoder" ], "applications": [ "workflow-decisions" ], "model_relationship": [ "independent_jev_like" ], "test_level": "model_test", "model_versions": [ { "model": "this-that-model-1.0", "version": "1.0" }, { "model": "Jev (comparison)", "version": null, "reason": "not_reported" } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "Inference package and scripts said to reproduce every measurement; thisthat/model.py (restricted softmax) was read.", "url": "https://github.com/FLock-io/this-that-model/blob/542d445efa5f68b14bfbd1f8ed25aacd8379d839/README.md" }, "weights": { "status": "available", "note": "Hugging Face flock-io/this-that-model-1.0 (MIT); model API returned 200 on 2026-09-23. The PDF text shows “thisthat-model-1.0”, a line-break artefact.", "url": "https://huggingface.co/flock-io/this-that-model-1.0" }, "data": { "status": "partial", "note": "The 7,305-question benchmark is released; the internal suite is not." }, "predictions": { "status": "not_located" }, "recomputable": { "status": "partial" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3–4; Table 1; comparison discussion; Availability; thisthat/model.py", "priority": "P0", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-23886-latency", "c-23886-cohort", "c-23886-subset", "c-23886-cost" ] }, { "id": "arxiv:2609.22753", "arxiv_id": "2609.22753", "versioned_id": "2609.22753v1", "doi": null, "title": "Replacing Large Language Models with Jev Decision Models for Low-Latency Edge Service Orchestration", "short_title": "Replacing LLMs with Jev at the Edge", "authors": [ "Delong Li", "Xu Wang", "Haochen Gong", "Rui Lang", "Guangsheng Yu" ], "published_at": "2026-09-19T04:26:37Z", "updated_at": "2026-09-19T04:26:37Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.DC", "cs.NI" ], "abstract": "Natural-language service requests can require a language-model decision before execution starts, consuming part of the request's latency budget. We integrate Jev's decision-oriented application programming interface (API) into edge service orchestration to reduce this overhead while retaining service completion. The integration extracts four bounded intent fields and applies a shared validator, admission policy, and scheduler, accounting for decision waiting throughout the request timeline. We compare Jev with a short, structured-output DeepSeek deployment using live API measurements followed by modeled execution, and then a real two-node optical character recognition (OCR) service with self-hosted Qwen and rule-based references. Across three consecutive measurement blocks, Jev reduces median client decision latency by 15.9-26.5%. In eight paired OCR conditions, Jev matches DeepSeek's correct, on-time completion count in seven and exceeds it in one. Without caching, median end-to-end latency on requests completed correctly by both systems is 11.1-25.3% lower; API fees per correct completion are 69.0-70.6% lower. Repeated-request caching largely removes the latency difference. The results demonstrate that decision-model substitution can lower both response latency and API fees in the tested service path, and identify fresh interpretation as the main opportunity for latency savings.", "url": "https://arxiv.org/abs/2609.22753v1", "pdf_url": "https://arxiv.org/pdf/2609.22753v1", "bibtex_key": "arxiv2609_22753", "included_by": [ "Q01_jev", "Q10_recent_decision_models" ], "inclusion_reason": "商业 Jev 替换结构化 LLM", "relationship": "Hosted Jev replaces a structured-output LLM for bounded intent extraction in an edge service path.", "summary": "Extracts four bounded intent fields with a shared validator, admission policy and scheduler, compares Jev with a matched short structured-output DeepSeek deployment, and measures what caching does to the difference.", "task": "Four bounded intent fields; three consecutive live-API measurement blocks followed by modeled execution; eight paired conditions on a real two-node OCR service.", "datasets": [ "live API measurement blocks", "two-node OCR service workload" ], "baselines": [ "structured-output DeepSeek", "self-hosted Qwen", "rule-based references" ], "models": [ "TypeSafe Jev (jev-1.13)" ], "main_finding": "Jev cut median client decision latency by 15.9–26.5%. Without caching, end-to-end latency on requests both systems completed correctly was 11.1–25.3% lower and API fees per correct completion 69.0–70.6% lower; correct on-time completion matched DeepSeek in 7 of 8 conditions and exceeded it in 1.", "caveats": [ "Repeated-request caching largely removes the latency difference.", "Results are specific to the measured services, provider regions and time windows.", "Same study family as 2609.23136." ], "survey_use": "A deployment-like matched structured-output baseline; compare cost per correct completion rather than price per token.", "topics": [ "efficiency", "structured-output" ], "stages": [ "contract", "use" ], "method_families": [ "hosted_service", "generative_adapter" ], "applications": [ "edge-orchestration" ], "model_relationship": [ "commercial_jev", "downstream_system" ], "test_level": "hybrid", "model_versions": [ { "model": "Jev", "version": "jev-1.13", "note": "Patch version not stated." } ], "study_family_id": "li-wang-edge-2026", "openness": { "code": { "status": "not_located", "note": "No public code repository confirmed." }, "weights": { "status": "not_applicable" }, "data": { "status": "partial" }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; execution and caching comparisons; §§ on end-to-end latency and the cache boundary", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-22753-latency", "c-22753-e2e-fees", "c-22753-cache" ] }, { "id": "arxiv:2609.23136", "arxiv_id": "2609.23136", "versioned_id": "2609.23136v1", "doi": null, "title": "Fast Intent-Driven Service Orchestration with Jev for 6G Edge Networks", "short_title": "Intent-Driven 6G Edge Orchestration", "authors": [ "Delong Li", "Xu Wang", "Haochen Gong", "Rui Lang", "Guangsheng Yu" ], "published_at": "2026-09-19T17:06:36Z", "updated_at": "2026-09-19T17:06:36Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.NI" ], "abstract": "Intent-driven services envisioned for sixth-generation (6G) edge networks must translate changing requirements into executable contracts while wireless requests continue to arrive. Interpretation consumes part of the same deadline budget as transmission, queueing, and execution. We evaluate Jev as a decision model for this stage, asking whether faster contract activation improves service timeliness while preserving interpretation quality. Contracts specify permitted execution locations, deadlines, and priorities; a numerical scheduler uses them to allocate edge work. The evaluation combines live Jev, DeepSeek, Gemini, and self-hosted Qwen responses with packet-level New Radio simulation, mobility events, shared edge queues, and a real image-reading service. Cached interpretation with numerical scheduling substantially improves completion over direct model-selected placement. With correct interpretation on all evaluated contracts, Jev reduces median decision latency by 22.4% against DeepSeek and 61.9% against Gemini; completion in the modeled update scenarios rises by 3.50 and 8.35 percentage points. A separate comparison with direct-attribute Qwen retains a 53.0% latency reduction and a 4.78-point completion gain. Exchanging recorded model and scheduling waits reproduces the comparator completion summaries for all 12 trajectories in each of two modeled comparisons. In the real image service, Jev retains faster decisions and completes 459 requests correctly and on time, compared with 463 for DeepSeek and 435 for Qwen out of 1,080 each. These findings connect fast intent decisions to timely edge service execution and support using Jev at contract activation, with numerical scheduling adapting placements to current network and computing state.", "url": "https://arxiv.org/abs/2609.23136v1", "pdf_url": "https://arxiv.org/pdf/2609.23136v1", "bibtex_key": "arxiv2609_23136", "included_by": [ "Q01_jev", "Q10_recent_decision_models" ], "inclusion_reason": "商业 Jev 解释意图，确定性调度执行", "relationship": "Hosted Jev interprets service intents into contracts; a deterministic numerical scheduler places the work.", "summary": "Combines live Jev, DeepSeek, Gemini and self-hosted Qwen responses with packet-level New Radio simulation, mobility, shared edge queues and a real image-reading service to ask whether faster contract activation improves timeliness.", "task": "Contract interpretation (locations, deadlines, priorities) for edge orchestration; 12 modeled trajectories per comparison; an image service with 1,080 arrivals per method.", "datasets": [ "5G-LENA NR simulation traces", "image-reading (OCR) service workload" ], "baselines": [ "DeepSeek", "Gemini", "self-hosted Qwen (direct attributes)", "direct model-selected placement" ], "models": [ "TypeSafe Jev 1.13" ], "main_finding": "With correct interpretation on all evaluated contracts, Jev cut median decision latency by 22.4% against DeepSeek and 61.9% against Gemini. In the real image service over simulated NR access it completed 459 of 1,080 requests correctly and on time, against 463 for DeepSeek and 435 for Qwen.", "caveats": [ "Faster decisions did not yield more correct on-time completions than DeepSeek in the image service.", "Network access is simulated; the modeled completion gains (+3.50 and +8.35 points) come from simulation.", "Same author group and service path as 2609.22753: one study family, not independent replication." ], "survey_use": "Decompose interpretation, network, queueing, scheduling and execution time; check whether decision latency is on the critical path.", "topics": [ "efficiency", "agents" ], "stages": [ "contract", "use" ], "method_families": [ "hosted_service" ], "applications": [ "edge-orchestration" ], "model_relationship": [ "commercial_jev", "downstream_system" ], "test_level": "hybrid", "model_versions": [ { "model": "Jev", "version": "Jev 1.13", "note": "Patch version not stated." } ], "study_family_id": "li-wang-edge-2026", "openness": { "code": { "status": "not_located", "note": "No public code repository confirmed." }, "weights": { "status": "not_applicable" }, "data": { "status": "partial" }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; real-service results; Fig. and table on 1,080 arrivals", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-23136-latency", "c-23136-completion", "c-23136-modeled" ] }, { "id": "arxiv:2609.25498", "arxiv_id": "2609.25498", "versioned_id": "2609.25498v1", "doi": "10.5281/zenodo.22867426", "title": "Universal Fractal Natural Language Decision Map: Real-Time Edge Triage Across Heterogeneous Domains", "short_title": "Universal Fractal Decision Map", "authors": [ "Volkan Dağlı", "Zerrin Dağlı", "Dağhan Dağlı" ], "published_at": "2026-09-21T23:57:31Z", "updated_at": "2026-09-21T23:57:31Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "peripheral", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.NE", "cs.AI", "cs.CL" ], "abstract": "Deploying Large Language Models for runtime operational triage incurs prohibitive latency (>100-500 ms), high VRAM requirements (>4-8 GB), and excessive energy dissipation. Extending Mandelbrot Fractal Neural Synthesis (Dagli et al., 2026), this paper presents the Universal Fractal Natural Language Decision Map, realized via the werr machine-native edge reflex runtime and the production answerr platform (https://answerr.me). Operating entirely without stored weight tensors (0 Bytes VRAM), the engine synthesizes deterministic decisions---noul (Boolean), choice (categorical), and score (ordinal)---by dynamically modulating 24-byte coordinate seeds along the chaotic boundary of the Mandelbrot set and evaluating 4-quadrant escape dynamics. Drawing inspiration from biological System-One reflex arcs, the engine introduces: (i) an Auto-Seed Router with domain projector Phi_D yielding a +28.8% accuracy gain over linear baselines; (ii) an Information-Theoretic Acoustic Damping Filter grounded in token entropy and phonetic spectral density that insulates against prompt injections (0.0% empirical bypass; 95% Wilson CI: [0.0%, 30.8%]) while pruning escape iterations by 45.8% (accelerating throughput 2.5x to 3.31 ms latency); and (iii) an Organic Dynamic Calibration framework using O(1) Exponential Moving Average (EMA, alpha=0.03) and quadrant phase rotation to eliminate positional bias. Benchmarked on bare-metal infrastructure (api.answerr.me:4431) across 1,150+ verified decisions (3,200+ questions) and ranked World #1 on the independent JevBench suite (81.65%), the framework achieves 92.6% macro-accuracy (95% CI: [90.8%, 94.1%]) with 7.08 ms median CPU latency. We provide an OpenAI-compatible API (/v1/chat/completions) and demonstrate feasibility on microcontrollers and 32-byte EVM smart contracts.", "url": "https://arxiv.org/abs/2609.25498v1", "pdf_url": "https://arxiv.org/pdf/2609.25498v1", "bibtex_key": "arxiv2609_25498", "included_by": [ "Q03_system_one", "Q11_recent_systemone" ], "inclusion_reason": "werr / Universal Fractal，自称 JevBench 替代方案", "relationship": "An independent, Jev wire-compatible runtime (werr) with Noul/Choice/Score-style outputs; not TypeSafe research.", "summary": "Proposes a procedural “fractal” decision engine without stored weight tensors and reports speed, accuracy and prompt-injection results on the authors’ own corpus and on a community leaderboard.", "task": "Author corpus of 1,150+ decisions (3,200+ questions); community JevBench submission.", "datasets": [ "author corpus (1,150+ decisions)", "JevBench (community)" ], "baselines": [ "LLM and neural baselines described by the authors" ], "models": [ "werr runtime" ], "main_finding": "Author-reported 92.6% macro-accuracy (95% CI 90.8–94.1%) on 1,150+ decisions, 7.08 ms median latency on CPU, and a first-place claim on the community JevBench board (81.65%).", "caveats": [ "All results are self-reported on the authors’ corpus and infrastructure; the leaderboard claim rests on a community submission.", "“0.0% injection bypass” carries a 95% Wilson interval of 0–30.8%: a small sample, not a security guarantee.", "The proposed mechanism has not been independently examined." ], "survey_use": "Kept as a boundary case for the scope; not pooled with controlled comparisons.", "topics": [ "open-models", "efficiency", "robustness" ], "stages": [ "readout" ], "method_families": [], "applications": [ "triage" ], "model_relationship": [ "independent_jev_like" ], "test_level": "model_test", "model_versions": [ { "model": "werr", "version": null, "reason": "not_reported" } ], "study_family_id": null, "openness": { "code": { "status": "partial", "note": "Public repository located.", "url": "https://github.com/pCwOrM/werr/blob/2c3138200f1725f0efd8598b8b84729abde67c95/README.md" }, "weights": { "status": "not_applicable" }, "data": { "status": "partial" }, "predictions": { "status": "not_located" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; evaluation and claims", "priority": "P2", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-25498-self", "c-25498-injection" ] }, { "id": "arxiv:2609.17977", "arxiv_id": "2609.17977", "versioned_id": "2609.17977v1", "doi": null, "title": "When to Call an LLM: A Confidence-Gated Hybrid for Cost-Effective Emotion Recognition in Conversational AI", "short_title": null, "authors": [ "Sai Babu Udayagiri", "Arjun Chouhan", "Ravisekhar Kanagala", "Trishala Pavagada" ], "published_at": "2026-09-16T01:05:22Z", "updated_at": "2026-09-16T01:05:22Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "Emotion recognition in conversation (ERC) is a production capability behind agent-assist prompts, escalation routing, and post-call analytics in contact-center-as-a-service (CCaaS) platforms, where cost and latency constraints matter as much as accuracy. We report a systems-level comparison of three deployment options for dialogue-contextual ERC: a low-cost stacked ensemble (sentence embeddings, windowed context, RandomForest/XGBoost/logistic-regression stacking), off-the-shelf LLM prompting (GPT-4o-mini; zero-shot, few-shot, chain-of-thought), and a confidence-gated hybrid that escalates only the ensemble's least-confident predictions to the LLM - modeled on IVA-to-human-agent escalation policies used in production contact centers. On IEMOCAP, the ensemble significantly outperforms every LLM configuration (0.595 vs. 0.460-0.536 weighted F1, p < 0.0001) at a fraction of the cost and sub-10ms latency; on MELD and CMU-MOSI the ranking reverses, showing neither pure system is a safe default. The confidence-gated hybrid resolves this by Pareto-dominating both pure systems on all three datasets (0.620, 0.643, 0.824 weighted F1) while routing the majority of traffic through the near-zero-cost ensemble, translating to roughly $10-85 per million utterances versus $99-170 for an LLM-only pipeline. The escalation policy is not an opaque cost/accuracy dial: escalated turns disproportionately follow an emotion or sentiment shift, giving operators an interpretable, auditable routing signal, and the ensemble's confidence is well-calibrated and safely under- rather than over-confident. The pattern holds across three datasets and two LLM providers. Confidence-gated cascading is established in general ML systems; our contribution is showing it transfers cleanly to dialogue-contextual ERC, yielding a concrete deployment recipe for CCaaS and conversational-AI platforms deciding how to allocate LLM spend.", "url": "https://arxiv.org/abs/2609.17977v1", "pdf_url": "https://arxiv.org/pdf/2609.17977v1", "bibtex_key": "arxiv2609_17977", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Confidence-gated escalation from a cheap trained ensemble to an LLM; direct precedent and baseline for first-pass-then-escalate designs (finding F3).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control", "routing", "classification" ], "stages": [ "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "selective", "role": "A cheap stacked ensemble that escalates only its least-confident predictions to an LLM; on IEMOCAP the ensemble alone beat every LLM configuration — trained cheap models are the first baseline for escalation designs.", "claims": [] }, { "id": "arxiv:2609.13288", "arxiv_id": "2609.13288", "versioned_id": "2609.13288v1", "doi": null, "title": "Target-Checked Reliability Score Refinement for Video Question Answering", "short_title": null, "authors": [ "Guoxiang Ren", "Rohitash Chandra" ], "published_at": "2026-09-09T08:29:00Z", "updated_at": "2026-09-09T08:29:00Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CV" ], "abstract": "Video-language models can answer multiple-choice questions with high confidence yet be wrong. We study whether answer-level reliability scores can be improved under target shift without retraining the models or changing their answers. We collect option-probability lists from three fixed video-language models under four deterministic video samplings and represent cross-view changes and cross-model agreement as a response graph. Using a labeled target pilot, we compare the original score, defined as the probability assigned to the chosen answer, with a histogram-based gradient-boosting (HGB) score trained on the development datasets and a regularized logistic-regression score trained on the target pilot. A candidate replaces the original score only when repeated video-level checks indicate a positive, stable improvement. We develop this rule on public VideoQA benchmarks and Video Hallucination Diagnosis (VHD), a controlled diagnostic dataset for shared high-confidence errors. Ranking quality is measured by the area under the risk-coverage curve (AURC), where lower is better. On a held-out 963-question HERBench split, the method reduces mean AURC across the three models by 16.64% (95% confidence interval (CI), 12.12 to 22.61%); the smallest model-level gain is 11.39%. On a separate held-out 911-question Perception Test split, the mean reduction is 18.87% (95% CI, 15.43 to 22.14%). For InternVL3.5, the target check retains the original scores. Using the same outputs, the method outperforms seven training-free baselines in mean AURC on both datasets. It also improves AUROC, reduces calibration error, and lowers the error rate at 50% coverage by 6.50 and 6.58 percentage points.", "url": "https://arxiv.org/abs/2609.13288v1", "pdf_url": "https://arxiv.org/pdf/2609.13288v1", "bibtex_key": "arxiv2609_13288", "included_by": [ "X05_typed_outputs" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Option-probability reliability scores refined under target shift with a labelled pilot, without changing answers (F2).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration", "robustness", "multimodal" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Answer-level reliability scores built from option-probability lists can be refined under target shift, without changing answers, when a labelled target pilot exists.", "claims": [] }, { "id": "arxiv:2609.07305", "arxiv_id": "2609.07305", "versioned_id": "2609.07305v1", "doi": null, "title": "Marginal Fidelity Does Not Establish User Simulation in Demographic Synthetic Survey Panels: Response Contracts, Support Collapse and Conditioning Failure", "short_title": null, "authors": [ "Alexander Doudkin" ], "published_at": "2026-09-07T10:22:22Z", "updated_at": "2026-09-07T10:22:22Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.HC" ], "abstract": "Demographic synthetic survey panels are often validated by matching aggregate answers to published surveys. We test what that certificate establishes across six multiselect batteries from four survey organisations in three countries. The headline analysis is restricted to three instruments whose synthetic cohort and human target share the stated population frame; three other batteries remain sensitivity analyses. The response contract dominates measured fidelity. In the aligned instruments, committed sets leave 66 of 128 model-battery option slots empty in panels of up to 500 respondents, versus 0 of 128 under per-option probability elicitation. Across eight uncapped model-instrument comparisons, probabilities reduce option-marginal MAE by 4.53 to 7.30 points. The capped instrument reverses on two models until the vectors are projected onto its stated maximum. These are measurement effects: human targets are realised check-all responses, whereas the vectors are latent inclusion propensities. Published marginal agreement also fails to discriminate respondent simulation from direct population estimation. On nine aligned model-battery pairs, a no-persona population-prevalence query averages 6.27 MAE versus 12.39 for committed panels and wins all nine comparisons. Constraint-aware probability vectors average 5.34 and beat the query on four of nine, so the baseline challenges the validation criterion rather than proving direct estimation uniformly best. On three unpublished demographic cells, neither approach beats reciting the national distribution. Population-marginal agreement is therefore evidence about an elicitation contract and an estimand obtainable without simulated respondents, not evidence of individual simulation.", "url": "https://arxiv.org/abs/2609.07305v1", "pdf_url": "https://arxiv.org/pdf/2609.07305v1", "bibtex_key": "arxiv2609_07305", "included_by": [ "X05_typed_outputs" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Response contract (committed sets vs per-option probabilities) dominates measured fidelity of synthetic panels; mirrors Noul-vs-Choice interface dependence (F1).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "interface-semantics", "calibration" ], "stages": [ "contract" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "label-robustness", "role": "In synthetic survey panels the response contract (committed sets vs per-option probabilities) dominates measured fidelity — the same interface dependence seen with Noul vs Choice.", "claims": [] }, { "id": "arxiv:2609.04445", "arxiv_id": "2609.04445", "versioned_id": "2609.04445v1", "doi": null, "title": "Conformity Breaks Conformal Prediction", "short_title": null, "authors": [ "Yibo Hu", "Hanyu Su" ], "published_at": "2026-09-03T20:05:47Z", "updated_at": "2026-09-03T20:05:47Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.CL" ], "abstract": "A conformal certificate can be valid when an LLM answers alone and invalid when the same LLM sees peers that unanimously assert a wrong answer. The question is unchanged; the model's score for the correct answer changes. We call this a score-mechanism shift: clean calibration certifies how the model scores answers alone, but not how it scores them under peer pressure. We show that this shift silently breaks conformal prediction in multi-agent LLM systems. Across open-weight models and multiple-choice QA tasks, coverage falls from a calibrated 90% to 74% under unanimous-wrong peers at the standard alpha = 0.10 operating point. The average hides a sharper failure: by targeting the low-confidence items the certificate still covers, an attacker nearly halves coverage on that subgroup, from 87% to 47%, while the monitored average remains much higher. The failure also reaches the decision layer: a system that should escalate when uncertain can instead become confident enough to act on the attacker's wrong answer. Standard conformal fixes do not solve the problem, because the question distribution has not changed; the model's scoring behavior has.", "url": "https://arxiv.org/abs/2609.04445v1", "pdf_url": "https://arxiv.org/pdf/2609.04445v1", "bibtex_key": "arxiv2609_04445", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Conformal coverage calibrated in isolation breaks under score-mechanism shift; attackers can target low-confidence items a gate still covers (F2, F3).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control", "calibration", "robustness" ], "stages": [ "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "selective", "role": "Conformal coverage calibrated in isolation breaks under a score-mechanism shift (peer pressure); an attacker can target the low-confidence items a gate still covers.", "claims": [] }, { "id": "arxiv:2608.20630", "arxiv_id": "2608.20630", "versioned_id": "2608.20630v1", "doi": null, "title": "SAGE: A Unified Algebra and Self-Adaptive Execution for AI Functions in SQL", "short_title": null, "authors": [ "Xiangqi Wang", "Nhan H. Pham", "Oktie Hassanzadeh", "Dharmashankar Subramanian", "Xiangliang Zhang" ], "published_at": "2026-08-21T00:12:41Z", "updated_at": "2026-08-21T00:12:41Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 8, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "SQL systems increasingly expose AI functions for tasks such as classification, extraction, filtering, ranking, retrieval, joining, and summarization. Despite their diverse APIs, these functions play only three relational roles: transforming individual rows, aggregating groups, or generating relationships between row pairs. We present SAGE (Self-Adaptive Generative Execution), a unified logical and physical framework that captures these roles with three typed primitives, AI_SCALAR, AI_AGG, and AI_JOIN, and composes them naturally with standard relational operators. All primitives share a confidence-gated execution interface while supporting physical strategies tailored to their relational shape. The main challenge is AI_JOIN, where SAGE analyzes the predicate, decomposes compound conditions when possible, and uses a recipe card together with a small label-free probe to select among complete execution strategies. Across a broad audit of public AI operators and evaluations spanning scalar, aggregate, and join workloads, this formulation covers common AI functionality while consistently improving execution quality and efficiency. SAGE achieves the strongest overall SemBench performance and, on a representative factorable join, reduces pairwise model calls by more than two orders of magnitude, yielding a 358-fold measured cost reduction.", "url": "https://arxiv.org/abs/2608.20630v1", "pdf_url": "https://arxiv.org/pdf/2608.20630v1", "bibtex_key": "arxiv2608_20630", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Typed AI primitives with a shared confidence-gated execution interface in SQL; the same design pattern arising independently in data systems (F5).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "selective-control" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "Typed AI primitives with a shared confidence-gated execution interface in SQL: the same design pattern arising independently in data systems.", "claims": [] }, { "id": "arxiv:2608.19558", "arxiv_id": "2608.19558", "versioned_id": "2608.19558v2", "doi": null, "title": "Reliable Financial Named Entity Recognition Under Domain Shift: Confidence Estimation and Selective Prediction", "short_title": null, "authors": [ "Zihao Zheng", "Baichuan Li", "Junyi Yao", "Jiayu Long" ], "published_at": "2026-08-20T01:55:41Z", "updated_at": "2026-08-26T04:07:40Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 8, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Financial AI systems often train information extractors on one textual register and deploy them across filings, news, and user-generated content, and standard F1 scores do not indicate which predictions remain safe to automate when that input distribution changes. We study confidence estimation and selective prediction for financial named entity recognition (NER) on a three-tier stress test spanning SEC filings, financial news, and general-topic social media as an extreme out-of-domain condition, evaluating a BERT tagger and LoRA-tuned Qwen2.5-0.5B/1.5B models with five inference-time confidence signals, three training seeds, and bootstrap intervals. Confidence rankings themselves change under shift: whole-output probability is the strongest in-domain error detector but deteriorates out of domain, whereas entity-span probability and self-consistency are more robust; self-consistency is also better calibrated without post-hoc fitting. Abstention reduces sentence error from 34.3% to below 2% on the highest-confidence 40% of in-domain inputs and remains useful on financial news, but recovers no usefully large clean subset under the extreme social-media shift. These results motivate a staged deployment strategy that detects severe distribution shift upstream before applying prediction-level confidence gating.", "url": "https://arxiv.org/abs/2608.19558v2", "pdf_url": "https://arxiv.org/pdf/2608.19558v2", "bibtex_key": "arxiv2608_19558", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Under domain shift the ranking of confidence signals changes; selective prediction must be re-validated per domain (F2).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration", "selective-control", "robustness" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Under domain shift the ranking of confidence signals changes: whole-output probability is the best in-domain error detector but degrades out of domain.", "claims": [] }, { "id": "arxiv:2608.08254", "arxiv_id": "2608.08254", "versioned_id": "2608.08254v1", "doi": null, "title": "Your Prompt Is Not the Only Prompt: How Much Do LLMs Weight Structured-Output Schema Descriptions?", "short_title": null, "authors": [ "Sin-Ying Lin" ], "published_at": "2026-08-08T17:27:22Z", "updated_at": "2026-08-08T17:27:22Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 8, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "Structured output, where an LLM populates a predefined JSON schema, has become a default mechanism for data labeling and information extraction, but it also introduces a second instruction channel through schema descriptions. We tested whether classification-label definitions are better placed in the system prompt, user prompt, or schema description using a single-field classification task with nonce labels across ten model configurations from two vendors. Schema descriptions did not consistently outperform prompt-based placement; for GPT-4.1 and GPT-5.4 without reasoning, schema placement underperformed system prompts by 11-13 percentage points. Yet schemas are not inert metadata: when prompts and schemas conflicted, incorrect schema instructions caused accuracy drops of 5-45 points, with Claude Haiku 4.5 falling from 52.5% to 7%, indicating that schema instructions can override prompt instructions, and GPT-5.5 falling from 100% to 73%. Further, adding a required intermediate reasoning field before the label field improved schema-only accuracy by 15-24 points when headroom existed, exceeding system-prompt-only performance in every case tested. The effect held even for Claude Sonnet 4.6 at medium reasoning, where extended thinking alone did not produce a comparable gain. This suggests that schema design can affect how effectively models use information encoded in field descriptions. Overall, these results indicate that schema influence is model-dependent. In practice, the system prompt remains a safe default for definitions, but the bigger discipline is maintaining a single source of truth and preventing prompt/schema drift. More importantly, schema design itself may be a stronger lever than instruction placement. Practitioners should treat prompts and schemas as a unified instruction surface and empirically validate both placement and field design for their target model.", "url": "https://arxiv.org/abs/2608.08254v1", "pdf_url": "https://arxiv.org/pdf/2608.08254v1", "bibtex_key": "arxiv2608_08254", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "interface-semantics", "structured-output" ], "stages": [ "contract" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "label-robustness", "role": "Schema descriptions act as prompts: interface validity is not semantic compliance.", "claims": [] }, { "id": "arxiv:2608.06571", "arxiv_id": "2608.06571", "versioned_id": "2608.06571v1", "doi": null, "title": "Model Confidence Under Answer-Preserving Attacks: An Informativeness-Manipulability Frontier", "short_title": null, "authors": [ "Reza Khanmohammadi", "Ivan Brugere", "Simerjot Kaur", "Charese H. Smiley", "Kundan Thind", "Mohammad M. Ghassemi" ], "published_at": "2026-08-06T20:27:52Z", "updated_at": "2026-08-06T20:27:52Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 8, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CR", "cs.CL" ], "abstract": "Deployed vision-language systems often gate their answers on confidence, making confidence robustness relevant to oversight. We study confidence readouts under white-box, image-only attacks constrained to preserve the generated answer byte-identically. Under a reachability assumption, an unmovable readout cannot outperform the answer-string accuracy prior, whose pooled value is 0.617. Independently of that assumption, a uniform amplitude certificate below a measurable threshold guarantees adversarial discrimination above the same floor. Across four vision-language models, three visual question answering benchmarks, five deployed confidence channels and two defense estimators, direct or surrogate-aimed attacks produce itemwise feasible perturbations that refute this uniform certificate in all 84 estimator-by-cell combinations. Coordinated correctness-label-aware attacks drive adversarial discrimination to or below the answer-string floor in all sixty deployed-channel cells, including all fifty-nine that begin above it. Hidden-state interventions and an open-ended text-model activation-space replication show that comparable confidence movement can be induced at the representation level rather than only through adversarial images. None of four tested defense families establishes a robust alternative under the specific evaluation applied to it. In a confidence-gated simulation, a coordinated token-probability attack transferred to a hidden-state gate causes up to 84.8% of previously rejected wrong answers to become accepted. After reweighting to each benchmark's natural correctness prevalence, accepted accuracy falls below the no-gate baseline in eight of twelve cells under transfer and all twelve under a direct gate-aimed attack. Under the studied threat model and budget, confidence is therefore an integrity-sensitive rather than intrinsically robust oversight signal.", "url": "https://arxiv.org/abs/2608.06571v1", "pdf_url": "https://arxiv.org/pdf/2608.06571v1", "bibtex_key": "arxiv2608_06571", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Answer-preserving attacks move confidence readouts; bounds what confidence gates can certify (F2).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration", "robustness" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Answer-preserving attacks move confidence readouts, bounding what confidence gates can certify.", "claims": [] }, { "id": "arxiv:2608.04355", "arxiv_id": "2608.04355", "versioned_id": "2608.04355v1", "doi": null, "title": "The Calibration Floor: Format Repair Can Masquerade as Self-Correction at Small-to-Mid Scale", "short_title": null, "authors": [ "Mingguang Chen", "Bo Qu", "Licheng Wang" ], "published_at": "2026-08-05T01:55:33Z", "updated_at": "2026-08-05T01:55:33Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 8, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Accuracy changes after language-model self-revision are usually interpreted as changes in reasoning. We show this can fail at the answer-extraction boundary, and test the failure causally rather than only observationally. Across Qwen3.5 (0.8B-9B), Gemma-4-12B, and two frontier models via API (Tencent Hy3, Nvidia Nemotron-3-Ultra-550B) in 29 primary cells plus a frontier arm, we decompose the always-revise accuracy shift into a content margin (both answers parseable) and format-recovery/loss margins (parseability changes). On 12 cells with meaningful unparseable-answer rates, format effects exceed content effects (Wilcoxon p=1.7e-3). To test this causally, we force already-generated reasoning through grammar-constrained decoding so every answer is parseable by construction: across 14 cells this closes a median 71% of the gap between the naive total effect and the content-margin estimate, with two cells converging exactly and a residual on the two largest-effect cells reported rather than dismissed. A clustered model confirms floor-scale (0.8B/2B) models have far higher odds of content-level change and harm than capable-scale models (p<1e-7). Replicating a cited confidence-gating protocol verbatim on Qwen3.5 does not reproduce its reported gain and shows the same near-zero content margin. A frontier check on much larger models shows format-dominance intensifying with scale: content margin is exactly zero in all 5 cells despite total effects up to +0.275, though this arm is lower-powered. The calibration-floor criterion on the content margin reveals a squeeze: floor-scale cells have headroom but insufficient signal, capable-scale cells have signal but little headroom; only one cell is marginally viable, with negligible sealed-holdout gain. Content is a minority share of what the field has measured as self-correction. We release the instrument, code, and derived results.", "url": "https://arxiv.org/abs/2608.04355v1", "pdf_url": "https://arxiv.org/pdf/2608.04355v1", "bibtex_key": "arxiv2608_04355", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Format recovery can masquerade as reasoning gains; parseability must be separated from content when typed and generative outputs are compared (F1, F7).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "evaluation-methodology" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "Format recovery can masquerade as reasoning gains; grammar-constrained decoding closes much of the gap, so parseability must be separated from content when typed and generative outputs are compared.", "claims": [] }, { "id": "arxiv:2606.30531", "arxiv_id": "2606.30531", "versioned_id": "2606.30531v1", "doi": null, "title": "Entity Binding Failures in Tool-Augmented Agents", "short_title": null, "authors": [ "Rahul Suresh Babu", "Shashank Indukuri" ], "published_at": "2026-06-29T16:34:30Z", "updated_at": "2026-06-29T16:34:30Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 6, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "Tool-augmented language-model agents are often evaluated by whether they select the correct tool, produce valid API arguments, and complete the requested task. However, an agent may choose the right tool and still act on the wrong external entity. For example, a request to \"email Alex about the launch\" may lead the agent to contact the wrong Alex, attach the wrong launch document, reply in the wrong thread, or update the wrong customer account. We call these errors entity binding failures. This paper studies entity binding failures as a distinct reliability and safety problem in tool-augmented agents. We formalize the separation between tool correctness and entity correctness, introduce a taxonomy of wrong-entity failures in enterprise workflows, and evaluate entity-aware execution mechanisms including entity-resolution preconditions, confidence-gated binding, clarification under ambiguity, and provenance tracking. In a controlled diagnostic evaluation across 60 tasks, five model backends, and six tool-use methods, all methods achieved 0.0 percent wrong-tool error, yet action-oriented baselines still produced wrong-entity actions in 24.0-26.0 percent of runs. Entity-aware methods eliminated wrong-entity actions and risk-weighted wrong-entity exposure in this setting, but reduced direct task completion by deferring under ambiguity. These findings show that safe tool use requires not only selecting the correct tool, but also reliably binding natural-language references to the correct real-world entity before action.", "url": "https://arxiv.org/abs/2606.30531v1", "pdf_url": "https://arxiv.org/pdf/2606.30531v1", "bibtex_key": "arxiv2606_30531", "included_by": [ "X06_confidence_gating" ], "inclusion_reason": "Added in increment 2026-09-23T0818Z after title/abstract screening: Entity binding failures separate tool correctness from entity correctness; a close analogue of option-name/rubric binding (F1).", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "interface-semantics", "agents", "selective-control" ], "stages": [ "contract" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "label-robustness", "role": "Entity binding failures — the right tool acting on the wrong entity — separate tool correctness from entity correctness, a close analogue of option-name binding.", "claims": [] }, { "id": "arxiv:2606.22807", "arxiv_id": "2606.22807", "versioned_id": "2606.22807v3", "doi": null, "title": "KaLM-Reranker-V1: Fast but Not Late Interaction for Compressed Document Reranking", "short_title": null, "authors": [ "Xinping Zhao", "Jiaxin Xu", "Ziqi Dai", "Xin Zhang", "Huiyao Chen", "Shouzheng Huang", "Xianhao Xiong", "Danyu Tang", "Xinshuo Hu", "Guohong Fu", "Meishan Zhang", "Baotian Hu" ], "published_at": "2026-06-22T03:36:31Z", "updated_at": "2026-09-22T06:59:06Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 6, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "As retrieval systems scale, effective and efficient reranking becomes increasingly important. However, most existing encoder- and decoder-based rerankers jointly process every query--passage pair, tightly coupling their online computation and limiting deployment efficiency and flexibility. We present KaLM-Reranker-V1, a fast but not late-interaction FBNL reranker that decouples query and passage computation while retaining expressive relevance modeling. Built on an encoder--decoder architecture, KaLM-Reranker-V1 pre-encodes passages using Matryoshka embedding pooling, while its decoder models system and user instructions together with query intent; cross-attention then captures fine-grained relevance between the resulting query context and passage representations. Together, these designs offer four key advantages: (i) efficiency from offline passage encoding, (ii) expressiveness from cross-attention, (iii) compactness from Matryoshka embedding pooling, and (iv) test-time compute through an adjustable compute budget. We instantiate KaLM-Reranker-V1 in three sizes, Nano, Small, and Large, with 0.27B, 1B, and 4B activated parameters, respectively. Extensive experiments on BEIR, MIRACL, and LMEB demonstrate strong reranking performance with superior efficiency. On BEIR and MIRACL, our models achieve competitive performance in multi-domain and multilingual reranking, on par with strong industrial rerankers such as the Qwen3/BGE-Reranker series. On LMEB-Dialogue, a compact embedding model paired with our Nano reranker, which has only 0.27B activated parameters, remains competitive with 7--12B embedding models. Data and models are available at https://huggingface.co/collections/KaLM-Embedding/lychee-kalm-reranker-and-jev.", "url": "https://arxiv.org/abs/2606.22807v3", "pdf_url": "https://arxiv.org/pdf/2606.22807v3", "bibtex_key": "arxiv2606_22807", "included_by": [ "Q01_jev" ], "inclusion_reason": "Found by the snapshot queries and retained as background: KaLM-Reranker-V1，相关 reranker；更新摘要中的 Jev 模型集合链接不使其成为 Jev 原始论文。", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification", "efficiency" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "full_text_spot_check", "background_group": "classification", "role": "KaLM-Reranker-V1: a reranker whose updated abstract links a Jev model collection; that link does not make it a Jev source paper. Full text spot-checked.", "claims": [] }, { "id": "arxiv:2607.20492", "arxiv_id": "2607.20492", "versioned_id": "2607.20492v2", "doi": null, "title": "PhantomFill: When the Form Demands an Answer, Language Models Invent One", "short_title": null, "authors": [ "Rana Muhammad Usman" ], "published_at": "2026-06-11T12:15:05Z", "updated_at": "2026-07-27T11:27:09Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 6, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "cs.CL" ], "abstract": "Language models in production do not write prose. They fill forms: JSON fields, function arguments, extraction templates. We show that the form itself causes hallucination. We ask thirteen models the same question about the same input and change only the answer format. The inputs are built so the question cannot be answered: a viral post showing 12,400 likes but no visible replies, a support ticket whose call was never transcribed. In free text, GPT-5.5 says there is no reply data 98% of the time. Given a required JSON field for sentiment, the same model invents an answer 40 times out of 40. It fabricates the mood of crowds it never saw and quotes customers it never heard. Required fields drive fabrication to 100% in ten of thirteen models. An explicit \"insufficient evidence\" option rescues only the frontier: all nine open-weight models ignore it. Under grammar-constrained decoding, where the escape token is guaranteed reachable by the sampler, five open models spend it zero times out of 203 trials on the three fields that carry the fabrication, and twelve times on the one field where escaping concedes nothing. They can emit the word. They decline to spend it where it costs them an answer. A direct instruction, do not infer sentiment, is overridden by the schema in four of six models. Resistance does not come with scale: within a single model family, the smallest model refuses, the mid-sized model fabricates, the largest refuses again. Honesty under format pressure is a training outcome that no one is measuring. Fabrication hides where hedging is impossible: in required enums and minimum-count arrays, fields where no disclaimer fits. We release PhantomFill, a benchmark with deterministic scoring and two reportable numbers: the Coerced Fabrication Rate and the Escape Utilization Rate. The fix we test is one line of schema. The failure we measure is everywhere.", "url": "https://arxiv.org/abs/2607.20492v2", "pdf_url": "https://arxiv.org/pdf/2607.20492v2", "bibtex_key": "arxiv2607_20492", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "interface-semantics" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "PhantomFill: when a form demands an answer, models invent one — the closed-set risk of forced choices.", "claims": [] }, { "id": "arxiv:2604.25359", "arxiv_id": "2604.25359", "versioned_id": "2604.25359v1", "doi": null, "title": "The Structured Output Benchmark: A Multi-Source Benchmark for Evaluating Structured Output Quality in Large Language Models", "short_title": null, "authors": [ "Abhinav Kumar Singh", "Harsha Vardhan Khurdula", "Yoeven D Khemlani", "Vineet Agarwal" ], "published_at": "2026-04-28T08:27:01Z", "updated_at": "2026-04-28T08:27:01Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 4, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_comment_submitted", "venue_note": "submitted to NeurIPS 2026", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Large Language Models are increasingly being deployed to extract structured data from unstructured and semi-structured sources: parsing invoices, medical records, and converting PDF documents to database entries. Yet existing benchmarks for structured output generation either focus on schema compliance alone, or evaluate value correctness within a single source domain. We introduce SOB (The Structured Output Benchmark), a multi-source benchmark spanning three source modalities: native text, images, and audio conversations. All models receive a text-normalized representation of their context regardless of source modality; this deliberate design isolates structured-output capability from raw vision or speech-processing quality, ensuring a fair, source-agnostic comparison. Our benchmark comprises 5,000 text evaluation records derived from multi-hop QA drawn from a 25,091-record full corpus, 209 image records from OCR-processed PDFs across seven document types including multi-column layouts, dense tables, scanned historical documents, small-print text, and mathematical typesetting, and 115 audio records from the AMI corpus. Each record pairs a natural-language question with a JSON schema that the model must follow and a ground-truth answer verified against the source context. We evaluate 21 frontier and open-weight models across three source domains and seven metrics. Our results reveal a consistent pattern: models achieve near-perfect schema compliance, yet the best Value Accuracy, measured by exact leaf-value match, reaches only 83.0% on text, 67.2% on images, and 23.7% on audio, where longer context makes extraction substantially harder. We release the dataset, evaluation pipeline, and all related code.", "url": "https://arxiv.org/abs/2604.25359v1", "pdf_url": "https://arxiv.org/pdf/2604.25359v1", "bibtex_key": "arxiv2604_25359", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "evaluation-methodology" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "Structured Output Benchmark: broader task coverage for structured-output quality.", "claims": [] }, { "id": "arxiv:2601.07206", "arxiv_id": "2601.07206", "versioned_id": "2601.07206v1", "doi": null, "title": "LLMRouterBench: A Massive Benchmark and Unified Framework for LLM Routing", "short_title": null, "authors": [ "Hao Li", "Yiqun Zhang", "Zhaoyan Guo", "Chenxu Wang", "Shengji Tang", "Qiaosheng Zhang", "Yang Chen", "Biqing Qi", "Peng Ye", "Lei Bai", "Zhen Wang", "Shuyue Hu" ], "published_at": "2026-01-12T05:01:15Z", "updated_at": "2026-01-12T05:01:15Z", "retrieved_at": "2026-09-23", "year": 2026, "month": 1, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "Large language model (LLM) routing assigns each query to the most suitable model from an ensemble. We introduce LLMRouterBench, a large-scale benchmark and unified framework for LLM routing. It comprises over 400K instances from 21 datasets and 33 models. Moreover, it provides comprehensive metrics for both performance-oriented routing and performance-cost trade-off routing, and integrates 10 representative routing baselines. Using LLMRouterBench, we systematically re-evaluate the field. While confirming strong model complementarity-the central premise of LLM routing-we find that many routing methods exhibit similar performance under unified evaluation, and several recent approaches, including commercial routers, fail to reliably outperform a simple baseline. Meanwhile, a substantial gap remains to the Oracle, driven primarily by persistent model-recall failures. We further show that backbone embedding models have limited impact, that larger ensembles exhibit diminishing returns compared to careful model curation, and that the benchmark also enables latency-aware analysis. All code and data are available at https://github.com/ynulihao/LLMRouterBench.", "url": "https://arxiv.org/abs/2601.07206v1", "pdf_url": "https://arxiv.org/pdf/2601.07206v1", "bibtex_key": "arxiv2601_07206", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "routing", "evaluation-methodology" ], "stages": [ "control", "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "routing", "role": "LLMRouterBench: a unified task set and protocol for routing evaluation.", "claims": [] }, { "id": "arxiv:2511.13699", "arxiv_id": "2511.13699", "versioned_id": "2511.13699v1", "doi": null, "title": "Efficient Calibration for Decision Making", "short_title": null, "authors": [ "Parikshit Gopalan", "Konstantinos Stavropoulos", "Kunal Talwar", "Pranay Tankala" ], "published_at": "2025-11-17T18:52:00Z", "updated_at": "2025-11-17T18:52:00Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 11, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.DS", "stat.ML" ], "abstract": "A decision-theoretic characterization of perfect calibration is that an agent seeking to minimize a proper loss in expectation cannot improve their outcome by post-processing a perfectly calibrated predictor. Hu and Wu (FOCS'24) use this to define an approximate calibration measure called calibration decision loss ($\\mathsf{CDL}$), which measures the maximal improvement achievable by any post-processing over any proper loss. Unfortunately, $\\mathsf{CDL}$ turns out to be intractable to even weakly approximate in the offline setting, given black-box access to the predictions and labels. We suggest circumventing this by restricting attention to structured families of post-processing functions $K$. We define the calibration decision loss relative to $K$, denoted $\\mathsf{CDL}_K$ where we consider all proper losses but restrict post-processings to a structured family $K$. We develop a comprehensive theory of when $\\mathsf{CDL}_K$ is information-theoretically and computationally tractable, and use it to prove both upper and lower bounds for natural classes $K$. In addition to introducing new definitions and algorithmic techniques to the theory of calibration for decision making, our results give rigorous guarantees for some widely used recalibration procedures in machine learning.", "url": "https://arxiv.org/abs/2511.13699v1", "pdf_url": "https://arxiv.org/pdf/2511.13699v1", "bibtex_key": "arxiv2511_13699", "included_by": [ "Q05_calibrated_decisions" ], "inclusion_reason": "Found by the snapshot queries and retained as background: 高效的决策导向校准；构造强后处理基线。", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability", "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "decision-calibration", "role": "Efficient decision-oriented calibration: a strong post-processing baseline.", "claims": [] }, { "id": "arxiv:2510.07750", "arxiv_id": "2510.07750", "versioned_id": "2510.07750v3", "doi": null, "title": "Calibrating Decision Robustness via Inverse Conformal Risk Control", "short_title": null, "authors": [ "Wenbin Zhou", "Shixiang Zhu" ], "published_at": "2025-10-09T03:38:17Z", "updated_at": "2026-06-10T02:44:06Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 10, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "stat.ML", "cs.LG" ], "abstract": "Robust optimization safeguards decisions against uncertainty by optimizing against worst-case scenarios, yet their effectiveness hinges on a prespecified robustness level that is often chosen ad hoc, leading to either insufficient protection or overly conservative and costly solutions. Recent approaches using conformal prediction construct data-driven uncertainty sets with finite-sample coverage guarantees, but they still fix coverage targets a priori and offer little guidance for selecting robustness levels. We propose a new framework that provides distribution-free, finite-sample guarantees on both miscoverage and regret for any family of robust predict-then-optimize policies. Our method constructs valid estimators that trace out the miscoverage--regret Pareto frontier, enabling decision-makers to reliably evaluate and calibrate robustness levels according to their cost--risk preferences. The framework is simple to implement, broadly applicable across classical optimization formulations, and achieves sharper finite-sample performance. This paper offers a principled data-driven methodology for guiding robustness selection and empowers practitioners to balance robustness and conservativeness in high-stakes decision-making.", "url": "https://arxiv.org/abs/2510.07750v3", "pdf_url": "https://arxiv.org/pdf/2510.07750v3", "bibtex_key": "arxiv2510_07750", "included_by": [ "Q05_calibrated_decisions" ], "inclusion_reason": "Found by the snapshot queries and retained as background: 逆 conformal risk control 与决策鲁棒性。", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration", "robustness" ], "stages": [ "probability", "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "decision-calibration", "role": "Inverse conformal risk control and decision robustness.", "claims": [] }, { "id": "arxiv:2510.01237", "arxiv_id": "2510.01237", "versioned_id": "2510.01237v1", "doi": null, "title": "Confidence-Aware Routing for Large Language Model Reliability Enhancement: A Multi-Signal Approach to Pre-Generation Hallucination Mitigation", "short_title": null, "authors": [ "Nandakishor M" ], "published_at": "2025-09-23T18:34:20Z", "updated_at": "2025-09-23T18:34:20Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Large Language Models suffer from hallucination, generating plausible yet factually incorrect content. Current mitigation strategies focus on post-generation correction, which is computationally expensive and fails to prevent unreliable content generation. We propose a confidence-aware routing system that proactively assesses model uncertainty before generation and redirects queries based on estimated reliability. Our approach combines three complementary signals: semantic alignment between internal representations and reference embeddings, internal convergence analysis across model layers, and learned confidence estimation. The unified confidence score determines routing to four pathways: local generation for high confidence, retrieval-augmented generation for medium confidence, larger models for low confidence, and human review for very low confidence. Evaluation on knowledge-intensive QA benchmarks demonstrates significant improvements in hallucination detection (0.74 vs. 0.42 baseline) while reducing computational costs by 40% compared to post-hoc methods. The F1 score improves from 0.61 to 0.82 with low false positive rates (0.09). This paradigm shift from reactive correction to proactive assessment offers a computationally efficient approach to LLM reliability enhancement.", "url": "https://arxiv.org/abs/2510.01237v1", "pdf_url": "https://arxiv.org/pdf/2510.01237v1", "bibtex_key": "arxiv2510_01237", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "routing", "calibration" ], "stages": [ "control", "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "routing", "role": "Confidence-aware routing: combining several signals to decide before generation whether to answer or route.", "claims": [] }, { "id": "arxiv:2508.07662", "arxiv_id": "2508.07662", "versioned_id": "2508.07662v1", "doi": null, "title": "GLiClass: Generalist Lightweight Model for Sequence Classification Tasks", "short_title": null, "authors": [ "Ihor Stepanov", "Mykhailo Shtopko", "Dmytro Vodianytskyi", "Oleksandr Lukashov", "Alexander Yavorskyi", "Mykyta Yaroshenko" ], "published_at": "2025-08-11T06:22:25Z", "updated_at": "2025-08-11T06:22:25Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 8, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "cs.CL" ], "abstract": "Classification is one of the most widespread tasks in AI applications, serving often as the first step in filtering, sorting, and categorizing data. Since modern AI systems must handle large volumes of input data and early pipeline stages can propagate errors downstream, achieving high efficiency and accuracy is critical. Moreover, classification requirements can change dynamically based on user needs, necessitating models with strong zero-shot capabilities. While generative LLMs have become mainstream for zero-shot classification due to their versatility, they suffer from inconsistent instruction following and computational inefficiency. Cross-encoders, commonly used as rerankers in RAG pipelines, face a different bottleneck: they must process text-label pairs sequentially, significantly reducing efficiency with large label sets. Embedding-based approaches offer good efficiency but struggle with complex scenarios involving logical and semantic constraints. We propose GLiClass, a novel method that adapts the GLiNER architecture for sequence classification tasks. Our approach achieves strong accuracy and efficiency comparable to embedding-based methods, while maintaining the flexibility needed for zero-shot and few-shot learning scenarios. Additionally, we adapted proximal policy optimization (PPO) for multi-label text classification, enabling training classifiers in data-sparse conditions or from human feedback.", "url": "https://arxiv.org/abs/2508.07662v1", "pdf_url": "https://arxiv.org/pdf/2508.07662v1", "bibtex_key": "arxiv2508_07662", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification", "open-models" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "GLiClass: a general lightweight sequence classifier and a direct adjacent baseline for typed decisions.", "claims": [] }, { "id": "arxiv:2504.15582", "arxiv_id": "2504.15582", "versioned_id": "2504.15582v1", "doi": null, "title": "Smooth Calibration and Decision Making", "short_title": null, "authors": [ "Jason Hartline", "Yifan Wu", "Yunran Yang" ], "published_at": "2025-04-22T04:55:41Z", "updated_at": "2025-04-22T04:55:41Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 4, "tier": "background", "publication_type": "conference", "venue": "FORC 2025", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.LG", "cs.DS", "stat.ML" ], "abstract": "Calibration requires predictor outputs to be consistent with their Bayesian posteriors. For machine learning predictors that do not distinguish between small perturbations, calibration errors are continuous in predictions, e.g., smooth calibration error (Foster and Hart, 2018), Distance to Calibration (Blasiok et al., 2023a). On the contrary, decision-makers who use predictions make optimal decisions discontinuously in probabilistic space, experiencing loss from miscalibration discontinuously. Calibration errors for decision-making are thus discontinuous, e.g., Expected Calibration Error (Foster and Vohra, 1997), and Calibration Decision Loss (Hu and Wu, 2024). Thus, predictors with a low calibration error for machine learning may suffer a high calibration error for decision-making, i.e., they may not be trustworthy for decision-makers optimizing assuming their predictions are correct. It is natural to ask if post-processing a predictor with a low calibration error for machine learning is without loss to achieve a low calibration error for decision-making. In our paper, we show that post-processing an online predictor with $ε$ distance to calibration achieves $O(\\sqrtε)$ ECE and CDL, which is asymptotically optimal. The post-processing algorithm adds noise to make predictions differentially private. The optimal bound from low distance to calibration predictors from post-processing is non-optimal compared with existing online calibration algorithms that directly optimize for ECE and CDL.", "url": "https://arxiv.org/abs/2504.15582v1", "pdf_url": "https://arxiv.org/pdf/2504.15582v1", "bibtex_key": "arxiv2504_15582", "included_by": [ "Q05_calibrated_decisions" ], "inclusion_reason": "Found by the snapshot queries and retained as background: 平滑校准和决策之间的联系。", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability", "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "decision-calibration", "role": "Smooth calibration and decision making.", "claims": [] }, { "id": "arxiv:2503.24377", "arxiv_id": "2503.24377", "versioned_id": "2503.24377v3", "doi": null, "title": "Harnessing the Reasoning Economy: A Survey of Efficient Reasoning for Large Language Models", "short_title": null, "authors": [ "Rui Wang", "Hongru Wang", "Boyang Xue", "Yixia Li", "Jianhui Pang", "Shudong Liu", "Yi Chen", "Jiahao Qiu", "Derek Fai Wong", "Guanhua Chen", "Heng Ji", "Kam-Fai Wong" ], "published_at": "2025-03-31T17:58:07Z", "updated_at": "2026-09-08T03:16:17Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 3, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Recent advancements in Large Language Models (LLMs) have significantly enhanced their ability to perform complex reasoning tasks, transitioning from fast and intuitive thinking (System 1) to slow and deep reasoning (System 2). While System 2 reasoning improves task accuracy, it often incurs substantial computational costs due to its slow thinking nature and inefficient or unnecessary reasoning behaviors. In contrast, System 1 reasoning is computationally efficient but leads to suboptimal performance. Consequently, it is critical to balance the trade-off between performance (benefits) and computational costs (budgets), giving rise to the concept of reasoning economy. In this survey, we provide a comprehensive analysis of reasoning economy in both the post-training and test-time inference stages of LLMs, encompassing i) the cause of reasoning inefficiency, ii) behavior analysis of different reasoning patterns, and iii) potential solutions to achieve reasoning economy. By offering actionable insights and highlighting open challenges, we aim to shed light on strategies for improving the reasoning economy of LLMs, thereby serving as a valuable resource for advancing research in this evolving area. We also provide a public repository to continually track developments in this fast-evolving field.", "url": "https://arxiv.org/abs/2503.24377v3", "pdf_url": "https://arxiv.org/pdf/2503.24377v3", "bibtex_key": "arxiv2503_24377", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "routing", "efficiency" ], "stages": [ "control", "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "routing", "role": "Reasoning economy: compare inference budgets, not only parameter counts.", "claims": [] }, { "id": "arxiv:2503.23303", "arxiv_id": "2503.23303", "versioned_id": "2503.23303v1", "doi": null, "title": "SalesRLAgent: A Reinforcement Learning Approach for Real-Time Sales Conversion Prediction and Optimization", "short_title": null, "authors": [ "Nandakishor M" ], "published_at": "2025-03-30T03:56:26Z", "updated_at": "2025-03-30T03:56:26Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 3, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI" ], "abstract": "Current approaches to sales conversation analysis and conversion prediction typically rely on Large Language Models (LLMs) combined with basic retrieval augmented generation (RAG). These systems, while capable of answering questions, fail to accurately predict conversion probability or provide strategic guidance in real time. In this paper, we present SalesRLAgent, a novel framework leveraging specialized reinforcement learning to predict conversion probability throughout sales conversations. Unlike systems from Kapa.ai, Mendable, Inkeep, and others that primarily use off-the-shelf LLMs for content generation, our approach treats conversion prediction as a sequential decision problem, training on synthetic data generated using GPT-4O to develop a specialized probability estimation model. Our system incorporates Azure OpenAI embeddings (3072 dimensions), turn-by-turn state tracking, and meta-learning capabilities to understand its own knowledge boundaries. Evaluations demonstrate that SalesRLAgent achieves 96.7% accuracy in conversion prediction, outperforming LLM-only approaches by 34.7% while offering significantly faster inference (85ms vs 3450ms for GPT-4). Furthermore, integration with existing sales platforms shows a 43.2% increase in conversion rates when representatives utilize our system's real-time guidance. SalesRLAgent represents a fundamental shift from content generation to strategic sales intelligence, providing moment-by-moment conversion probability estimation with actionable insights for sales professionals.", "url": "https://arxiv.org/abs/2503.23303v1", "pdf_url": "https://arxiv.org/pdf/2503.23303v1", "bibtex_key": "arxiv2503_23303", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification", "calibration" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "SalesRLAgent (2025): probabilistic decisions in a sales domain; historically related work.", "claims": [] }, { "id": "arxiv:2503.15850", "arxiv_id": "2503.15850", "versioned_id": "2503.15850v2", "doi": null, "title": "Uncertainty Quantification and Confidence Calibration in Large Language Models: A Survey", "short_title": null, "authors": [ "Xiaoou Liu", "Tiejin Chen", "Longchao Da", "Chacha Chen", "Zhen Lin", "Hua Wei" ], "published_at": "2025-03-20T05:04:29Z", "updated_at": "2025-06-03T22:16:32Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 3, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Large Language Models (LLMs) excel in text generation, reasoning, and decision-making, enabling their adoption in high-stakes domains such as healthcare, law, and transportation. However, their reliability is a major concern, as they often produce plausible but incorrect responses. Uncertainty quantification (UQ) enhances trustworthiness by estimating confidence in outputs, enabling risk mitigation and selective prediction. However, traditional UQ methods struggle with LLMs due to computational constraints and decoding inconsistencies. Moreover, LLMs introduce unique uncertainty sources, such as input ambiguity, reasoning path divergence, and decoding stochasticity, that extend beyond classical aleatoric and epistemic uncertainty. To address this, we introduce a new taxonomy that categorizes UQ methods based on computational efficiency and uncertainty dimensions (input, reasoning, parameter, and prediction uncertainty). We evaluate existing techniques, assess their real-world applicability, and identify open challenges, emphasizing the need for scalable, interpretable, and robust UQ approaches to enhance LLM reliability.", "url": "https://arxiv.org/abs/2503.15850v2", "pdf_url": "https://arxiv.org/pdf/2503.15850v2", "bibtex_key": "arxiv2503_15850", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Survey of uncertainty quantification and calibration in LLMs.", "claims": [] }, { "id": "arxiv:2503.02623", "arxiv_id": "2503.02623", "versioned_id": "2503.02623v6", "doi": null, "title": "Rewarding Doubt: A Reinforcement Learning Approach to Calibrated Confidence Expression of Large Language Models", "short_title": null, "authors": [ "David Bani-Harouni", "Chantal Pellegrini", "Paul Stangel", "Ege Özsoy", "Kamilia Zaripova", "Nassir Navab", "Matthias Keicher" ], "published_at": "2025-03-04T13:48:50Z", "updated_at": "2026-02-28T17:29:41Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 3, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "A safe and trustworthy use of Large Language Models (LLMs) requires an accurate expression of confidence in their answers. We propose a novel Reinforcement Learning approach that allows to directly fine-tune LLMs to express calibrated confidence estimates alongside their answers to factual questions. Our method optimizes a reward based on the logarithmic scoring rule, explicitly penalizing both over- and under-confidence. This encourages the model to align its confidence estimates with the actual predictive accuracy. The optimal policy under our reward design would result in perfectly calibrated confidence expressions. Unlike prior approaches that decouple confidence estimation from response generation, our method integrates confidence calibration seamlessly into the generative process of the LLM. Empirically, we demonstrate that models trained with our approach exhibit substantially improved calibration and generalize to unseen tasks without further fine-tuning, suggesting the emergence of general confidence awareness.", "url": "https://arxiv.org/abs/2503.02623v6", "pdf_url": "https://arxiv.org/pdf/2503.02623v6", "bibtex_key": "arxiv2503_02623", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Rewarding Doubt: reinforcement learning for calibrated confidence expression.", "claims": [] }, { "id": "arxiv:2502.17419", "arxiv_id": "2502.17419", "versioned_id": "2502.17419v6", "doi": null, "title": "From System 1 to System 2: A Survey of Reasoning Large Language Models", "short_title": null, "authors": [ "Zhong-Zhi Li", "Duzhen Zhang", "Ming-Liang Zhang", "Jiaxin Zhang", "Zengyan Liu", "Yuxuan Yao", "Haotian Xu", "Junhao Zheng", "Pei-Jie Wang", "Xiuyi Chen", "Yingying Zhang", "Fei Yin", "Jiahua Dong", "Zhiwei Li", "Bao-Long Bi", "Ling-Rui Mei", "Junfeng Fang", "Xiao Liang", "Zhijiang Guo", "Le Song", "Cheng-Lin Liu" ], "published_at": "2025-02-24T18:50:52Z", "updated_at": "2025-06-25T02:24:46Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 2, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI" ], "abstract": "Achieving human-level intelligence requires refining the transition from the fast, intuitive System 1 to the slower, more deliberate System 2 reasoning. While System 1 excels in quick, heuristic decisions, System 2 relies on logical reasoning for more accurate judgments and reduced biases. Foundational Large Language Models (LLMs) excel at fast decision-making but lack the depth for complex reasoning, as they have not yet fully embraced the step-by-step analysis characteristic of true System 2 thinking. Recently, reasoning LLMs like OpenAI's o1/o3 and DeepSeek's R1 have demonstrated expert-level performance in fields such as mathematics and coding, closely mimicking the deliberate reasoning of System 2 and showcasing human-like cognitive abilities. This survey begins with a brief overview of the progress in foundational LLMs and the early development of System 2 technologies, exploring how their combination has paved the way for reasoning LLMs. Next, we discuss how to construct reasoning LLMs, analyzing their features, the core methods enabling advanced reasoning, and the evolution of various reasoning LLMs. Additionally, we provide an overview of reasoning benchmarks, offering an in-depth comparison of the performance of representative reasoning LLMs. Finally, we explore promising directions for advancing reasoning LLMs and maintain a real-time \\href{https://github.com/zzli2022/Awesome-Slow-Reason-System}{GitHub Repository} to track the latest developments. We hope this survey will serve as a valuable resource to inspire innovation and drive progress in this rapidly evolving field.", "url": "https://arxiv.org/abs/2502.17419v6", "pdf_url": "https://arxiv.org/pdf/2502.17419v6", "bibtex_key": "arxiv2502_17419", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "routing" ], "stages": [ "control", "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "routing", "role": "From System 1 to System 2: a reasoning-LLM survey whose terminology predates and is broader than the product.", "claims": [] }, { "id": "arxiv:2501.10868", "arxiv_id": "2501.10868", "versioned_id": "2501.10868v3", "doi": null, "title": "JSONSchemaBench: A Rigorous Benchmark of Structured Outputs for Language Models", "short_title": null, "authors": [ "Saibo Geng", "Hudson Cooper", "Michał Moskal", "Samuel Jenkins", "Julian Berman", "Nathan Ranchin", "Robert West", "Eric Horvitz", "Harsha Nori" ], "published_at": "2025-01-18T20:26:00Z", "updated_at": "2025-02-27T15:37:21Z", "retrieved_at": "2026-09-23", "year": 2025, "month": 1, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Reliably generating structured outputs has become a critical capability for modern language model (LM) applications. Constrained decoding has emerged as the dominant technology across sectors for enforcing structured outputs during generation. Despite its growing adoption, little has been done with the systematic evaluation of the behaviors and performance of constrained decoding. Constrained decoding frameworks have standardized around JSON Schema as a structured data format, with most uses guaranteeing constraint compliance given a schema. However, there is poor understanding of the effectiveness of the methods in practice. We present an evaluation framework to assess constrained decoding approaches across three critical dimensions: efficiency in generating constraint-compliant outputs, coverage of diverse constraint types, and quality of the generated outputs. To facilitate this evaluation, we introduce JSONSchemaBench, a benchmark for constrained decoding comprising 10K real-world JSON schemas that encompass a wide range of constraints with varying complexity. We pair the benchmark with the existing official JSON Schema Test Suite and evaluate six state-of-the-art constrained decoding frameworks, including Guidance, Outlines, Llamacpp, XGrammar, OpenAI, and Gemini. Through extensive experiments, we gain insights into the capabilities and limitations of constrained decoding on structured generation with real-world JSON schemas. Our work provides actionable insights for improving constrained decoding frameworks and structured generation tasks, setting a new standard for evaluating constrained decoding and structured generation. We release JSONSchemaBench at https://github.com/guidance-ai/jsonschemabench", "url": "https://arxiv.org/abs/2501.10868v3", "pdf_url": "https://arxiv.org/pdf/2501.10868v3", "bibtex_key": "arxiv2501_10868", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "evaluation-methodology" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "JSONSchemaBench: structural correctness and efficiency of constrained decoding (title updated on arXiv).", "claims": [] }, { "id": "arxiv:2412.13663", "arxiv_id": "2412.13663", "versioned_id": "2412.13663v2", "doi": null, "title": "Smarter, Better, Faster, Longer: A Modern Bidirectional Encoder for Fast, Memory Efficient, and Long Context Finetuning and Inference", "short_title": null, "authors": [ "Benjamin Warner", "Antoine Chaffin", "Benjamin Clavié", "Orion Weller", "Oskar Hallström", "Said Taghadouini", "Alexis Gallagher", "Raja Biswas", "Faisal Ladhak", "Tom Aarsen", "Nathan Cooper", "Griffin Adams", "Jeremy Howard", "Iacopo Poli" ], "published_at": "2024-12-18T09:39:44Z", "updated_at": "2024-12-19T06:32:26Z", "retrieved_at": "2026-09-23", "year": 2024, "month": 12, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Encoder-only transformer models such as BERT offer a great performance-size tradeoff for retrieval and classification tasks with respect to larger decoder-only models. Despite being the workhorse of numerous production pipelines, there have been limited Pareto improvements to BERT since its release. In this paper, we introduce ModernBERT, bringing modern model optimizations to encoder-only models and representing a major Pareto improvement over older encoders. Trained on 2 trillion tokens with a native 8192 sequence length, ModernBERT models exhibit state-of-the-art results on a large pool of evaluations encompassing diverse classification tasks and both single and multi-vector retrieval on different domains (including code). In addition to strong downstream performance, ModernBERT is also the most speed and memory efficient encoder and is designed for inference on common GPUs.", "url": "https://arxiv.org/abs/2412.13663v2", "pdf_url": "https://arxiv.org/pdf/2412.13663v2", "bibtex_key": "arxiv2412_13663", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification", "open-models" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "ModernBERT: the backbone of several encoder-style open alternatives and a strong lightweight baseline.", "claims": [] }, { "id": "arxiv:2411.15100", "arxiv_id": "2411.15100", "versioned_id": "2411.15100v3", "doi": null, "title": "XGrammar: Flexible and Efficient Structured Generation Engine for Large Language Models", "short_title": null, "authors": [ "Yixin Dong", "Charlie F. Ruan", "Yaxing Cai", "Ruihang Lai", "Ziyi Xu", "Yilong Zhao", "Tianqi Chen" ], "published_at": "2024-11-22T18:01:37Z", "updated_at": "2025-05-12T08:20:08Z", "retrieved_at": "2026-09-23", "year": 2024, "month": 11, "tier": "background", "publication_type": "conference", "venue": "MLSys 2025", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI", "cs.PL" ], "abstract": "The applications of LLM Agents are becoming increasingly complex and diverse, leading to a high demand for structured outputs that can be parsed into code, structured function calls, and embodied agent commands. These developments bring significant demands for structured generation in LLM inference. Context-free grammar is a flexible approach to enable structured generation via constrained decoding. However, executing context-free grammar requires going through several stack states over all tokens in vocabulary during runtime, bringing non-negligible overhead for structured generation. In this paper, we propose XGrammar, a flexible and efficient structure generation engine for large language models. XGrammar accelerates context-free grammar execution by dividing the vocabulary into context-independent tokens that can be prechecked and context-dependent tokens that need to be interpreted during runtime. We further build transformations to expand the grammar context and reduce the number of context-independent tokens. Additionally, we build an efficient persistent stack to accelerate the context-dependent token checks. Finally, we co-design the grammar engine with LLM inference engine to overlap grammar computation with GPU executions. Evaluation results show that XGrammar can achieve up to 100x speedup over existing solutions. Combined with an LLM inference engine, it can generate near-zero overhead structure generation in end-to-end low-LLM serving.", "url": "https://arxiv.org/abs/2411.15100v3", "pdf_url": "https://arxiv.org/pdf/2411.15100v3", "bibtex_key": "arxiv2411_15100", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "efficiency" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "XGrammar: an efficient constrained-decoding engine for fair generative baselines.", "claims": [] }, { "id": "arxiv:2406.18665", "arxiv_id": "2406.18665", "versioned_id": "2406.18665v4", "doi": null, "title": "RouteLLM: Learning to Route LLMs with Preference Data", "short_title": null, "authors": [ "Isaac Ong", "Amjad Almahairi", "Vincent Wu", "Wei-Lin Chiang", "Tianhao Wu", "Joseph E. Gonzalez", "M Waleed Kadous", "Ion Stoica" ], "published_at": "2024-06-26T18:10:22Z", "updated_at": "2025-02-23T08:50:33Z", "retrieved_at": "2026-09-23", "year": 2024, "month": 6, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "cs.CL" ], "abstract": "Large language models (LLMs) exhibit impressive capabilities across a wide range of tasks, yet the choice of which model to use often involves a trade-off between performance and cost. More powerful models, though effective, come with higher expenses, while less capable models are more cost-effective. To address this dilemma, we propose several efficient router models that dynamically select between a stronger and a weaker LLM during inference, aiming to optimize the balance between cost and response quality. We develop a training framework for these routers leveraging human preference data and data augmentation techniques to enhance performance. Our evaluation on widely-recognized benchmarks shows that our approach significantly reduces costs-by over 2 times in certain cases-without compromising the quality of responses. Interestingly, our router models also demonstrate significant transfer learning capabilities, maintaining their performance even when the strong and weak models are changed at test time. This highlights the potential of these routers to provide a cost-effective yet high-performance solution for deploying LLMs.", "url": "https://arxiv.org/abs/2406.18665v4", "pdf_url": "https://arxiv.org/pdf/2406.18665v4", "bibtex_key": "arxiv2406_18665", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "routing" ], "stages": [ "control", "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "routing", "role": "RouteLLM: a learned-routing baseline.", "claims": [] }, { "id": "arxiv:2404.13503", "arxiv_id": "2404.13503", "versioned_id": "2404.13503v5", "doi": null, "title": "Calibration Error for Decision Making", "short_title": null, "authors": [ "Lunjia Hu", "Yifan Wu" ], "published_at": "2024-04-21T01:53:20Z", "updated_at": "2024-10-10T22:15:26Z", "retrieved_at": "2026-09-23", "year": 2024, "month": 4, "tier": "background", "publication_type": "conference", "venue": "FOCS 2024", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.LG", "cs.DS", "stat.ML" ], "abstract": "Calibration allows predictions to be reliably interpreted as probabilities by decision makers. We propose a decision-theoretic calibration error, the Calibration Decision Loss (CDL), defined as the maximum improvement in decision payoff obtained by calibrating the predictions, where the maximum is over all payoff-bounded decision tasks. Vanishing CDL guarantees the payoff loss from miscalibration vanishes simultaneously for all downstream decision tasks. We show separations between CDL and existing calibration error metrics, including the most well-studied metric Expected Calibration Error (ECE). Our main technical contribution is a new efficient algorithm for online calibration that achieves near-optimal $O(\\frac{\\log T}{\\sqrt{T}})$ expected CDL, bypassing the $Ω(T^{-0.472})$ lower bound for ECE by Qiao and Valiant (2021).", "url": "https://arxiv.org/abs/2404.13503v5", "pdf_url": "https://arxiv.org/pdf/2404.13503v5", "bibtex_key": "arxiv2404_13503", "included_by": [ "Q05_calibrated_decisions" ], "inclusion_reason": "Found by the snapshot queries and retained as background: 面向下游决策的校准误差；比单一 ECE 更贴近效用。", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability", "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "decision-calibration", "role": "Calibration error for decision making: a utility-aware alternative to a single ECE.", "claims": [] }, { "id": "arxiv:2403.13787", "arxiv_id": "2403.13787", "versioned_id": "2403.13787v2", "doi": null, "title": "RewardBench: Evaluating Reward Models for Language Modeling", "short_title": null, "authors": [ "Nathan Lambert", "Valentina Pyatkin", "Jacob Morrison", "LJ Miranda", "Bill Yuchen Lin", "Khyathi Chandu", "Nouha Dziri", "Sachin Kumar", "Tom Zick", "Yejin Choi", "Noah A. Smith", "Hannaneh Hajishirzi" ], "published_at": "2024-03-20T17:49:54Z", "updated_at": "2024-06-08T16:40:12Z", "retrieved_at": "2026-09-23", "year": 2024, "month": 3, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG" ], "abstract": "Reward models (RMs) are at the crux of successfully using RLHF to align pretrained models to human preferences, yet there has been relatively little study that focuses on evaluation of those models. Evaluating reward models presents an opportunity to understand the opaque technologies used for alignment of language models and which values are embedded in them. Resources for reward model training and understanding are sparse in the nascent open-source community around them. To enhance scientific understanding of reward models, we present RewardBench, a benchmark dataset and code-base for evaluation. The RewardBench dataset is a collection of prompt-chosen-rejected trios spanning chat, reasoning, and safety, to benchmark how reward models perform on challenging, structured and out-of-distribution queries. We create specific comparison datasets for RMs that have subtle, but verifiable reasons (e.g. bugs, incorrect facts) why one answer should be preferred to another. On the RewardBench leaderboard, we evaluate reward models trained with a variety of methods, such as the direct MLE training of classifiers and the implicit reward modeling of Direct Preference Optimization (DPO). We present many findings on propensity for refusals, reasoning limitations, and instruction following shortcomings of various reward models towards a better understanding of the RLHF process.", "url": "https://arxiv.org/abs/2403.13787v2", "pdf_url": "https://arxiv.org/pdf/2403.13787v2", "bibtex_key": "arxiv2403_13787", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "judging" ], "stages": [ "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "judging", "role": "RewardBench: discriminative reward models as strong judging comparators.", "claims": [] }, { "id": "arxiv:2312.07104", "arxiv_id": "2312.07104", "versioned_id": "2312.07104v2", "doi": null, "title": "SGLang: Efficient Execution of Structured Language Model Programs", "short_title": null, "authors": [ "Lianmin Zheng", "Liangsheng Yin", "Zhiqiang Xie", "Chuyue Sun", "Jeff Huang", "Cody Hao Yu", "Shiyi Cao", "Christos Kozyrakis", "Ion Stoica", "Joseph E. Gonzalez", "Clark Barrett", "Ying Sheng" ], "published_at": "2023-12-12T09:34:27Z", "updated_at": "2024-06-06T00:10:06Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 12, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.AI", "cs.PL" ], "abstract": "Large language models (LLMs) are increasingly used for complex tasks that require multiple generation calls, advanced prompting techniques, control flow, and structured inputs/outputs. However, efficient systems are lacking for programming and executing these applications. We introduce SGLang, a system for efficient execution of complex language model programs. SGLang consists of a frontend language and a runtime. The frontend simplifies programming with primitives for generation and parallelism control. The runtime accelerates execution with novel optimizations like RadixAttention for KV cache reuse and compressed finite state machines for faster structured output decoding. Experiments show that SGLang achieves up to 6.4x higher throughput compared to state-of-the-art inference systems on various large language and multi-modal models on tasks including agent control, logical reasoning, few-shot learning benchmarks, JSON decoding, retrieval-augmented generation pipelines, and multi-turn chat. The code is publicly available at https://github.com/sgl-project/sglang", "url": "https://arxiv.org/abs/2312.07104v2", "pdf_url": "https://arxiv.org/pdf/2312.07104v2", "bibtex_key": "arxiv2312_07104", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output", "efficiency" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "SGLang: shared-prefix reuse and scheduling, whose gains must be separated from decision-head gains.", "claims": [] }, { "id": "arxiv:2311.08526", "arxiv_id": "2311.08526", "versioned_id": "2311.08526v1", "doi": null, "title": "GLiNER: Generalist Model for Named Entity Recognition using Bidirectional Transformer", "short_title": null, "authors": [ "Urchade Zaratiana", "Nadi Tomeh", "Pierre Holat", "Thierry Charnois" ], "published_at": "2023-11-14T20:39:12Z", "updated_at": "2023-11-14T20:39:12Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 11, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI", "cs.LG" ], "abstract": "Named Entity Recognition (NER) is essential in various Natural Language Processing (NLP) applications. Traditional NER models are effective but limited to a set of predefined entity types. In contrast, Large Language Models (LLMs) can extract arbitrary entities through natural language instructions, offering greater flexibility. However, their size and cost, particularly for those accessed via APIs like ChatGPT, make them impractical in resource-limited scenarios. In this paper, we introduce a compact NER model trained to identify any type of entity. Leveraging a bidirectional transformer encoder, our model, GLiNER, facilitates parallel entity extraction, an advantage over the slow sequential token generation of LLMs. Through comprehensive testing, GLiNER demonstrate strong performance, outperforming both ChatGPT and fine-tuned LLMs in zero-shot evaluations on various NER benchmarks.", "url": "https://arxiv.org/abs/2311.08526v1", "pdf_url": "https://arxiv.org/pdf/2311.08526v1", "bibtex_key": "arxiv2311_08526", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "GLiNER: open-type extraction with a bidirectional encoder; related to finite-candidate decisions but a different output task.", "claims": [] }, { "id": "arxiv:2310.11324", "arxiv_id": "2310.11324", "versioned_id": "2310.11324v2", "doi": null, "title": "Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting", "short_title": null, "authors": [ "Melanie Sclar", "Yejin Choi", "Yulia Tsvetkov", "Alane Suhr" ], "published_at": "2023-10-17T15:03:30Z", "updated_at": "2024-07-01T22:28:01Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 10, "tier": "background", "publication_type": "conference", "venue": "ICLR 2024", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI", "cs.LG" ], "abstract": "As large language models (LLMs) are adopted as a fundamental component of language technologies, it is crucial to accurately characterize their performance. Because choices in prompt design can strongly influence model behavior, this design process is critical in effectively using any modern pre-trained generative language model. In this work, we focus on LLM sensitivity to a quintessential class of meaning-preserving design choices: prompt formatting. We find that several widely used open-source LLMs are extremely sensitive to subtle changes in prompt formatting in few-shot settings, with performance differences of up to 76 accuracy points when evaluated using LLaMA-2-13B. Sensitivity remains even when increasing model size, the number of few-shot examples, or performing instruction tuning. Our analysis suggests that work evaluating LLMs with prompting-based methods would benefit from reporting a range of performance across plausible prompt formats, instead of the currently-standard practice of reporting performance on a single format. We also show that format performance only weakly correlates between models, which puts into question the methodological validity of comparing models with an arbitrarily chosen, fixed prompt format. To facilitate systematic analysis we propose FormatSpread, an algorithm that rapidly evaluates a sampled set of plausible prompt formats for a given task, and reports the interval of expected performance without accessing model weights. Furthermore, we present a suite of analyses that characterize the nature of this sensitivity, including exploring the influence of particular atomic perturbations and the internal representation of particular formats.", "url": "https://arxiv.org/abs/2310.11324v2", "pdf_url": "https://arxiv.org/pdf/2310.11324v2", "bibtex_key": "arxiv2310_11324", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "interface-semantics", "robustness" ], "stages": [ "contract" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "label-robustness", "role": "Sensitivity to spurious prompt-format features: a template for order and format invariance tests.", "claims": [] }, { "id": "arxiv:2310.08491", "arxiv_id": "2310.08491", "versioned_id": "2310.08491v2", "doi": null, "title": "Prometheus: Inducing Fine-grained Evaluation Capability in Language Models", "short_title": null, "authors": [ "Seungone Kim", "Jamin Shin", "Yejin Cho", "Joel Jang", "Shayne Longpre", "Hwaran Lee", "Sangdoo Yun", "Seongjin Shin", "Sungdong Kim", "James Thorne", "Minjoon Seo" ], "published_at": "2023-10-12T16:50:08Z", "updated_at": "2024-03-09T10:44:58Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 10, "tier": "background", "publication_type": "conference", "venue": "ICLR 2024", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL", "cs.LG" ], "abstract": "Recently, using a powerful proprietary Large Language Model (LLM) (e.g., GPT-4) as an evaluator for long-form responses has become the de facto standard. However, for practitioners with large-scale evaluation tasks and custom criteria in consideration (e.g., child-readability), using proprietary LLMs as an evaluator is unreliable due to the closed-source nature, uncontrolled versioning, and prohibitive costs. In this work, we propose Prometheus, a fully open-source LLM that is on par with GPT-4's evaluation capabilities when the appropriate reference materials (reference answer, score rubric) are accompanied. We first construct the Feedback Collection, a new dataset that consists of 1K fine-grained score rubrics, 20K instructions, and 100K responses and language feedback generated by GPT-4. Using the Feedback Collection, we train Prometheus, a 13B evaluator LLM that can assess any given long-form text based on customized score rubric provided by the user. Experimental results show that Prometheus scores a Pearson correlation of 0.897 with human evaluators when evaluating with 45 customized score rubrics, which is on par with GPT-4 (0.882), and greatly outperforms ChatGPT (0.392). Furthermore, measuring correlation with GPT-4 with 1222 customized score rubrics across four benchmarks (MT Bench, Vicuna Bench, Feedback Bench, Flask Eval) shows similar trends, bolstering Prometheus's capability as an evaluator LLM. Lastly, Prometheus achieves the highest accuracy on two human preference benchmarks (HHH Alignment & MT Bench Human Judgment) compared to open-sourced reward models explicitly trained on human preference datasets, highlighting its potential as an universal reward model. We open-source our code, dataset, and model at https://kaistai.github.io/prometheus/.", "url": "https://arxiv.org/abs/2310.08491v2", "pdf_url": "https://arxiv.org/pdf/2310.08491v2", "bibtex_key": "arxiv2310_08491", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "judging" ], "stages": [ "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "judging", "role": "Prometheus: a dedicated evaluator model and a relevant baseline for decision-only judging.", "claims": [] }, { "id": "arxiv:2310.05921", "arxiv_id": "2310.05921", "versioned_id": "2310.05921v3", "doi": null, "title": "Conformal Decision Theory: Safe Autonomous Decisions from Imperfect Predictions", "short_title": null, "authors": [ "Jordan Lekeufack", "Anastasios N. Angelopoulos", "Andrea Bajcsy", "Michael I. Jordan", "Jitendra Malik" ], "published_at": "2023-10-09T17:59:30Z", "updated_at": "2024-05-02T13:10:06Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 10, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "stat.ML", "cs.LG", "cs.RO", "stat.ME" ], "abstract": "We introduce Conformal Decision Theory, a framework for producing safe autonomous decisions despite imperfect machine learning predictions. Examples of such decisions are ubiquitous, from robot planning algorithms that rely on pedestrian predictions, to calibrating autonomous manufacturing to exhibit high throughput and low error, to the choice of trusting a nominal policy versus switching to a safe backup policy at run-time. The decisions produced by our algorithms are safe in the sense that they come with provable statistical guarantees of having low risk without any assumptions on the world model whatsoever; the observations need not be I.I.D. and can even be adversarial. The theory extends results from conformal prediction to calibrate decisions directly, without requiring the construction of prediction sets. Experiments demonstrate the utility of our approach in robot motion planning around humans, automated stock trading, and robot manufacturing.", "url": "https://arxiv.org/abs/2310.05921v3", "pdf_url": "https://arxiv.org/pdf/2310.05921v3", "bibtex_key": "arxiv2310_05921", "included_by": [ "Q05_calibrated_decisions" ], "inclusion_reason": "Found by the snapshot queries and retained as background: Conformal Decision Theory，把不完美预测转为受约束决策。", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control", "calibration" ], "stages": [ "probability", "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "decision-calibration", "role": "Conformal decision theory: turning imperfect predictions into constrained decisions.", "claims": [] }, { "id": "arxiv:2309.04992", "arxiv_id": "2309.04992", "versioned_id": "2309.04992v1", "doi": null, "title": "Mitigating Word Bias in Zero-shot Prompt-based Classifiers", "short_title": null, "authors": [ "Adian Liusie", "Potsawee Manakul", "Mark J. F. Gales" ], "published_at": "2023-09-10T10:57:41Z", "updated_at": "2023-09-10T10:57:41Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Prompt-based classifiers are an attractive approach for zero-shot classification. However, the precise choice of the prompt template and label words can largely influence performance, with semantically equivalent settings often showing notable performance difference. This discrepancy can be partly attributed to word biases, where the classifier may be biased towards classes. To address this problem, it is possible to optimise classification thresholds on a labelled data set, however, this mitigates some of the advantages of prompt-based classifiers. This paper instead approaches this problem by examining the expected marginal probabilities of the classes. Here, probabilities are reweighted to have a uniform prior over classes, in an unsupervised fashion. Further, we draw a theoretical connection between the class priors and the language models' word prior, and offer the ability to set a threshold in a zero-resource fashion. We show that matching class priors correlates strongly with the oracle upper bound performance and demonstrate large consistent performance gains for prompt settings over a range of NLP tasks.", "url": "https://arxiv.org/abs/2309.04992v1", "pdf_url": "https://arxiv.org/pdf/2309.04992v1", "bibtex_key": "arxiv2309_04992", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "interface-semantics", "robustness" ], "stages": [ "contract" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "label-robustness", "role": "Word bias in zero-shot prompt-based classifiers: why option names can become shortcuts.", "claims": [] }, { "id": "arxiv:2307.13565", "arxiv_id": "2307.13565", "versioned_id": "2307.13565v4", "doi": "10.1613/jair.1.15320", "title": "Decision-Focused Learning: Foundations, State of the Art, Benchmark and Future Opportunities", "short_title": null, "authors": [ "Jayanta Mandi", "James Kotary", "Senne Berden", "Maxime Mulamba", "Victor Bucarey", "Tias Guns", "Ferdinando Fioretto" ], "published_at": "2023-07-25T15:17:31Z", "updated_at": "2024-09-04T11:47:12Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 7, "tier": "background", "publication_type": "journal", "venue": "Journal of Artificial Intelligence Research 81 (2024) 1623-1701", "venue_source": "arxiv_journal_ref", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "math.OC" ], "abstract": "Decision-focused learning (DFL) is an emerging paradigm that integrates machine learning (ML) and constrained optimization to enhance decision quality by training ML models in an end-to-end system. This approach shows significant potential to revolutionize combinatorial decision-making in real-world applications that operate under uncertainty, where estimating unknown parameters within decision models is a major challenge. This paper presents a comprehensive review of DFL, providing an in-depth analysis of both gradient-based and gradient-free techniques used to combine ML and constrained optimization. It evaluates the strengths and limitations of these techniques and includes an extensive empirical evaluation of eleven methods across seven problems. The survey also offers insights into recent advancements and future research directions in DFL. Code and benchmark: https://github.com/PredOpt/predopt-benchmarks", "url": "https://arxiv.org/abs/2307.13565v4", "pdf_url": "https://arxiv.org/pdf/2307.13565v4", "bibtex_key": "arxiv2307_13565", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability", "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "decision-calibration", "role": "Decision-focused learning survey; its optimisation semantics differ from an API returning a choice.", "claims": [] }, { "id": "arxiv:2307.09702", "arxiv_id": "2307.09702", "versioned_id": "2307.09702v4", "doi": null, "title": "Efficient Guided Generation for Large Language Models", "short_title": null, "authors": [ "Brandon T. Willard", "Rémi Louf" ], "published_at": "2023-07-19T01:14:49Z", "updated_at": "2023-08-19T21:27:51Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 7, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.LG" ], "abstract": "In this article we show how the problem of neural text generation can be constructively reformulated in terms of transitions between the states of a finite-state machine. This framework leads to an efficient approach to guiding text generation with regular expressions and context-free grammars by allowing the construction of an index over a language model's vocabulary. The approach is model agnostic, allows one to enforce domain-specific knowledge and constraints, and enables the construction of reliable interfaces by guaranteeing the structure of the generated text. It adds little overhead to the token sequence generation process and significantly outperforms existing solutions. An implementation is provided in the open source Python library Outlines", "url": "https://arxiv.org/abs/2307.09702v4", "pdf_url": "https://arxiv.org/pdf/2307.09702v4", "bibtex_key": "arxiv2307_09702", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "Guided generation: valid outputs by construction without solving semantic errors.", "claims": [] }, { "id": "arxiv:2306.07536", "arxiv_id": "2306.07536", "versioned_id": "2306.07536v1", "doi": null, "title": "TART: A plug-and-play Transformer module for task-agnostic reasoning", "short_title": null, "authors": [ "Kush Bhatia", "Avanika Narayan", "Christopher De Sa", "Christopher Ré" ], "published_at": "2023-06-13T04:37:00Z", "updated_at": "2023-06-13T04:37:00Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 6, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "cs.CL" ], "abstract": "Large language models (LLMs) exhibit in-context learning abilities which enable the same model to perform several tasks without any task-specific training. In contrast, traditional adaptation approaches, such as fine-tuning, modify the underlying models for each specific task. In-context learning, however, consistently underperforms task-specific tuning approaches even when presented with the same examples. While most existing approaches (e.g., prompt engineering) focus on the LLM's learned representations to patch this performance gap, our analysis actually reveal that LLM representations contain sufficient information to make good predictions. As such, we focus on the LLM's reasoning abilities and demonstrate that this performance gap exists due to their inability to perform simple probabilistic reasoning tasks. This raises an intriguing question: Are LLMs actually capable of learning how to reason in a task-agnostic manner? We answer this in the affirmative and propose TART which generically improves an LLM's reasoning abilities using a synthetically trained Transformer-based reasoning module. TART trains this reasoning module in a task-agnostic manner using only synthetic logistic regression tasks and composes it with an arbitrary real-world pre-trained model without any additional training. With a single inference module, TART improves performance across different model families (GPT-Neo, Pythia, BLOOM), model sizes (100M - 6B), tasks (14 NLP binary classification tasks), and even across different modalities (audio and vision). Additionally, on the RAFT Benchmark, TART improves GPT-Neo (125M)'s performance such that it outperforms BLOOM (176B), and is within 4% of GPT-3 (175B). Our code and models are available at https://github.com/HazyResearch/TART .", "url": "https://arxiv.org/abs/2306.07536v1", "pdf_url": "https://arxiv.org/pdf/2306.07536v1", "bibtex_key": "arxiv2306_07536", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "TART: a task-agnostic discriminative reasoning module that predates the product framing.", "claims": [] }, { "id": "arxiv:2306.05685", "arxiv_id": "2306.05685", "versioned_id": "2306.05685v4", "doi": null, "title": "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena", "short_title": null, "authors": [ "Lianmin Zheng", "Wei-Lin Chiang", "Ying Sheng", "Siyuan Zhuang", "Zhanghao Wu", "Yonghao Zhuang", "Zi Lin", "Zhuohan Li", "Dacheng Li", "Eric P. Xing", "Hao Zhang", "Joseph E. Gonzalez", "Ion Stoica" ], "published_at": "2023-06-09T05:55:52Z", "updated_at": "2023-12-24T02:01:34Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 6, "tier": "background", "publication_type": "conference", "venue": "NeurIPS Datasets and Benchmarks 2023", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "abstract": "Evaluating large language model (LLM) based chat assistants is challenging due to their broad capabilities and the inadequacy of existing benchmarks in measuring human preferences. To address this, we explore using strong LLMs as judges to evaluate these models on more open-ended questions. We examine the usage and limitations of LLM-as-a-judge, including position, verbosity, and self-enhancement biases, as well as limited reasoning ability, and propose solutions to mitigate some of them. We then verify the agreement between LLM judges and human preferences by introducing two benchmarks: MT-bench, a multi-turn question set; and Chatbot Arena, a crowdsourced battle platform. Our results reveal that strong LLM judges like GPT-4 can match both controlled and crowdsourced human preferences well, achieving over 80% agreement, the same level of agreement between humans. Hence, LLM-as-a-judge is a scalable and explainable way to approximate human preferences, which are otherwise very expensive to obtain. Additionally, we show our benchmark and traditional benchmarks complement each other by evaluating several variants of LLaMA and Vicuna. The MT-bench questions, 3K expert votes, and 30K conversations with human preferences are publicly available at https://github.com/lm-sys/FastChat/tree/main/fastchat/llm_judge.", "url": "https://arxiv.org/abs/2306.05685v4", "pdf_url": "https://arxiv.org/pdf/2306.05685v4", "bibtex_key": "arxiv2306_05685", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "judging" ], "stages": [ "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "judging", "role": "MT-Bench and Chatbot Arena: foundations and biases of LLM-as-a-judge.", "claims": [] }, { "id": "arxiv:2305.05176", "arxiv_id": "2305.05176", "versioned_id": "2305.05176v1", "doi": null, "title": "FrugalGPT: How to Use Large Language Models While Reducing Cost and Improving Performance", "short_title": null, "authors": [ "Lingjiao Chen", "Matei Zaharia", "James Zou" ], "published_at": "2023-05-09T05:11:02Z", "updated_at": "2023-05-09T05:11:02Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 5, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "cs.CL", "cs.SE" ], "abstract": "There is a rapidly growing number of large language models (LLMs) that users can query for a fee. We review the cost associated with querying popular LLM APIs, e.g. GPT-4, ChatGPT, J1-Jumbo, and find that these models have heterogeneous pricing structures, with fees that can differ by two orders of magnitude. In particular, using LLMs on large collections of queries and text can be expensive. Motivated by this, we outline and discuss three types of strategies that users can exploit to reduce the inference cost associated with using LLMs: 1) prompt adaptation, 2) LLM approximation, and 3) LLM cascade. As an example, we propose FrugalGPT, a simple yet flexible instantiation of LLM cascade which learns which combinations of LLMs to use for different queries in order to reduce cost and improve accuracy. Our experiments show that FrugalGPT can match the performance of the best individual LLM (e.g. GPT-4) with up to 98% cost reduction or improve the accuracy over GPT-4 by 4% with the same cost. The ideas and findings presented here lay a foundation for using LLMs sustainably and efficiently.", "url": "https://arxiv.org/abs/2305.05176v1", "pdf_url": "https://arxiv.org/pdf/2305.05176v1", "bibtex_key": "arxiv2305_05176", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "routing", "selective-control" ], "stages": [ "control", "use" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "routing", "role": "FrugalGPT: model cascades and cost optimisation have clear precedents.", "claims": [] }, { "id": "arxiv:2302.09664", "arxiv_id": "2302.09664", "versioned_id": "2302.09664v3", "doi": null, "title": "Semantic Uncertainty: Linguistic Invariances for Uncertainty Estimation in Natural Language Generation", "short_title": null, "authors": [ "Lorenz Kuhn", "Yarin Gal", "Sebastian Farquhar" ], "published_at": "2023-02-19T20:10:07Z", "updated_at": "2023-04-15T12:55:45Z", "retrieved_at": "2026-09-23", "year": 2023, "month": 2, "tier": "background", "publication_type": "conference", "venue": "ICLR 2023", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI", "cs.LG" ], "abstract": "We introduce a method to measure uncertainty in large language models. For tasks like question answering, it is essential to know when we can trust the natural language outputs of foundation models. We show that measuring uncertainty in natural language is challenging because of \"semantic equivalence\" -- different sentences can mean the same thing. To overcome these challenges we introduce semantic entropy -- an entropy which incorporates linguistic invariances created by shared meanings. Our method is unsupervised, uses only a single model, and requires no modifications to off-the-shelf language models. In comprehensive ablation studies we show that the semantic entropy is more predictive of model accuracy on question answering data sets than comparable baselines.", "url": "https://arxiv.org/abs/2302.09664v3", "pdf_url": "https://arxiv.org/pdf/2302.09664v3", "bibtex_key": "arxiv2302_09664", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Semantic uncertainty: an alternative definition of confidence for generative models.", "claims": [] }, { "id": "arxiv:2209.11055", "arxiv_id": "2209.11055", "versioned_id": "2209.11055v1", "doi": null, "title": "Efficient Few-Shot Learning Without Prompts", "short_title": null, "authors": [ "Lewis Tunstall", "Nils Reimers", "Unso Eun Seo Jo", "Luke Bates", "Daniel Korat", "Moshe Wasserblat", "Oren Pereg" ], "published_at": "2022-09-22T14:48:11Z", "updated_at": "2022-09-22T14:48:11Z", "retrieved_at": "2026-09-23", "year": 2022, "month": 9, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Recent few-shot methods, such as parameter-efficient fine-tuning (PEFT) and pattern exploiting training (PET), have achieved impressive results in label-scarce settings. However, they are difficult to employ since they are subject to high variability from manually crafted prompts, and typically require billion-parameter language models to achieve high accuracy. To address these shortcomings, we propose SetFit (Sentence Transformer Fine-tuning), an efficient and prompt-free framework for few-shot fine-tuning of Sentence Transformers (ST). SetFit works by first fine-tuning a pretrained ST on a small number of text pairs, in a contrastive Siamese manner. The resulting model is then used to generate rich text embeddings, which are used to train a classification head. This simple framework requires no prompts or verbalizers, and achieves high accuracy with orders of magnitude less parameters than existing techniques. Our experiments show that SetFit obtains comparable results with PEFT and PET techniques, while being an order of magnitude faster to train. We also show that SetFit can be applied in multilingual settings by simply switching the ST body. Our code is available at https://github.com/huggingface/setfit and our datasets at https://huggingface.co/setfit .", "url": "https://arxiv.org/abs/2209.11055v1", "pdf_url": "https://arxiv.org/pdf/2209.11055v1", "bibtex_key": "arxiv2209_11055", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "SetFit: a few-shot discriminative baseline for asking whether convenience comes from the zero-training interface.", "claims": [] }, { "id": "arxiv:2207.05221", "arxiv_id": "2207.05221", "versioned_id": "2207.05221v4", "doi": null, "title": "Language Models (Mostly) Know What They Know", "short_title": null, "authors": [ "Saurav Kadavath", "Tom Conerly", "Amanda Askell", "Tom Henighan", "Dawn Drain", "Ethan Perez", "Nicholas Schiefer", "Zac Hatfield-Dodds", "Nova DasSarma", "Eli Tran-Johnson", "Scott Johnston", "Sheer El-Showk", "Andy Jones", "Nelson Elhage", "Tristan Hume", "Anna Chen", "Yuntao Bai", "Sam Bowman", "Stanislav Fort", "Deep Ganguli", "Danny Hernandez", "Josh Jacobson", "Jackson Kernion", "Shauna Kravec", "Liane Lovitt", "Kamal Ndousse", "Catherine Olsson", "Sam Ringer", "Dario Amodei", "Tom Brown", "Jack Clark", "Nicholas Joseph", "Ben Mann", "Sam McCandlish", "Chris Olah", "Jared Kaplan" ], "published_at": "2022-07-11T22:59:39Z", "updated_at": "2022-11-21T16:38:35Z", "retrieved_at": "2026-09-23", "year": 2022, "month": 7, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI", "cs.LG" ], "abstract": "We study whether language models can evaluate the validity of their own claims and predict which questions they will be able to answer correctly. We first show that larger models are well-calibrated on diverse multiple choice and true/false questions when they are provided in the right format. Thus we can approach self-evaluation on open-ended sampling tasks by asking models to first propose answers, and then to evaluate the probability \"P(True)\" that their answers are correct. We find encouraging performance, calibration, and scaling for P(True) on a diverse array of tasks. Performance at self-evaluation further improves when we allow models to consider many of their own samples before predicting the validity of one specific possibility. Next, we investigate whether models can be trained to predict \"P(IK)\", the probability that \"I know\" the answer to a question, without reference to any particular proposed answer. Models perform well at predicting P(IK) and partially generalize across tasks, though they struggle with calibration of P(IK) on new tasks. The predicted P(IK) probabilities also increase appropriately in the presence of relevant source materials in the context, and in the presence of hints towards the solution of mathematical word problems. We hope these observations lay the groundwork for training more honest models, and for investigating how honesty generalizes to cases where models are trained on objectives other than the imitation of human writing.", "url": "https://arxiv.org/abs/2207.05221v4", "pdf_url": "https://arxiv.org/pdf/2207.05221v4", "bibtex_key": "arxiv2207_05221", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Language models (mostly) know what they know: self-knowledge and confidence expression; native and verbalised confidence differ.", "claims": [] }, { "id": "arxiv:2109.05093", "arxiv_id": "2109.05093", "versioned_id": "2109.05093v1", "doi": null, "title": "PICARD: Parsing Incrementally for Constrained Auto-Regressive Decoding from Language Models", "short_title": null, "authors": [ "Torsten Scholak", "Nathan Schucher", "Dzmitry Bahdanau" ], "published_at": "2021-09-10T20:14:08Z", "updated_at": "2021-09-10T20:14:08Z", "retrieved_at": "2026-09-23", "year": 2021, "month": 9, "tier": "background", "publication_type": "conference", "venue": "EMNLP 2021", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL", "cs.PL" ], "abstract": "Large pre-trained language models for textual data have an unconstrained output space; at each decoding step, they can produce any of 10,000s of sub-word tokens. When fine-tuned to target constrained formal languages like SQL, these models often generate invalid code, rendering it unusable. We propose PICARD (code and trained models available at https://github.com/ElementAI/picard), a method for constraining auto-regressive decoders of language models through incremental parsing. PICARD helps to find valid output sequences by rejecting inadmissible tokens at each decoding step. On the challenging Spider and CoSQL text-to-SQL translation tasks, we show that PICARD transforms fine-tuned T5 models with passable performance into state-of-the-art solutions.", "url": "https://arxiv.org/abs/2109.05093v1", "pdf_url": "https://arxiv.org/pdf/2109.05093v1", "bibtex_key": "arxiv2109_05093", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "structured-output" ], "stages": [ "contract", "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "structured-output", "role": "PICARD: incremental constrained autoregressive decoding, a historical structured-output baseline.", "claims": [] }, { "id": "arxiv:2107.07511", "arxiv_id": "2107.07511", "versioned_id": "2107.07511v6", "doi": null, "title": "A Gentle Introduction to Conformal Prediction and Distribution-Free Uncertainty Quantification", "short_title": null, "authors": [ "Anastasios N. Angelopoulos", "Stephen Bates" ], "published_at": "2021-07-15T17:59:50Z", "updated_at": "2022-12-07T05:08:01Z", "retrieved_at": "2026-09-23", "year": 2021, "month": 7, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI", "math.ST", "stat.ME", "stat.ML" ], "abstract": "Black-box machine learning models are now routinely used in high-risk settings, like medical diagnostics, which demand uncertainty quantification to avoid consequential model failures. Conformal prediction is a user-friendly paradigm for creating statistically rigorous uncertainty sets/intervals for the predictions of such models. Critically, the sets are valid in a distribution-free sense: they possess explicit, non-asymptotic guarantees even without distributional assumptions or model assumptions. One can use conformal prediction with any pre-trained model, such as a neural network, to produce sets that are guaranteed to contain the ground truth with a user-specified probability, such as 90%. It is easy-to-understand, easy-to-use, and general, applying naturally to problems arising in the fields of computer vision, natural language processing, deep reinforcement learning, and so on. This hands-on introduction is aimed to provide the reader a working understanding of conformal prediction and related distribution-free uncertainty quantification techniques with one self-contained document. We lead the reader through practical theory for and examples of conformal prediction and describe its extensions to complex machine learning tasks involving structured outputs, distribution shift, time-series, outliers, models that abstain, and more. Throughout, there are many explanatory illustrations, examples, and code samples in Python. With each code sample comes a Jupyter notebook implementing the method on a real-data example; the notebooks can be accessed and easily run using our codebase.", "url": "https://arxiv.org/abs/2107.07511v6", "pdf_url": "https://arxiv.org/pdf/2107.07511v6", "bibtex_key": "arxiv2107_07511", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control", "calibration" ], "stages": [ "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "selective", "role": "Conformal prediction: finite-sample coverage guarantees and their exchangeability assumptions.", "claims": [] }, { "id": "arxiv:2006.01862", "arxiv_id": "2006.01862", "versioned_id": "2006.01862v3", "doi": null, "title": "Consistent Estimators for Learning to Defer to an Expert", "short_title": null, "authors": [ "Hussein Mozannar", "David Sontag" ], "published_at": "2020-06-02T18:21:38Z", "updated_at": "2021-01-25T01:43:28Z", "retrieved_at": "2026-09-23", "year": 2020, "month": 6, "tier": "background", "publication_type": "conference", "venue": "ICML 2020", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.LG", "cs.HC", "stat.ML" ], "abstract": "Learning algorithms are often used in conjunction with expert decision makers in practical scenarios, however this fact is largely ignored when designing these algorithms. In this paper we explore how to learn predictors that can either predict or choose to defer the decision to a downstream expert. Given only samples of the expert's decisions, we give a procedure based on learning a classifier and a rejector and analyze it theoretically. Our approach is based on a novel reduction to cost sensitive learning where we give a consistent surrogate loss for cost sensitive learning that generalizes the cross entropy loss. We show the effectiveness of our approach on a variety of experimental tasks.", "url": "https://arxiv.org/abs/2006.01862v3", "pdf_url": "https://arxiv.org/pdf/2006.01862v3", "bibtex_key": "arxiv2006_01862", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control" ], "stages": [ "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "selective", "role": "Learning to defer: modelling expert cost jointly with prediction error.", "claims": [] }, { "id": "arxiv:1909.00161", "arxiv_id": "1909.00161", "versioned_id": "1909.00161v1", "doi": null, "title": "Benchmarking Zero-shot Text Classification: Datasets, Evaluation and Entailment Approach", "short_title": null, "authors": [ "Wenpeng Yin", "Jamaal Hay", "Dan Roth" ], "published_at": "2019-08-31T07:42:11Z", "updated_at": "2019-08-31T07:42:11Z", "retrieved_at": "2026-09-23", "year": 2019, "month": 8, "tier": "background", "publication_type": "conference", "venue": "EMNLP 2019", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "Zero-shot text classification (0Shot-TC) is a challenging NLU problem to which little attention has been paid by the research community. 0Shot-TC aims to associate an appropriate label with a piece of text, irrespective of the text domain and the aspect (e.g., topic, emotion, event, etc.) described by the label. And there are only a few articles studying 0Shot-TC, all focusing only on topical categorization which, we argue, is just the tip of the iceberg in 0Shot-TC. In addition, the chaotic experiments in literature make no uniform comparison, which blurs the progress. This work benchmarks the 0Shot-TC problem by providing unified datasets, standardized evaluations, and state-of-the-art baselines. Our contributions include: i) The datasets we provide facilitate studying 0Shot-TC relative to conceptually different and diverse aspects: the ``topic'' aspect includes ``sports'' and ``politics'' as labels; the ``emotion'' aspect includes ``joy'' and ``anger''; the ``situation'' aspect includes ``medical assistance'' and ``water shortage''. ii) We extend the existing evaluation setup (label-partially-unseen) -- given a dataset, train on some labels, test on all labels -- to include a more challenging yet realistic evaluation label-fully-unseen 0Shot-TC (Chang et al., 2008), aiming at classifying text snippets without seeing task specific training data at all. iii) We unify the 0Shot-TC of diverse aspects within a textual entailment formulation and study it this way. Code & Data: https://github.com/yinwenpeng/BenchmarkingZeroShot", "url": "https://arxiv.org/abs/1909.00161v1", "pdf_url": "https://arxiv.org/pdf/1909.00161v1", "bibtex_key": "arxiv1909_00161", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification", "interface-semantics" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "Zero-shot text classification with NLI-style label descriptions: historical context for runtime-defined option semantics.", "claims": [] }, { "id": "arxiv:1906.02530", "arxiv_id": "1906.02530", "versioned_id": "1906.02530v2", "doi": null, "title": "Can You Trust Your Model's Uncertainty? Evaluating Predictive Uncertainty Under Dataset Shift", "short_title": null, "authors": [ "Yaniv Ovadia", "Emily Fertig", "Jie Ren", "Zachary Nado", "D Sculley", "Sebastian Nowozin", "Joshua V. Dillon", "Balaji Lakshminarayanan", "Jasper Snoek" ], "published_at": "2019-06-06T11:42:53Z", "updated_at": "2019-12-17T21:30:28Z", "retrieved_at": "2026-09-23", "year": 2019, "month": 6, "tier": "background", "publication_type": "conference", "venue": "NeurIPS 2019", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "stat.ML", "cs.LG" ], "abstract": "Modern machine learning methods including deep learning have achieved great success in predictive accuracy for supervised learning tasks, but may still fall short in giving useful estimates of their predictive {\\em uncertainty}. Quantifying uncertainty is especially critical in real-world settings, which often involve input distributions that are shifted from the training distribution due to a variety of factors including sample bias and non-stationarity. In such settings, well calibrated uncertainty estimates convey information about when a model's output should (or should not) be trusted. Many probabilistic deep learning methods, including Bayesian-and non-Bayesian methods, have been proposed in the literature for quantifying predictive uncertainty, but to our knowledge there has not previously been a rigorous large-scale empirical comparison of these methods under dataset shift. We present a large-scale benchmark of existing state-of-the-art methods on classification problems and investigate the effect of dataset shift on accuracy and calibration. We find that traditional post-hoc calibration does indeed fall short, as do several other previous methods. However, some methods that marginalize over models give surprisingly strong results across a broad spectrum of tasks.", "url": "https://arxiv.org/abs/1906.02530v2", "pdf_url": "https://arxiv.org/pdf/1906.02530v2", "bibtex_key": "arxiv1906_02530", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration", "robustness" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Predictive uncertainty under dataset shift: in-distribution calibration does not transfer to shifted data.", "claims": [] }, { "id": "arxiv:1901.09192", "arxiv_id": "1901.09192", "versioned_id": "1901.09192v4", "doi": null, "title": "SelectiveNet: A Deep Neural Network with an Integrated Reject Option", "short_title": null, "authors": [ "Yonatan Geifman", "Ran El-Yaniv" ], "published_at": "2019-01-26T10:09:16Z", "updated_at": "2019-06-26T18:55:09Z", "retrieved_at": "2026-09-23", "year": 2019, "month": 1, "tier": "background", "publication_type": "conference", "venue": "ICML 2019", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.LG", "stat.ML" ], "abstract": "We consider the problem of selective prediction (also known as reject option) in deep neural networks, and introduce SelectiveNet, a deep neural architecture with an integrated reject option. Existing rejection mechanisms are based mostly on a threshold over the prediction confidence of a pre-trained network. In contrast, SelectiveNet is trained to optimize both classification (or regression) and rejection simultaneously, end-to-end. The result is a deep neural network that is optimized over the covered domain. In our experiments, we show a consistently improved risk-coverage trade-off over several well-known classification and regression datasets, thus reaching new state-of-the-art results for deep selective classification.", "url": "https://arxiv.org/abs/1901.09192v4", "pdf_url": "https://arxiv.org/pdf/1901.09192v4", "bibtex_key": "arxiv1901_09192", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control" ], "stages": [ "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "selective", "role": "SelectiveNet: jointly learning prediction and a reject option.", "claims": [] }, { "id": "arxiv:1810.04805", "arxiv_id": "1810.04805", "versioned_id": "1810.04805v2", "doi": null, "title": "BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding", "short_title": null, "authors": [ "Jacob Devlin", "Ming-Wei Chang", "Kenton Lee", "Kristina Toutanova" ], "published_at": "2018-10-11T00:50:01Z", "updated_at": "2019-05-24T20:37:26Z", "retrieved_at": "2026-09-23", "year": 2018, "month": 10, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "abstract": "We introduce a new language representation model called BERT, which stands for Bidirectional Encoder Representations from Transformers. Unlike recent language representation models, BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly conditioning on both left and right context in all layers. As a result, the pre-trained BERT model can be fine-tuned with just one additional output layer to create state-of-the-art models for a wide range of tasks, such as question answering and language inference, without substantial task-specific architecture modifications. BERT is conceptually simple and empirically powerful. It obtains new state-of-the-art results on eleven natural language processing tasks, including pushing the GLUE score to 80.5% (7.7% point absolute improvement), MultiNLI accuracy to 86.7% (4.6% absolute improvement), SQuAD v1.1 question answering Test F1 to 93.2 (1.5 point absolute improvement) and SQuAD v2.0 Test F1 to 83.1 (5.1 point absolute improvement).", "url": "https://arxiv.org/abs/1810.04805v2", "pdf_url": "https://arxiv.org/pdf/1810.04805v2", "bibtex_key": "arxiv1810_04805", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "classification" ], "stages": [ "readout" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "classification", "role": "Bidirectional encoders are the basic reference point; a natural-language interface does not make conventional classifiers irrelevant.", "claims": [] }, { "id": "arxiv:1706.04599", "arxiv_id": "1706.04599", "versioned_id": "1706.04599v2", "doi": null, "title": "On Calibration of Modern Neural Networks", "short_title": null, "authors": [ "Chuan Guo", "Geoff Pleiss", "Yu Sun", "Kilian Q. Weinberger" ], "published_at": "2017-06-14T17:33:50Z", "updated_at": "2017-08-03T13:29:46Z", "retrieved_at": "2026-09-23", "year": 2017, "month": 6, "tier": "background", "publication_type": "conference", "venue": "ICML 2017", "venue_source": "arxiv_comment", "venue_verified_at": null, "categories": [ "cs.LG" ], "abstract": "Confidence calibration -- the problem of predicting probability estimates representative of the true correctness likelihood -- is important for classification models in many applications. We discover that modern neural networks, unlike those from a decade ago, are poorly calibrated. Through extensive experiments, we observe that depth, width, weight decay, and Batch Normalization are important factors influencing calibration. We evaluate the performance of various post-processing calibration methods on state-of-the-art architectures with image and document classification datasets. Our analysis and experiments not only offer insights into neural network learning, but also provide a simple and straightforward recipe for practical settings: on most datasets, temperature scaling -- a single-parameter variant of Platt Scaling -- is surprisingly effective at calibrating predictions.", "url": "https://arxiv.org/abs/1706.04599v2", "pdf_url": "https://arxiv.org/pdf/1706.04599v2", "bibtex_key": "arxiv1706_04599", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "calibration" ], "stages": [ "probability" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "calibration", "role": "Calibration of modern neural networks and temperature scaling: the post-hoc baseline for any returned probability.", "claims": [] }, { "id": "arxiv:1705.08500", "arxiv_id": "1705.08500", "versioned_id": "1705.08500v2", "doi": null, "title": "Selective Classification for Deep Neural Networks", "short_title": null, "authors": [ "Yonatan Geifman", "Ran El-Yaniv" ], "published_at": "2017-05-23T19:43:56Z", "updated_at": "2017-06-01T14:10:34Z", "retrieved_at": "2026-09-23", "year": 2017, "month": 5, "tier": "background", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.LG", "cs.AI" ], "abstract": "Selective classification techniques (also known as reject option) have not yet been considered in the context of deep neural networks (DNNs). These techniques can potentially significantly improve DNNs prediction performance by trading-off coverage. In this paper we propose a method to construct a selective classifier given a trained neural network. Our method allows a user to set a desired risk level. At test time, the classifier rejects instances as needed, to grant the desired risk (with high probability). Empirical results over CIFAR and ImageNet convincingly demonstrate the viability of our method, which opens up possibilities to operate DNNs in mission-critical applications. For example, using our method an unprecedented 2% error in top-5 ImageNet classification can be guaranteed with probability 99.9%, and almost 60% test coverage.", "url": "https://arxiv.org/abs/1705.08500v2", "pdf_url": "https://arxiv.org/pdf/1705.08500v2", "bibtex_key": "arxiv1705_08500", "included_by": [ "background_by_id" ], "inclusion_reason": "Purposively selected background reference, retrieved by explicit arXiv ID and checked against arXiv metadata.", "relationship": null, "summary": null, "task": null, "datasets": [], "baselines": [], "models": [], "main_finding": null, "caveats": [], "survey_use": null, "topics": [ "selective-control" ], "stages": [ "control" ], "method_families": [], "applications": [], "model_relationship": [ "adjacent_method" ], "test_level": null, "model_versions": [], "study_family_id": null, "openness": { "code": { "status": "not_assessed" }, "weights": { "status": "not_assessed" }, "data": { "status": "not_assessed" }, "predictions": { "status": "not_assessed" }, "recomputable": { "status": "not_assessed" }, "reproduction": { "status": "not_assessed" } }, "evidence_locator": null, "priority": "context", "review_depth": "metadata_and_abstract", "background_group": "selective", "role": "Selective classification: risk–coverage and rejection; confidence gating long predates typed decision models.", "claims": [] }, { "id": "arxiv:2609.27678", "arxiv_id": "2609.27678", "versioned_id": "2609.27678v1", "doi": null, "title": "Same Scores, Different Decisions: Evaluating JEV and Language Models for Legal Document Understanding", "short_title": "Jev on Contract Inference", "authors": [ "Fan Zhang", "Yankai Chen", "Zhuohan Xie", "Yixi Zhou", "Sijia Peng", "Lei Fan", "Xinhua Ji", "Cunyuan Zheng", "Huangyong Shan", "Philip S. Yu", "Xue Liu", "Yu Chen", "Preslav Nakov", "Songwei He" ], "published_at": "2026-09-23T10:53:01Z", "updated_at": "2026-09-23T10:53:01Z", "retrieved_at": "2026-09-24", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL" ], "url": "https://arxiv.org/abs/2609.27678v1", "pdf_url": "https://arxiv.org/pdf/2609.27678v1", "bibtex_key": "arxiv2609_27678", "included_by": [ "Q01_jev" ], "inclusion_reason": "Evaluates hosted Jev against nine language models on contract inference.", "relationship": "Evaluates hosted jev-1.13.0 against nine language models on legal contract inference.", "summary": "Compares Jev with nine language models on ContractNLI, measuring cost, response time, baseline accuracy and whether individual judgments stay correct when the visible hypotheses, requested outputs and output order change.", "task": "Three-way contract inference (entailment, contradiction, not mentioned) over 17 hypotheses per contract, plus controlled request conditions on one fixed anchor per contract.", "datasets": [ "ContractNLI official test split (123 contracts, 2,091 judgments)", "Stability panel: 30 contracts × 4 conditions × 3 repeats", "ContractNLI development contracts (diagnostics)" ], "baselines": [ "Gemini 3.5 Flash-Lite", "Gemini 3.1 Pro Preview", "GPT-5.6 Luna and Terra", "GPT-6 Astra", "Claude Sonnet 5", "Claude Haiku 4.5", "Qwen3.5-4B and 9B (local)" ], "models": [ "TypeSafe Jev (jev-1.13.0)" ], "main_finding": "Jev had the lowest cost ($0.000228 per contract) and median response time (1.24 s) but lower baseline accuracy (77.38%) than all seven hosted language models (78.96–83.21%). On 30 fixed anchors it kept 23 correct in all twelve responses, one fewer than Claude Sonnet 5, while five of its anchors were answered wrongly every time.", "caveats": [ "Stability panel of 30 anchors; paired intervals are descriptive and not adjusted for multiple comparisons.", "Three comparators were added after earlier results were inspected.", "Jev’s output-order condition changes question-key order, which is not the same as autoregressive output order.", "Latency is measured from one client across different providers and collection times." ], "survey_use": "Report persistent correctness and stable errors next to accuracy, and tie cost and latency to a stated workload.", "topics": [ "evaluation-methodology", "efficiency", "robustness", "classification" ], "stages": [ "contract", "use" ], "method_families": [ "hosted_service" ], "applications": [ "legal-review" ], "model_relationship": [ "commercial_jev" ], "test_level": "model_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13.0" } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "Benchmark code, configurations and analysis.", "url": "https://github.com/ZF-Utokyo/Jev-Benchmark" }, "weights": { "status": "not_applicable" }, "data": { "status": "available", "note": "ContractNLI (CC BY 4.0) with the study’s splits." }, "predictions": { "status": "available", "note": "Per-request predictions released as predictions.jsonl." }, "recomputable": { "status": "available", "note": "Reported metrics can be recomputed from the released predictions." }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§2–4 (Tables 1–2, Figure 3); Limitations", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-27678-cost", "c-27678-accuracy", "c-27678-persistent" ], "abstract": "Contract inference requires multiple judgments about a shared document, but aggregate accuracy can conceal changes in the individual decisions. Repeated agreement is also insufficient: a model may consistently return the wrong answer. In this paper, we compare Jev with nine language models on ContractNLI, evaluating inference cost, response time, average correctness, and correctness across repeated request conditions. Controlled comparisons vary hypothesis visibility, requested outputs, and output order while keeping the contract and target judgment fixed. Jev has the lowest cost and median response time among the evaluated configurations, while hosted language models achieve higher baseline accuracy. Rankings by baseline accuracy differ from rankings by correctness across every condition and repeat, although small differences in the latter do not establish a general stability advantage. Development diagnostics further reveal compensating corrections and regressions, as well as persistent errors. These findings motivate evaluating cost and response time alongside whether individual judgments remain correct as the request configuration changes. Code: https://github.com/ZF-Utokyo/Jev-Benchmark" }, { "id": "arxiv:2609.27607", "arxiv_id": "2609.27607", "versioned_id": "2609.27607v1", "doi": null, "title": "Can Jev Judge Radiology Reports? Evaluating a System One Model for Clinical Factuality", "short_title": "Jev as a Radiology Report Judge", "authors": [ "Jiaju Huang", "Hao Yang", "Xinyu Ma", "Xinglong Liang", "Kunyan Cai", "Junqiang Ma", "Shaobin Chen", "Yue Sun", "Tao Tan" ], "published_at": "2026-09-23T09:26:46Z", "updated_at": "2026-09-23T09:26:46Z", "retrieved_at": "2026-09-24", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.CL", "cs.AI" ], "url": "https://arxiv.org/abs/2609.27607v1", "pdf_url": "https://arxiv.org/pdf/2609.27607v1", "bibtex_key": "arxiv2609_27607", "included_by": [ "Q01_jev", "Q03_system_one", "Q10_recent_decision_models", "Q11_recent_systemone", "X04_system1_model", "X08_decision_models_plural" ], "inclusion_reason": "Uses hosted Jev as a factuality judge for generated radiology reports.", "relationship": "Uses hosted jev-1.13.0 as a statement-level support judge for generated radiology reports.", "summary": "Scores factual disagreement between a generated and a reference radiology report by asking Jev whether each extracted statement is supported, contradicted or not addressed by the other report, in both directions.", "task": "Report-level agreement with expert error counts; sentence-level detection of injected errors.", "datasets": [ "RadEvalX (100 report pairs)", "RadEvalExpert (624 pairs from 208 studies)", "ReXErr sentence test set (19,513 sentences)", "RaTE-Eval (supplementary)" ], "baselines": [ "Open NLI judge (DeBERTa-v3-large)", "GREEN", "RadCliQ", "RadGraph F1", "BERTScore", "RadFact and RadMatch on a local Qwen 27B backend", "Finding-count baseline" ], "models": [ "TypeSafe Jev (jev-1.13.0)" ], "main_finding": "A one-question Jev configuration reached Kendall τ 0.573 (RadEvalX) and 0.398 (RadEvalExpert) with expert error counts, above an open NLI judge in the same pipeline, at under US$0.03 per 100 report pairs for the judgment calls; RadMatch on a local 27B model agreed better on clinically significant errors.", "caveats": [ "Agreement is measured against reference text, not against the image.", "Statements are extracted by a local Qwen3.5-9B model whose cost is excluded.", "Raw probabilities need task-specific calibration before thresholding.", "One Jev version and small expert benchmarks." ], "survey_use": "Separate the judge from decomposition and aggregation, compare with a count baseline, and calibrate before thresholding.", "topics": [ "judging", "calibration", "efficiency", "evaluation-methodology" ], "stages": [ "probability", "use" ], "method_families": [ "hosted_service" ], "applications": [ "judging" ], "model_relationship": [ "commercial_jev" ], "test_level": "model_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13.0" } ], "study_family_id": null, "openness": { "code": { "status": "not_located", "note": "The authors state that analysis scripts and aggregate results are being prepared for release." }, "weights": { "status": "not_applicable" }, "data": { "status": "restricted", "note": "Benchmarks are available through PhysioNet and their own distributions under their terms." }, "predictions": { "status": "not_located" }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3–5 (Tables 1–3); §6.1", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-27607-agreement", "c-27607-cost", "c-27607-calibration" ], "abstract": "An AI-generated radiology report can resemble a physician's report while omitting an abnormality, adding an unsupported finding, or reversing its presence. Measuring these factual differences is essential for evaluating report generators. We study Jev, a System One decision model, as a simple, low-cost judge of agreement with physician-written reference reports. Our evaluator checks whether each statement is supported by the other report and combines these judgments in both directions to capture unsupported claims and omissions. A single-question configuration reaches Kendall correlations of 0.573 on RadEvalX and 0.398 on RadEvalExpert with expert error counts, outperforming an open natural language inference judge under matched decomposition and aggregation. One support question per statement retains similar expert agreement to seven while using 43-45% fewer judgment input tokens. At the documented API price, judgments cost under three cents per hundred report pairs, excluding local decomposition. In a separate controlled-error test, Jev detects false negation with an AUROC of 0.977. Local RadMatch achieves stronger agreement on clinically significant errors in both expert datasets and on total errors in the shared RadEvalExpert subset. Finding-count and error-scope analyses show that benchmark agreement reflects report size and error definitions as well as medical error detection. These results support Jev as a practical judgment component for measuring factual differences in generated radiology reports and identify where more elaborate evaluation remains valuable." }, { "id": "arxiv:2609.27535", "arxiv_id": "2609.27535", "versioned_id": "2609.27535v1", "doi": null, "title": "KITE: Scaling Jev Population Experiments with Sparse Flagship Calibration", "short_title": "KITE", "authors": [ "Hengyu Li" ], "published_at": "2026-09-23T08:28:15Z", "updated_at": "2026-09-23T08:28:15Z", "retrieved_at": "2026-09-24", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.MA", "cs.CY" ], "url": "https://arxiv.org/abs/2609.27535v1", "pdf_url": "https://arxiv.org/pdf/2609.27535v1", "bibtex_key": "arxiv2609_27535", "included_by": [ "Q01_jev" ], "inclusion_reason": "Uses hosted Jev as the typed behavioural kernel of a population-experiment system.", "relationship": "Uses hosted jev-1.13.0 as a typed behavioural kernel for population experiments, corrected with sparse GPT-6 Astra anchors.", "summary": "Queries Jev once per unique respondent state for a distribution over survey answers, runs populations of any size from the resulting table, corrects intervention effects with sparse flagship-model anchors, and carries measured human–model discrepancy into every conclusion.", "task": "Simulating survey responses and intervention effects in social-science and misinformation experiments.", "datasets": [ "SocSci210 (TESS experiments; 37 held-out studies)", "Arechar et al. 2023 COVID headlines (16 countries)", "Epstein et al. 2021 (9,070 participants)" ], "baselines": [ "GPT-6 Astra (flagship)", "GPT-5.6 Luna", "Human half-sample pilot", "Sampling-only intervals" ], "models": [ "TypeSafe Jev (jev-1.13.0)" ], "main_finding": "Flagship anchors covering 1.7% of states cut Epstein effect error by 41% (MAE 0.0305 → 0.0180), and 0.5–1.5% anchor coverage raised captured decision gain on 37 held-out SocSci210 experiments from 0.27 to 0.39; Jev alone passed content-fidelity gates but failed the intervention-sensitivity gates.", "caveats": [ "Only two held-out human-referenced datasets test the hybrid; two preregistered primary endpoints were not shown.", "Corrections can import the flagship’s errors.", "The Jev response cache is not released, and vendor updates prevent guaranteed regeneration of outputs.", "Single-author work with recent code." ], "survey_use": "Keep cheap typed prediction apart from validity against people; a provider’s confidence field is not a calibrated probability about human conclusions.", "topics": [ "calibration", "efficiency", "evaluation-methodology", "agents" ], "stages": [ "probability", "use" ], "method_families": [ "hosted_service" ], "applications": [ "social-simulation" ], "model_relationship": [ "downstream_system" ], "test_level": "system_test", "model_versions": [ { "model": "Jev", "version": "jev-1.13.0", "note": "One Noul question per answer option, renormalised." } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "MIT-licensed package, frozen criteria and provenance records.", "url": "https://github.com/HengyuLi-Ozaki-lab/kite_population_simulator" }, "weights": { "status": "not_applicable" }, "data": { "status": "partial", "note": "Aggregates only; source microdata are not redistributed." }, "predictions": { "status": "not_located", "note": "The authors state the Jev response cache is not released." }, "recomputable": { "status": "partial", "note": "Aggregates and provenance records are released." }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3, 5–9; §11; Table 2", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-27535-anchors", "c-27535-gates", "c-27535-cost", "c-27535-discrepancy" ], "abstract": "KITE queries a typed behavioral kernel once per unique state, then executes populations of any size from the table with event-keyed randomness and common random numbers. An expensive flagship model is reserved for sparse paired anchors that estimate intervention effects. Measured human-model discrepancy is propagated as shared error into every conclusion. Population-experiment cost thus scales with unique states and anchors, while uncertainty is governed by evidence about people rather than Monte Carlo noise. On Epstein experiments with 9,070 participants, anchors covering 1.7% of states reduced effect error by 41% (absolute MAE reduction 0.0125). On 37 held-out SocSci210 experiments, 0.5-1.5% anchor coverage raised captured decision gain from 0.27 to 0.39. The kernel passed content-fidelity criteria in all 15 new countries of a 16-country study. Shared discrepancy yielded retrospective coverage of 93% and 96% at nominal 80% and 90%, versus 29% and 36% from human sampling uncertainty alone. A million agents executed 20 tabulated steps in 0.9 seconds on a laptop. This architecture offers a route to screening candidate interventions before human trials, multi-country content audits, and uncertainty-aware policy comparison at the cost of a few thousand kernel calls with sparse flagship anchors. Property-specific evidence records connect each use to its validation scope, correction provenance, and uncertainty, making these applications auditable." }, { "id": "arxiv:2609.27331", "arxiv_id": "2609.27331", "versioned_id": "2609.27331v1", "doi": null, "title": "JEV-Star: Fast, Low-Cost StarCraft II Control with Language-Model Planning", "short_title": "JEV-Star", "authors": [ "Weiyu Ma", "Liangbing Zhao", "Yongcheng Zeng", "Jian Zhao" ], "published_at": "2026-09-23T04:09:10Z", "updated_at": "2026-09-23T04:09:10Z", "retrieved_at": "2026-09-24", "year": 2026, "month": 9, "tier": "core", "publication_type": "preprint", "venue": "arXiv", "venue_source": "arxiv_metadata", "venue_verified_at": null, "categories": [ "cs.GT" ], "url": "https://arxiv.org/abs/2609.27331v1", "pdf_url": "https://arxiv.org/pdf/2609.27331v1", "bibtex_key": "arxiv2609_27331", "included_by": [ "Q01_jev" ], "inclusion_reason": "Uses hosted Jev for action selection in a StarCraft II controller.", "relationship": "Uses hosted Jev 1.13 for frequent action selection in a StarCraft II controller, with GPT-6 Astra writing persistent plans.", "summary": "Lets Jev choose among enumerated legal actions for full-game and unit control while GPT-6 periodically writes plans that reserve resources and fix army posture; compares the combined system with an earlier Jev-only controller.", "task": "StarCraft II full-game control against the built-in AI and SMAC-Hard micromanagement.", "datasets": [ "StarCraft II 5.0.16 on AltitudeLE (Protoss vs Zerg)", "35 SMAC-Hard maps × 3 episodes" ], "baselines": [ "Initial Jev-only controller", "LLM Play SC2 (reported results, not matched)" ], "models": [ "TypeSafe Jev (Jev 1.13)" ], "main_finding": "With GPT-6 planning, the controller won all four completed non-cheating games, including two against Lv7, at a median Jev response of 0.422 s and about USD 3.71 per game (USD 0.15 for Jev); across 35 battle maps mean enemy elimination rose from 16.50% to 37.69% and wins from 3 to 7 of 105.", "caveats": [ "Observational: the two controllers also differ in candidate descriptions and execution handling, so planning’s contribution is not isolated.", "Four full games and three episodes per map; one map and matchup for full games.", "Planning accounts for 96% of full-game model cost." ], "survey_use": "Attribute system gains to modules with matched ablations, and report action-layer and planning costs separately.", "topics": [ "agents", "efficiency", "selective-control" ], "stages": [ "control", "use" ], "method_families": [ "hosted_service" ], "applications": [ "agent-control" ], "model_relationship": [ "downstream_system" ], "test_level": "system_test", "model_versions": [ { "model": "Jev", "version": "Jev 1.13" } ], "study_family_id": null, "openness": { "code": { "status": "available", "note": "Controller code; GitHub detects no licence file.", "url": "https://github.com/sc2musa/Jev_Star" }, "weights": { "status": "not_applicable" }, "data": { "status": "partial", "note": "Winning replays are released; full event logs are not in the repository." }, "predictions": { "status": "partial", "note": "Selected decision logs." }, "recomputable": { "status": "no" }, "reproduction": { "status": "not_attempted" } }, "evidence_locator": "Abstract; §§3–4 (Tables 1–6); §5", "priority": "P1", "review_depth": "full_text_and_selected_tables", "background_group": null, "role": null, "claims": [ "c-27331-games", "c-27331-micro", "c-27331-cost" ], "abstract": "We present JEV-Star, a StarCraft II controller that defeats the strongest non-cheating built-in AI, Lv7, by combining fast JEV action selection with persistent GPT-6 planning. The combined system wins four full games at Lv5--Lv7, including two Lv7 victories with different seeds, while retaining a median JEV response time of 0.422 seconds. Mean estimated model cost across these games is USD~3.71 per game: USD~0.15 for JEV and USD~3.56 for GPT-6. We compare this system with an initial JEV-only controller in full-game macro control and multi-unit micromanagement. The standalone controller reaches a 20-minute limit against Lv2 without expanding. Across 35 battle maps with three episodes per map and controller, the combined system raises mean enemy elimination from 16.50\\% to 37.69\\% and wins from 3 to 7 out of 105. Replay frames and decision logs document resource reservation, persistent economic goals, and stable army objectives in successful full games. The results demonstrate a practical division between inexpensive, subsecond decisions and longer-horizon planning. The comparison evaluates complete systems; concurrent interface improvements mean that planning's contribution is not isolated by a controlled ablation. Code is available at https://github.com/sc2musa/Jev_Star" } ] }