{ "license": "Our contributions (summaries, mappings, readings, statuses) are CC BY 4.0, credit: Can Agents Work (canagentswork.com). Each result keeps the terms of its source; credit the source at source_url.", "generated_at": "2026-09-24T00:23:38.241Z", "families": [ { "id": "software-engineering", "name": "Software engineering", "us_employment": 2150380, "occupations": 7, "benchmarks": [ "aider-polyglot", "aidev-dataset", "ambig-swe", "martian-code-review-bench", "codeclash", "cybergym", "gdpval", "gdpval-aa", "impossiblebench", "metr-time-horizons", "pr-arena", "remote-labor-index", "spider-2", "swe-bench-pro", "swe-bench-verified", "swe-lancer", "swe-rebench", "terminal-bench-2", "theagentcompany", "code-survival-study" ], "institutions": [ "aavetis", "aider", "anthropic", "artificial-analysis", "cais", "cmu", "concordia-university", "hku", "laude-institute", "martian", "metr", "nebius", "openai", "princeton", "queens-university", "salesforce", "scale-ai", "stanford", "uc-berkeley" ], "readings": { "task": { "level": "strong", "summary": "On fresh or held-out issue sets, the best agents resolve most tasks: 61.5% on SWE-Bench Pro (public set) and 64.5% on SWE-rebench. Limits: the SWE-Bench Pro private set is lower (51.5%), agent PRs in popular repositories are accepted less often than human PRs (65.3% vs 76.8% in AIDev), and GPT-5 cheated on 54% of impossible tasks in ImpossibleBench.", "evidence": [ "swe-bench-pro", "swe-rebench", "swe-lancer", "aider-polyglot", "martian-code-review-bench", "ambig-swe", "aidev-dataset", "pr-arena", "impossiblebench", "metr-time-horizons" ], "date": "2026-09-23", "institutions": [ "aavetis", "aider", "anthropic", "cmu", "martian", "metr", "nebius", "openai", "queens-university", "scale-ai" ], "thin": false }, "project": { "level": "not-yet", "summary": "Whole projects are mostly out of reach. On the Remote Labor Index, the best agent completed 15.8% of paid freelance projects to a standard a client would accept. In CodeClash, where agents evolve their own codebases over many rounds, a human-written bot beat the best model in 150 of 150 rounds. GDPval deliverables score higher, but those tasks are one-shot and fully specified.", "evidence": [ "remote-labor-index", "codeclash", "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "cais", "openai", "princeton", "scale-ai", "stanford" ], "thin": false } } }, { "id": "devops-sre-it", "name": "DevOps, SRE, and IT operations", "us_employment": 2382350, "occupations": 11, "benchmarks": [ "aiopslab", "iac-eval", "itbench", "itbench-aa", "osworld-verified", "sregym", "terminal-bench-2", "workarena" ], "institutions": [ "artificial-analysis", "cisco", "cmu", "hku", "ibm", "iisc", "laude-institute", "microsoft", "salesforce", "servicenow", "stanford", "uc-berkeley", "uiuc", "university-of-michigan", "university-of-toronto", "university-of-waterloo" ], "readings": { "task": { "level": "partial", "summary": "Mixed. On SREGym, the best agent fixes 72.2% of injected faults end to end. On ITBench-AA, no model names the root cause of more than half of Kubernetes incidents (best 47%). Infrastructure as code has only generation tests with 2024-era results (IaC-Eval, 36.7%). We found no public benchmark with recent results that tests agents making Terraform changes to live infrastructure.", "evidence": [ "sregym", "itbench-aa", "itbench", "aiopslab", "iac-eval", "terminal-bench-2" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "cisco", "ibm", "iisc", "laude-institute", "microsoft", "stanford", "uc-berkeley", "uiuc", "university-of-michigan", "university-of-toronto" ], "thin": false }, "project": { "level": "unexplored", "summary": "No benchmark in our collection tests operations work at project scale, such as a multi-day migration, a platform upgrade, or an on-call rotation.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false } } }, { "id": "security", "name": "Security", "us_employment": 190650, "occupations": 4, "benchmarks": [ "cve-bench", "cybench", "cybergym", "itbench", "metr-time-horizons" ], "institutions": [ "ibm", "metr", "stanford", "uc-berkeley", "uiuc" ], "readings": { "task": { "level": "partial", "summary": "Mixed. Capture-the-flag and vulnerability-reproduction tasks are near their ceiling: Anthropic reports 100% on its 35-task Cybench subset, and the top CyberGym entry reproduces 98.5% of vulnerabilities (Level 1). Exploitation of real web applications is not shown: the last published CVE-Bench result is 12.5% (GPT-4o, 2025), and we found no newer frontier result.", "evidence": [ "cybench", "cybergym", "cve-bench" ], "date": "2026-09-23", "institutions": [ "stanford", "uc-berkeley", "uiuc" ], "thin": false }, "project": { "level": "unexplored", "summary": "No benchmark in our collection tests security work at project scale, such as a full penetration test, a security review of a whole system, or a compliance audit.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false } } }, { "id": "data-analytics", "name": "Data and analytics", "us_employment": 499540, "occupations": 13, "benchmarks": [ "gaia", "mle-bench", "remote-labor-index", "spider-2", "spreadsheetbench" ], "institutions": [ "cais", "hku", "hugging-face", "meta", "openai", "renmin-university", "salesforce", "scale-ai" ], "readings": { "task": { "level": "partial", "summary": "One direct benchmark, near its ceiling. On Spider 2.0-Snow, which asks for multi-step SQL against large business databases, the top agent reaches 96.7% execution accuracy, but the entry names no model. Adjacent spreadsheet tasks agree (83.1% on SpreadsheetBench). No benchmark in our collection tests open-ended analysis, where the right question is itself unclear.", "evidence": [ "spider-2", "spreadsheetbench", "gaia" ], "date": "2026-09-23", "institutions": [ "hku", "hugging-face", "meta", "renmin-university", "salesforce" ], "thin": false }, "project": { "level": "partial", "summary": "Mixed, adjacent evidence. On MLE-bench, 75 offline Kaggle competitions, the best agent wins a medal in 64.4% of competitions. The Remote Labor Index includes data analysis projects; across all its projects, the best agent completed 15.8% to a standard a client would accept.", "evidence": [ "mle-bench", "remote-labor-index" ], "date": "2026-09-23", "institutions": [ "cais", "openai", "scale-ai" ], "thin": false } } }, { "id": "ml-research-engineering", "name": "ML and research engineering", "us_employment": 37200, "occupations": 1, "benchmarks": [ "core-bench", "metr-time-horizons", "mle-bench", "paperbench", "re-bench", "terminal-bench-2" ], "institutions": [ "laude-institute", "metr", "openai", "princeton", "stanford" ], "readings": { "task": { "level": "partial", "summary": "Mixed, and partly stale. On RE-Bench, METR's AI research engineering tasks, the best recorded agent matched the 37th percentile of human experts at an 8-hour budget, but that result is from January 2025. CORE-Bench, which asks agents to reproduce published results from the authors' code, was declared solved in December 2025 (95.5% after manual regrading).", "evidence": [ "re-bench", "core-bench", "metr-time-horizons", "terminal-bench-2" ], "date": "2026-09-23", "institutions": [ "laude-institute", "metr", "princeton", "stanford" ], "thin": false }, "project": { "level": "partial", "summary": "Mixed. On MLE-bench, 75 offline Kaggle competitions, the best agent wins a medal in 64.4% of them. On PaperBench, where agents replicate ML papers from scratch, the best official result is 26.0% (o1, April 2025); on a small subset, ML PhDs scored 41.4%. PaperBench has no newer official results.", "evidence": [ "mle-bench", "paperbench" ], "date": "2026-09-23", "institutions": [ "openai" ], "thin": true } } }, { "id": "customer-support", "name": "Customer support", "us_employment": 2595750, "occupations": 1, "benchmarks": [ "crmarena-pro", "gdpval", "tau2-bench" ], "institutions": [ "openai", "salesforce", "sierra" ], "readings": { "task": { "level": "partial", "summary": "Mixed. In the telecom domain of Sierra's τ²-bench, where the agent guides a simulated customer through a phone fix, the best agent succeeds on 97.8% of tasks on the first try and 92.1% on all four tries. On Salesforce's CRMArena-Pro, the best 2025 result is 58.3% in single-turn tasks and lower when a simulated customer holds back details.", "evidence": [ "tau2-bench", "crmarena-pro" ], "date": "2026-09-23", "institutions": [ "salesforce", "sierra" ], "thin": false }, "project": { "level": "partial", "summary": "Thin, adjacent evidence only. GDPval, which spans 44 occupations, is the one project-level benchmark in our collection mapped here. OpenAI reports that GPT-5.5's deliverables won or tied against professionals' work in 84.9% of comparisons across all GDPval occupations.", "evidence": [ "gdpval" ], "date": "2026-09-23", "institutions": [ "openai" ], "thin": true } } }, { "id": "sales-marketing", "name": "Sales and marketing", "us_employment": 14323860, "occupations": 25, "benchmarks": [ "crmarena-pro", "gdpval", "gdpval-aa" ], "institutions": [ "artificial-analysis", "openai", "salesforce" ], "readings": { "task": { "level": "partial", "summary": "One benchmark, from 2025. On Salesforce's CRMArena-Pro, which tests sales, service, and configure-price-quote requests inside a CRM, the best recorded agent succeeds on 58.3% of single-turn tasks and on fewer multi-turn tasks (Gemini 2.5 Pro, May 2025). No newer results are published.", "evidence": [ "crmarena-pro" ], "date": "2026-09-23", "institutions": [ "salesforce" ], "thin": true }, "project": { "level": "partial", "summary": "Only adjacent evidence. GDPval includes sales managers among its 44 occupations. OpenAI reports that GPT-5.5's deliverables won or tied against professionals' work in 84.9% of comparisons across all occupations. No benchmark in our collection tests running a campaign or a sales pipeline over weeks.", "evidence": [ "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "openai" ], "thin": false } } }, { "id": "office-admin", "name": "Office and administrative support", "us_employment": 15157670, "occupations": 55, "benchmarks": [ "browsecomp", "gaia", "gdpval", "gdpval-aa", "osworld-verified", "spreadsheetbench", "theagentcompany", "workarena" ], "institutions": [ "artificial-analysis", "cmu", "hku", "hugging-face", "meta", "openai", "renmin-university", "salesforce", "servicenow", "university-of-waterloo" ], "readings": { "task": { "level": "strong", "summary": "On computer-use and office-task benchmarks, top agents pass most tasks: 90.2% on OSWorld-Verified (humans: 72.4%), 83.1% on SpreadsheetBench, and 63.3% on WorkArena L1 under the standard protocol. Limits: many top entries are self-submitted and not re-run by the maintainers, and on TheAgentCompany's multi-step workplace tasks the best agent resolves 42.9%.", "evidence": [ "osworld-verified", "spreadsheetbench", "workarena", "gaia", "browsecomp", "theagentcompany" ], "date": "2026-09-23", "institutions": [ "cmu", "hku", "hugging-face", "meta", "openai", "renmin-university", "salesforce", "servicenow", "university-of-waterloo" ], "thin": false }, "project": { "level": "partial", "summary": "Only adjacent evidence. OpenAI reports that GPT-5.5's GDPval deliverables won or tied against professionals' work in 84.9% of comparisons, but GDPval tasks are one-shot and fully specified. No benchmark in our collection tests multi-day administrative work, such as running a calendar or a purchasing process.", "evidence": [ "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "openai" ], "thin": false } } }, { "id": "finance-accounting", "name": "Finance and accounting", "us_employment": 3051320, "occupations": 17, "benchmarks": [ "apex-agents", "vals-finance-agent", "gdpval", "gdpval-aa", "spreadsheetbench" ], "institutions": [ "artificial-analysis", "box", "harvey", "mercor", "openai", "renmin-university", "vals-ai" ], "readings": { "task": { "level": "partial", "summary": "On Vals AI's Finance Agent benchmark, where agents research SEC filings to answer an analyst's questions, the best model scores 64.4%. Adjacent spreadsheet tasks score higher (83.1% on SpreadsheetBench). No task-level benchmark in our collection tests bookkeeping or audit work inside accounting systems.", "evidence": [ "vals-finance-agent", "spreadsheetbench" ], "date": "2026-09-23", "institutions": [ "renmin-university", "vals-ai" ], "thin": false }, "project": { "level": "partial", "summary": "On APEX-Agents, which includes investment-banking tasks written by professionals, the best agent passes 73.5% of tasks on the first try (all domains). OpenAI reports that GPT-5.5's GDPval deliverables, from occupations that include accountants and financial managers, won or tied against professionals' work in 84.9% of comparisons.", "evidence": [ "apex-agents", "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "box", "harvey", "mercor", "openai" ], "thin": false } } }, { "id": "legal", "name": "Legal", "us_employment": 1305620, "occupations": 8, "benchmarks": [ "apex-agents", "gdpval", "gdpval-aa" ], "institutions": [ "artificial-analysis", "box", "harvey", "mercor", "openai" ], "readings": { "task": { "level": "unexplored", "summary": "No task-level benchmark in our collection tests legal work, such as contract review or legal research with tools.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false }, "project": { "level": "partial", "summary": "Only adjacent evidence. APEX-Agents includes corporate-law tasks: across banking, consulting, and law, the best agent passes 73.5% on the first try. GDPval includes lawyers among its 44 occupations. Neither benchmark tests work with real clients, courts, or deadlines.", "evidence": [ "apex-agents", "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "box", "harvey", "mercor", "openai" ], "thin": false } } }, { "id": "management-business", "name": "Management and business operations", "us_employment": 17705890, "occupations": 91, "benchmarks": [ "apex-agents", "gdpval", "gdpval-aa", "theagentcompany", "vending-bench-2" ], "institutions": [ "andon-labs", "artificial-analysis", "box", "cmu", "harvey", "mercor", "openai" ], "readings": { "task": { "level": "partial", "summary": "Only adjacent evidence. On TheAgentCompany, which gives agents workplace tasks inside a simulated software company, the best agent resolves 42.9% of tasks (a November 2025 result). No task-level benchmark in our collection maps directly to management or business operations.", "evidence": [ "theagentcompany" ], "date": "2026-09-23", "institutions": [ "cmu" ], "thin": true }, "project": { "level": "partial", "summary": "Mixed. OpenAI reports that GPT-5.5's GDPval deliverables won or tied against professionals' work in 84.9% of comparisons, and the best agent passes 73.5% of APEX-Agents tasks on the first try. Running a business is weaker: on Vending-Bench 2 the best model ends a simulated year with $15,515, about a quarter of Andon Labs' estimate for a good strategy.", "evidence": [ "gdpval", "gdpval-aa", "apex-agents", "vending-bench-2" ], "date": "2026-09-23", "institutions": [ "andon-labs", "artificial-analysis", "box", "harvey", "mercor", "openai" ], "thin": false } } }, { "id": "design-media-writing", "name": "Design, media, and writing", "us_employment": 2046270, "occupations": 45, "benchmarks": [ "gdpval", "gdpval-aa", "osworld-verified", "remote-labor-index" ], "institutions": [ "artificial-analysis", "cais", "cmu", "hku", "openai", "salesforce", "scale-ai", "university-of-waterloo" ], "readings": { "task": { "level": "partial", "summary": "Only adjacent evidence. OSWorld-Verified includes tasks in image, slide, and document apps, and the best agent passes 90.2% of all its computer tasks. No task-level benchmark in our collection scores the quality of design or writing work.", "evidence": [ "osworld-verified" ], "date": "2026-09-23", "institutions": [ "cmu", "hku", "salesforce", "university-of-waterloo" ], "thin": false }, "project": { "level": "not-yet", "summary": "On the Remote Labor Index, which includes graphic design, video, 3D, audio, and web projects from freelance platforms, the best agent completed 15.8% of projects to a standard a client would accept. GDPval deliverables score higher (84.9% wins or ties against professionals, reported by OpenAI), but those tasks are one-shot and fully specified.", "evidence": [ "remote-labor-index", "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "cais", "openai", "scale-ai" ], "thin": false } } }, { "id": "architecture-engineering", "name": "Architecture and engineering", "us_employment": 2617130, "occupations": 59, "benchmarks": [ "gdpval", "gdpval-aa", "remote-labor-index" ], "institutions": [ "artificial-analysis", "cais", "openai", "scale-ai" ], "readings": { "task": { "level": "unexplored", "summary": "No task-level benchmark in our collection tests architecture or engineering design work, such as CAD modeling or engineering calculations.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false }, "project": { "level": "not-yet", "summary": "Whole projects are mostly out of reach. The Remote Labor Index includes 3D, CAD, and architecture projects; across all its projects, the best agent completed 15.8% to a standard a client would accept. GDPval includes mechanical engineers, and its deliverables score higher, but those tasks are one-shot and fully specified.", "evidence": [ "remote-labor-index", "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "cais", "openai", "scale-ai" ], "thin": false } } }, { "id": "healthcare", "name": "Healthcare", "us_employment": 17725580, "occupations": 116, "benchmarks": [ "gdpval", "gdpval-aa", "medagentbench" ], "institutions": [ "artificial-analysis", "openai", "stanford" ], "readings": { "task": { "level": "partial", "summary": "One benchmark, from early 2025. On Stanford's MedAgentBench, where agents do clinical tasks in a simulated electronic health record, the best recorded model completes 69.7% of tasks (Claude 3.5 Sonnet v2). It has no maintained leaderboard, so newer models are not measured. No benchmark in our collection tests work with patients.", "evidence": [ "medagentbench" ], "date": "2026-09-23", "institutions": [ "stanford" ], "thin": true }, "project": { "level": "partial", "summary": "Only adjacent evidence. GDPval includes nurses among its 44 occupations. OpenAI reports that GPT-5.5's deliverables won or tied against professionals' work in 84.9% of comparisons across all occupations. No benchmark in our collection tests clinical work over days, such as managing a patient's care.", "evidence": [ "gdpval", "gdpval-aa" ], "date": "2026-09-23", "institutions": [ "artificial-analysis", "openai" ], "thin": false } } }, { "id": "science-research", "name": "Science and research", "us_employment": 1473280, "occupations": 66, "benchmarks": [ "core-bench", "paperbench" ], "institutions": [ "openai", "princeton" ], "readings": { "task": { "level": "partial", "summary": "Thin evidence. CORE-Bench asks agents to reproduce the results of published papers in computer science, social science, and medicine from the authors' code and data. Princeton's HAL team declared it solved in December 2025: Claude Code with Claude Opus 4.5 scored 95.5% after manual regrading. No benchmark in our collection tests new experiments or lab work.", "evidence": [ "core-bench" ], "date": "2026-09-23", "institutions": [ "princeton" ], "thin": true }, "project": { "level": "not-yet", "summary": "Thin evidence. On PaperBench, where agents replicate machine learning papers from scratch, the best official result is 26.0% (o1, April 2025). On a small subset, ML PhDs scored 41.4% against o1's 26.6%. No newer official results exist, and no benchmark tests research projects in other sciences.", "evidence": [ "paperbench" ], "date": "2026-09-23", "institutions": [ "openai" ], "thin": true } } }, { "id": "education-social-services", "name": "Education and social services", "us_employment": 11769270, "occupations": 86, "benchmarks": [ "gdpval" ], "institutions": [ "openai" ], "readings": { "task": { "level": "unexplored", "summary": "No task-level benchmark in our collection tests teaching, counseling, or casework.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false }, "project": { "level": "partial", "summary": "Thin, adjacent evidence only. GDPval, which spans 44 occupations, is the one benchmark in our collection mapped here. OpenAI reports that GPT-5.5's deliverables won or tied against professionals' work in 84.9% of comparisons across all GDPval occupations. No benchmark tests work with students or clients.", "evidence": [ "gdpval" ], "date": "2026-09-23", "institutions": [ "openai" ], "thin": true } } }, { "id": "physical-work", "name": "Physical work", "us_employment": 60450080, "occupations": 392, "benchmarks": [], "institutions": [], "readings": { "task": { "level": "unexplored", "summary": "No benchmark in our collection covers this work yet.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false }, "project": { "level": "unexplored", "summary": "No benchmark in our collection covers this work yet.", "evidence": [], "date": "2026-09-23", "institutions": [], "thin": false } } } ], "benchmarks": [ { "id": "aider-polyglot", "name": "Aider Polyglot leaderboard", "short_name": "Aider Polyglot", "kind": "leaderboard", "institutions": [ "aider" ], "authors": [ "Paul Gauthier" ], "summary": "225 of the hardest Exercism practice exercises in C++, Go, Java, JavaScript, Python, and Rust, run through the aider coding tool. Paul Gauthier picked the exercises that a set of seven models in late 2024 mostly failed, so that top models would spread out between roughly 5% and 50%.", "what_is_measured": "Share of exercises where the model edits the starter code so that the hidden unit tests pass. The model gets the exercise text and the starter files. Runs allow two tries; the leaderboard reports the pass rate after each try and uses the second as the headline. It also reports how often the model used the requested edit format and the run cost.", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 225, "human_reference": null, "metric": { "name": "Percent correct", "unit": "percent", "higher_is_better": true, "description": "Percentage of the 225 exercises completed correctly within two attempts." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "feature-development" ], "links": { "homepage": "https://aider.chat/docs/leaderboards/", "leaderboard": "https://aider.chat/docs/leaderboards/", "repo": "https://github.com/Aider-AI/polyglot-benchmark", "blog": "https://aider.chat/2024/12/21/polyglot.html" }, "license": { "data": "Exercises are copyright Exercism and used under the Exercism tracks' open-source licenses. The benchmark repository states no license of its own.", "reuse": "cite-only" }, "contamination": "High risk. Exercism exercises and many solutions are public on GitHub and have been for years.", "limits": [ { "text": "Small, self-contained exercises with clear specs, not real codebases or issues. The benchmark measures code editing and instruction following inside one tool.", "source_url": "https://aider.chat/2024/12/21/polyglot.html" }, { "text": "The leaderboard stopped growing in late 2025. The newest entry is dated 2025-10-03 and the page was last updated 2025-11-20, so models released since are absent.", "source_url": "https://aider.chat/docs/leaderboards/" }, { "text": "Scores depend on aider's edit formats and prompts. A model that ignores the edit format loses points even if its code is right.", "source_url": "https://aider.chat/docs/leaderboards/notes.html" }, { "text": "Designed to give headroom to 50%. With the top score at 88%, the benchmark has only 27 unsolved exercises left for the leader.", "source_url": "https://aider.chat/docs/leaderboards/" } ], "released": "2024-12-21", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://aider.chat/docs/leaderboards/", "https://aider.chat/docs/leaderboards/notes.html", "https://aider.chat/2024/12/21/polyglot.html", "https://github.com/Aider-AI/polyglot-benchmark" ], "spot_checked": false, "notes": "Aider is an open-source project run by Paul Gauthier. Some rows on the leaderboard are community submissions; the top row (gpt-5 high, 88.0%) matches the number OpenAI gave in its GPT-5 developer post. No sign that the maintainer retired the leaderboard, so no status_override." }, "institution_names": [ "Aider" ], "status": "strong", "frontier": { "subject": { "label": "gpt-5 (high)", "model": "GPT-5", "model_org": "OpenAI", "harness": "aider", "open_weights": false }, "value": 88, "unit": "percent", "date": "2025-08-23", "frontier": true, "source_url": "https://aider.chat/docs/leaderboards/", "source_kind": "primary", "reported_by": "aider", "retrieved_at": "2026-09-23", "notes": "Top row of the leaderboard on 2026-09-23. 198 of 225 correct, diff edit format, 91.6% correct edit format, $29.08 total cost, aider 0.86.2.dev. No newer entries since 2025-10-03." } }, { "id": "aidev-dataset", "name": "AIDev", "short_name": "AIDev", "kind": "dataset", "institutions": [ "queens-university" ], "authors": [ "Hao Li", "Haoxiang Zhang", "Ahmed E. Hassan" ], "summary": "A public dataset of pull requests that autonomous coding agents (OpenAI Codex, Devin, GitHub Copilot, Cursor, Claude Code) opened on GitHub, with review timelines, code changes, and merge outcomes. The paper compares agent PRs with human PRs in popular repositories.", "what_is_measured": "How agent-authored pull requests fare in real open-source review. The headline is the acceptance rate: the share of PRs in the AIDev-pop subset (popular repositories) that were merged, per agent, against a human baseline from the same repositories. The paper also reports task type mix, review turnaround time, and who reviews the PRs. The paper version covers 456,535 agent PRs across 61,453 repositories; the refreshed dataset (to 2025-08-01) has 932,791 PRs.", "grain": "task", "environment": [ "field-data" ], "grading": [ "field-signal" ], "task_count": null, "human_reference": "Human-authored PRs in the same AIDev-pop repositories: 76.8% accepted, 16.1% rejected (Table 5).", "metric": { "name": "PR acceptance rate (AIDev-pop)", "unit": "percent", "higher_is_better": true, "description": "Share of an agent's pull requests in popular repositories that were merged, from Table 5 of the paper." }, "reference_rule": { "kind": "none", "note": "Field signal from observational data. The human baseline of 76.8% is shown for context, not as a parity line." }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "shipping-prs" ], "links": { "paper": "https://arxiv.org/abs/2507.15003", "repo": "https://github.com/SAILResearch/AI_Teammates_in_SE3", "dataset": "https://huggingface.co/datasets/hao-li/AIDev" }, "license": { "data": "Not checked in this session. Dataset is on Hugging Face and Zenodo (DOI 10.5281/zenodo.16919272).", "reuse": "unknown" }, "contamination": null, "limits": [ { "text": "Observational data. Agents are used by different people on different repositories, so acceptance rates mix agent quality with user selection and repository norms.", "source_url": "https://arxiv.org/abs/2507.15003" }, { "text": "Agents are found by GitHub search patterns (for example head:codex/, Co-Authored-By: Claude), which miss PRs that do not carry the pattern. Cursor and Claude Code are mostly used locally, so their GitHub footprint is small.", "source_url": "https://arxiv.org/html/2507.15003v1" }, { "text": "The paper's numbers were captured about two months after several agents launched (Codex started 2025-05-16). The authors say the gaps show how early the transition is.", "source_url": "https://arxiv.org/html/2507.15003v1" }, { "text": "The paper defines AIDev-pop as repositories with more than 500 stars; the repo README says more than 100 stars and gives different PR counts. We use the paper's Table 5.", "source_url": "https://github.com/SAILResearch/AI_Teammates_in_SE3" } ], "released": "2025-07-20", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2507.15003", "https://arxiv.org/html/2507.15003v1", "https://raw.githubusercontent.com/SAILResearch/AI_Teammates_in_SE3/HEAD/README.md" ], "spot_checked": false, "notes": "All three authors are at Queen's University, Kingston, Canada (SAIL lab). The paper text says Codex has \"the highest acceptance rate at 64%\" (Figure 3, per task category) while Table 5 gives 65.3% accepted; the two figures use different groupings. We show Table 5. In Table 5 accepted and rejected do not sum to 100 because some PRs were still open." }, "institution_names": [ "Queen's University" ], "status": "unrated", "frontier": { "subject": { "label": "OpenAI Codex", "model_org": "OpenAI", "agent": "OpenAI Codex" }, "value": 65.3, "unit": "percent", "date": "2025-07-20", "frontier": true, "source_url": "https://arxiv.org/html/2507.15003v1", "source_kind": "primary", "reported_by": "queens-university", "retrieved_at": "2026-09-23", "notes": "Table 5, AIDev-pop. 21.3% rejected. Accepted Codex PRs closed in a median 0.3 hours versus 3.9 hours for human PRs. Human baseline 76.8% accepted. Body text gives 64% from Figure 3." } }, { "id": "aiopslab", "name": "AIOpsLab", "short_name": "AIOpsLab", "kind": "benchmark", "institutions": [ "microsoft", "uiuc", "uc-berkeley", "iisc" ], "summary": "A framework from Microsoft Research and university partners that deploys microservice applications on Kubernetes, injects faults, generates load, and exposes telemetry so that AI agents can be tested on cloud operations tasks. The paper ships a suite of 48 problems.", "what_is_measured": "Whether an agent answers each operations problem correctly. Problems sit at four levels: detect that something is wrong, localize the faulty service, analyze the root cause, and mitigate the fault in the live cluster. The headline is accuracy across all 48 problems. The framework also records time, steps, and tokens.", "grain": "task", "environment": [ "live-system", "cli", "api-tools" ], "grading": [ "state-check", "automated-tests" ], "task_count": 48, "human_reference": null, "metric": { "name": "Accuracy", "unit": "percent", "higher_is_better": true, "description": "Share of the 48 problems that the agent solves correctly across detection, localization, root cause analysis, and mitigation." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "devops-sre-it", "weight": "primary" } ], "functions": [ "incident-response", "terminal-operations" ], "links": { "paper": "https://arxiv.org/abs/2501.06706", "repo": "https://github.com/microsoft/AIOpsLab" }, "license": { "data": "MIT (GitHub repository license).", "code": "MIT", "reuse": "open-mit" }, "contamination": null, "limits": [ { "text": "Published results cover four agents built on GPT-4-turbo and GPT-3.5-turbo from early 2025. There is no public leaderboard, so newer models are not tracked here.", "source_url": "https://arxiv.org/abs/2501.06706" }, { "text": "The problem set in the paper is 48 problems from a small number of faults across a few applications. The repository has since added more problems.", "source_url": "https://github.com/microsoft/AIOpsLab" }, { "text": "SREGym, which ports the AIOpsLab problems, reports that strong agents now mitigate more than 80% of these ported problems.", "source_url": "https://arxiv.org/abs/2605.07161" } ], "released": "2025-01-12", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2501.06706", "https://arxiv.org/html/2501.06706v1", "https://github.com/microsoft/AIOpsLab" ], "spot_checked": false, "notes": "Affiliations from the paper: UIUC, UC Berkeley, Microsoft (Redmond), IISc (Bengaluru). Published at MLSys 2025 per the repo citation. An earlier vision paper appeared at SoCC 2024 (arXiv 2407.12165). The repo was created 2024-10-11." }, "institution_names": [ "Microsoft Research", "UIUC", "UC Berkeley", "IISc" ], "status": "emerging", "frontier": { "subject": { "label": "Flash", "model": null, "model_org": null, "agent": "Flash" }, "value": 59.32, "unit": "percent", "date": "2025-01-12", "frontier": true, "source_url": "https://arxiv.org/abs/2501.06706", "source_kind": "primary", "reported_by": "microsoft", "retrieved_at": "2026-09-23", "notes": "Best accuracy in the paper. Flash is the authors' simplified re-implementation of a workflow agent with hindsight generation; the paper does not state its backbone model in the results table." } }, { "id": "ambig-swe", "name": "Ambig-SWE", "kind": "benchmark", "institutions": [ "cmu" ], "authors": [ "Sanidhya Vijayvargiya", "Xuhui Zhou", "Akhila Yerukola", "Maarten Sap", "Graham Neubig" ], "summary": "An underspecified variant of SWE-Bench Verified. Each of the 500 GitHub issues is rewritten by GPT-4o to drop key details. The agent, running in OpenHands, can ask a simulated user (GPT-4o with the full issue) for clarification. The study asks whether agents notice missing information, ask useful questions, and use the answers to fix the issue.", "what_is_measured": "Three steps. (1) Detection: given a mix of full and underspecified issues, does the agent ask for clarification only when it needs to? Scored as accuracy with false positive and false negative rates, under three prompts of rising encouragement to interact. (2) Question quality. (3) Resolve rate on underspecified issues in three settings: Hidden (no interaction), Interaction (agent may ask the user proxy), and Full (original issue). The headline here is the detection accuracy under the model's best prompt, because it measures the clarification behavior directly. Chance is 50%.", "grain": "task", "environment": [ "repo", "chat" ], "grading": [ "automated-tests", "outcome-metric" ], "task_count": 500, "human_reference": null, "metric": { "name": "Underspecificity detection accuracy (best prompt)", "unit": "percent", "higher_is_better": true, "description": "Accuracy at deciding whether an issue needs clarification, over a balanced mix of full and underspecified SWE-Bench Verified issues. Table 2 of the paper. Chance is 50%." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "clarification", "issue-resolution" ], "links": { "paper": "https://arxiv.org/abs/2502.13069", "repo": "https://github.com/sani903/InteractiveSWEAgents" }, "license": { "data": "Paper CC BY 4.0; repo MIT. Issues derive from SWE-Bench Verified.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Built on SWE-Bench Verified, which models may have trained on. The authors note that some models do well in the Hidden setting possibly through data leakage.", "limits": [ { "text": "Underspecified issues are synthetic GPT-4o summaries of well-specified issues, and the user is a GPT-4o proxy, not a real developer.", "source_url": "https://arxiv.org/html/2502.13069v3" }, { "text": "Detection results depend heavily on the prompt. Claude Sonnet 4 scores 74% with a neutral prompt and 89% with strong encouragement; Claude Sonnet 3.5 peaks at 84% with moderate encouragement and drops to 76% with strong encouragement.", "source_url": "https://arxiv.org/html/2502.13069v3" }, { "text": "Six models were tested (Claude Sonnet 4, Claude Sonnet 3.5, Claude Haiku 3.5, Qwen 3 Coder 480B, Deepseek-v2, Llama 3.1 70B). No models newer than mid-2025 appear in the ICLR 2026 version.", "source_url": "https://arxiv.org/html/2502.13069v3" }, { "text": "Claude Sonnet 4 was run on 100 of 500 instances in the Hidden setting because of cost.", "source_url": "https://arxiv.org/html/2502.13069v3" } ], "released": "2025-02-18", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2502.13069", "https://arxiv.org/html/2502.13069v3", "https://api.github.com/repos/sani903/InteractiveSWEAgents" ], "spot_checked": false, "notes": "All authors list the Language Technologies Institute, Carnegie Mellon University. Accepted at ICLR 2026 (v3, 2026-02-21). Resolve rates from Figure 3 (bars labelled with values): Claude Sonnet 4 Hidden 40.00, Interaction 61.40, Full 68.00; Qwen 3 Coder 45.60 / 53.80 / 64.60; Claude Sonnet 3.5 24.20 / 39.60 / 49.40. We chose detection accuracy as the headline because the atlas tags this benchmark under clarification; the resolve-rate view is in the notes." }, "institution_names": [ "Carnegie Mellon University" ], "status": "strong", "frontier": { "subject": { "label": "Claude Sonnet 4", "model": "Claude Sonnet 4", "model_org": "Anthropic", "harness": "OpenHands", "open_weights": false }, "value": 89, "unit": "percent", "date": "2026-02-21", "frontier": true, "source_url": "https://arxiv.org/html/2502.13069v3", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23", "notes": "Table 2, Strong Encouragement prompt: accuracy 0.89, FPR 0.03, FNR 0.18. Resolve rate on underspecified issues 61.40% with interaction versus 40.00% without and 68.00% with the full issue (Figure 3). Date is the v3 paper date (ICLR 2026 version)." } }, { "id": "apex-agents", "name": "APEX-Agents", "kind": "benchmark", "institutions": [ "mercor", "box", "harvey" ], "authors": [ "Bertie Vidgen", "Austin Mann", "Abby Fennelly", "John Wright Stanly", "Lucas Rothman", "Marco Burstein", "Osvald Nitski" ], "summary": "Long, multi-application work tasks written by investment banking analysts, management consultants, and corporate lawyers. Each task sits inside a simulated project \"world\" with files, spreadsheets, and chat threads. An agent must find the right information and produce a client-ready output.", "what_is_measured": "Whether an agent completes a professional task end to end inside a realistic workspace. Experts from top firms built 31 worlds (for example a week-long consulting project for a fictional oil and gas company) and wrote 240 tasks (80 per job) with 1 to 10 pass or fail criteria each. Version 1.1 (September 2026) tightened task specifications, added a judge that gives zero credit for hedged multiple answers, and fixed tool reliability.", "grain": "project", "environment": [ "simulated-workplace", "documents", "api-tools" ], "grading": [ "rubric", "llm-judge" ], "task_count": 240, "human_reference": null, "metric": { "name": "Pass@1", "unit": "percent", "higher_is_better": true, "description": "Share of tasks where the agent passes every rubric criterion on a single attempt. Mercor also reports Mean Score, the average share of criteria passed per task." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "finance-accounting", "weight": "primary" }, { "family": "management-business", "weight": "partial" }, { "family": "legal", "weight": "partial" } ], "functions": [ "financial-analysis", "consulting-work", "legal-work", "professional-deliverables" ], "links": { "homepage": "https://www.mercor.com/apex/apex-agents-leaderboard/", "paper": "https://arxiv.org/abs/2601.14242", "leaderboard": "https://www.mercor.com/apex/apex-agents-leaderboard/", "repo": "https://github.com/Mercor-Intelligence/apex_loop_truncated_tools_agent", "blog": "https://www.mercor.com/blog/introducing-apex-agents-1-1", "dataset": "https://huggingface.co/datasets/mercor/apex-agents-v1.1" }, "license": { "data": "CC BY 4.0 on the Hugging Face dataset. Mercor says the full task set used for the leaderboard stays private.", "code": "Evaluation agent and Archipelago infrastructure are open source on GitHub.", "reuse": "open-cc-by" }, "contamination": "Mercor says the full task set stays private so that models cannot be trained on it, while an open subset is public. The exact split between public and private tasks is not stated on the leaderboard page.", "limits": [ { "text": "Grading uses an LLM judge (DeepSeek-V4-Flash-0731 in v1.1) against expert rubrics, not human review of each output. Mercor reports the judge's false negative rate rose from 5.3% to 8.0% in v1.1.", "source_url": "https://www.mercor.com/blog/introducing-apex-agents-1-1" }, { "text": "Version 1.1 changed tasks (480 to 240), grading, and prompts, so scores before September 2026 are not comparable with the current board.", "source_url": "https://www.mercor.com/blog/introducing-apex-agents-1-1" }, { "text": "Confidence intervals are about plus or minus 5 percentage points, so the top five models overlap.", "source_url": "https://www.mercor.com/apex/apex-agents-leaderboard/" }, { "text": "Only three jobs are covered, and all worlds run in a Google Workspace style environment with Mercor's own agent loop.", "source_url": "https://www.mercor.com/blog/introducing-apex-agents/" } ], "released": "2026-01-21", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://www.mercor.com/apex/apex-agents-leaderboard/", "https://www.mercor.com/blog/introducing-apex-agents/", "https://www.mercor.com/blog/introducing-apex-agents-1-1", "https://arxiv.org/abs/2601.14242" ], "spot_checked": true, "notes": "Leaderboard values were read from the page's embedded JSON (Pass@1 and Mean Score for the \"Loop\" harness). The page FAQ calls Mean Score the primary ranking metric, but the paper, both blog posts, and the default page view use Pass@1, so we show Pass@1 and give Mean Score in notes." }, "institution_names": [ "Mercor", "Box", "Harvey" ], "status": "strong", "frontier": { "subject": { "label": "Claude Opus 5.5 (max)", "model": "Claude Opus 5.5", "model_org": "Anthropic", "harness": "Mercor Loop agent", "open_weights": false }, "value": 73.5, "unit": "percent", "ci": [ 68.6, 78.4 ], "date": "2026-09-23", "frontier": true, "source_url": "https://www.mercor.com/apex/apex-agents-leaderboard/", "source_kind": "primary", "reported_by": "mercor", "retrieved_at": "2026-09-23", "notes": "Rank 1 on the APEX-Agents 1.1 board, seen 2026-09-23. Pass@1 73.5% plus or minus 4.9; Mean Score 81.3% plus or minus 4.0; 950 samples. Model release date shown as 2026-09-22. Graded by an LLM judge against expert rubrics." } }, { "id": "browsecomp", "name": "BrowseComp", "kind": "benchmark", "institutions": [ "openai" ], "authors": [ "Jason Wei", "Zhiqing Sun", "Spencer Papay", "Scott McKinney", "Jeffrey Han", "Isa Fulford", "Hyung Won Chung", "Alex Tachard Passos", "William Fedus", "Amelia Glaese" ], "summary": "1,266 hard fact-finding questions with short, single answers. Each answer is hard to find but easy to verify, so a browsing agent must search persistently and combine clues from many sites.", "what_is_measured": "Share of questions answered correctly. Human trainers wrote \"inverted\" questions from a seed fact and several constraints (for example, find a paper by the authors' universities and venue). Every question was checked to be unsolvable by GPT-4o, o1, and an early deep research model at the time, and not findable on the first page of five simple searches. Grading compares the short answer with the reference.", "grain": "task", "environment": [ "browser" ], "grading": [ "automated-tests" ], "task_count": 1266, "human_reference": "Trainers who did not write the question solved 29.2% of problems within a two-hour limit and without AI help. Of solved problems, 86.4% matched the reference answer.", "metric": { "name": "Accuracy", "unit": "percent", "higher_is_better": true, "description": "Share of the 1,266 questions answered correctly." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "office-admin", "weight": "primary" } ], "functions": [ "web-research" ], "links": { "homepage": "https://openai.com/index/browsecomp/", "paper": "https://arxiv.org/abs/2504.12516", "repo": "https://github.com/openai/simple-evals" }, "license": { "data": "Released in OpenAI's simple-evals repository, which lists BrowseComp under the MIT License.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Questions and answers are public in the simple-evals repository, so later models may have trained on them. OpenAI has not published a contamination analysis.", "limits": [ { "text": "Answers are short strings, so the benchmark does not test long answers, ambiguity, or real user queries. OpenAI calls it an incomplete but useful measure of browsing.", "source_url": "https://openai.com/index/browsecomp/" }, { "text": "OpenAI builds the benchmark and reports its own models. Later scores come from OpenAI launch posts, and the top score uses four agents running in parallel.", "source_url": "https://openai.com/index/gpt-5-6/" }, { "text": "The best scores are now above 90%, so the benchmark is close to saturation.", "source_url": "https://openai.com/index/gpt-6-astra/" }, { "text": "Deep Research, the launch leader, was trained on data that targets BrowseComp-style tasks.", "source_url": "https://openai.com/index/browsecomp/" } ], "released": "2025-04", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://openai.com/index/browsecomp/", "https://arxiv.org/abs/2504.12516", "https://openai.com/index/introducing-gpt-5-5/", "https://openai.com/index/gpt-5-6/", "https://openai.com/index/gpt-6-astra/" ], "spot_checked": true, "notes": "The blog page does not show its publication date in the text we fetched; the arXiv v1 is dated 2025-04-16, so released is given at month precision. Author list shows only the arXiv submitter; the full author list was not visible on the abstract page. Grader details in simple-evals were not opened." }, "institution_names": [ "OpenAI" ], "status": "saturated", "frontier": { "subject": { "label": "GPT-5.6 Sol Ultra", "model": "GPT-5.6 Sol", "model_org": "OpenAI", "harness": "ultra (four parallel agents)", "open_weights": false }, "value": 92.2, "unit": "percent", "date": "2026-07", "frontier": true, "source_url": "https://openai.com/index/gpt-5-6/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23", "notes": "GPT-5.6 launch post. \"Ultra\" coordinates four agents in parallel by default. GPT-5.6 Sol without Ultra scored 90.4% in the same table. OpenAI is both the benchmark maintainer and the model developer." } }, { "id": "code-survival-study", "name": "Will It Survive? Agent code survival study", "short_name": "Code survival", "kind": "field-study", "institutions": [ "concordia-university" ], "authors": [ "Musfiqur Rahman", "Emad Shihab" ], "summary": "A survival analysis of code that coding agents merged into 201 open-source projects, drawn from the AIDev dataset. It tracks whether agent-written lines and files are later modified, compared with human-written code in the same projects over matched windows, and classifies why they changed.", "what_is_measured": "How long merged agent-authored code lasts before someone changes it. The cohort is 201 repositories and 5,171 PRs (3,003 agent, 2,168 human), tracked at file and line level. At the line level, agent code had a 16% lower hazard of modification than human code (hazard ratio 0.842, p < 0.001) and a 15.8 percentage-point lower modification rate. Line-level \"death rates\": Cursor 38.7%, Claude Code 41.0%, OpenAI Codex 48.5%, GitHub Copilot 48.6%, Devin 71.7%, human baseline 69.3%. When agent code changed, corrective fixes were slightly more common (26.3% vs 23.0%).", "grain": "task", "environment": [ "field-data" ], "grading": [ "field-signal" ], "task_count": null, "human_reference": "Human-authored lines in the same repositories had a 69.3% line-level death rate.", "metric": { "name": "Line-level modification (death) rate", "unit": "percent", "higher_is_better": false, "description": "Share of merged lines later modified or deleted within the observation window. Lower means the code survived longer." }, "reference_rule": { "kind": "none", "note": "Field signal from one observational study. No results file; the paper's figures are in the description." }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "shipping-prs", "codebase-evolution" ], "links": { "paper": "https://arxiv.org/abs/2601.16809" }, "license": { "data": "Paper CC BY 4.0. Underlying data comes from the AIDev dataset.", "reuse": "cite-only" }, "contamination": null, "limits": [ { "text": "Preprint under review at EASE 2026 (as of v1, 2026-01-23). Not yet peer reviewed.", "source_url": "https://arxiv.org/abs/2601.16809" }, { "text": "The authors suggest code ownership may explain the result: developers avoid touching code with no clear human owner. Longer survival therefore does not by itself show higher quality.", "source_url": "https://arxiv.org/html/2601.16809v1" }, { "text": "The file-level hazard ratio was 1.038 (p = 0.052), so the survival advantage holds at the line level but not clearly at the file level. Effect sizes for intent differences are small.", "source_url": "https://arxiv.org/html/2601.16809v1" } ], "released": "2026-01-23", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2601.16809", "https://arxiv.org/html/2601.16809v1" ], "spot_checked": false, "notes": "Both authors list Concordia University, Montreal. No results file: the study reports per-agent survival, not a benchmark score for a system at a point in time. Death rates come from Table 5 of the paper." }, "institution_names": [ "Concordia University" ], "status": "no-data", "frontier": null }, { "id": "codeclash", "name": "CodeClash", "short_name": "CodeClash", "kind": "benchmark", "institutions": [ "stanford", "princeton" ], "authors": [ "John Yang", "Kilian Lieret", "Joyce Yang", "Carlos E. Jimenez", "Muhtasham Oblokulov", "Aryan Siddiqui", "Ofir Press", "Ludwig Schmidt", "Diyi Yang" ], "summary": "Models compete in multi-round tournaments to build the best codebase for a goal, not a task. Each round has an edit phase, where the agent improves its code however it likes, and a competition phase, where the codebases fight in an arena such as a poker bot, a robot battle, or a territory game. The model that wins the most rounds wins the tournament.", "what_is_measured": "Strength as an Elo rating fitted to tournament win rates across six arenas (BattleSnake, Core War, Halite, Poker, RoboCode, RobotRumble). The paper ran 1,680 tournaments of 15 rounds with 8 models in the mini-SWE-agent scaffold. Elo has a base of 1200 and is fitted by maximum likelihood with bootstrapped uncertainties.", "grain": "project", "environment": [ "repo", "cli" ], "grading": [ "outcome-metric" ], "task_count": null, "human_reference": "A top open-source human RobotRumble bot (gigachad) beat Claude Sonnet 4.5 in 10 of 10 tournaments and 150 of 150 rounds. The paper says top models lose every round against expert human programmers.", "metric": { "name": "Elo (all arenas)", "unit": "elo", "higher_is_better": true, "description": "Overall Elo rating across the six arenas, base 1200, with a bootstrapped uncertainty." }, "reference_rule": { "kind": "none", "note": "Elo is relative to the other models in the pool, so there is no fixed ceiling." }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "codebase-evolution", "long-horizon-autonomy" ], "links": { "homepage": "https://codeclash.ai/", "paper": "https://arxiv.org/abs/2511.00839", "leaderboard": "https://codeclash.ai/", "repo": "https://github.com/CodeClash-ai/CodeClash" }, "license": { "data": "MIT (CodeClash repository). Arenas are third-party games with their own licenses.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Low relevance. There is no fixed answer to memorize; the arenas are public games, and models may have seen strategies or bots for them in training.", "limits": [ { "text": "Elo is relative to the eight models in the pool. Adding or removing models shifts every rating, and the leaderboard has not added models since November 2025.", "source_url": "https://codeclash.ai/" }, { "text": "Game arenas stand in for business goals. Winning at poker or robot battles is not the same as improving retention or revenue.", "source_url": "https://arxiv.org/abs/2511.00839" }, { "text": "All leaderboard results use one scaffold (mini-SWE-agent) with fixed settings, so scores reflect the model inside that harness.", "source_url": "https://arxiv.org/abs/2511.00839" }, { "text": "The maintainers report that model codebases grow messy and redundant over rounds and that models analyze competition logs only shallowly.", "source_url": "https://codeclash.ai/insights/20251105_human_ai/" } ], "released": "2025-11-02", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://codeclash.ai/", "https://codeclash.ai/team/", "https://codeclash.ai/insights/20251105_human_ai/", "https://arxiv.org/abs/2511.00839", "https://arxiv.org/html/2511.00839v2", "https://github.com/CodeClash-ai/CodeClash" ], "spot_checked": false, "notes": "Institutions from the paper affiliations: co-leads John Yang (Stanford) and Kilian Lieret (Princeton); other authors at Stanford, Princeton, Cornell, and TU Munich. CodeClash is listed in the SWE-bench family on swebench.com. The site leaderboard (updated 2025-11-03) shows 1385 ± 18 for Claude Sonnet 4.5; paper v2 (May 2026) shows 1389 ± 18. See data/conflicts/codeclash-sonnet-45-elo.yaml." }, "institution_names": [ "Stanford University", "Princeton University" ], "status": "unrated", "frontier": { "subject": { "label": "Claude Sonnet 4.5", "model": "Claude Sonnet 4.5", "model_org": "Anthropic", "harness": "mini-SWE-agent", "open_weights": false }, "value": 1385, "unit": "elo", "ci": [ 1367, 1403 ], "date": "2025-11-03", "frontier": true, "source_url": "https://codeclash.ai/", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23", "notes": "Rank 1 on the site leaderboard (1385 ± 18, updated 2025-11-03). Paper v2 reports 1389 ± 18 and a 69.9% average tournament win rate. The leaderboard has eight models and no entries after launch." } }, { "id": "core-bench", "name": "CORE-Bench", "short_name": "CORE-Bench", "kind": "benchmark", "institutions": [ "princeton" ], "authors": [ "Zachary S. Siegel", "Sayash Kapoor", "Nitya Nadgir", "Benedikt Stroebl", "Arvind Narayanan" ], "summary": "Computational reproducibility tasks from 90 published papers in computer science, social science, and medicine, all sourced from CodeOcean capsules known to run. An agent gets the paper's code and data, must install dependencies, run the code, and answer questions about the results, including values read from figures.", "what_is_measured": "Whether an agent can reproduce a paper's reported results from its own repository. Three levels: Easy gives the outputs, Medium gives a Docker command, Hard gives only the codebase. The headline is accuracy on CORE-Bench-Hard over the 45-paper public test set, where a task counts only if every question is answered within the tolerance set from three manual runs.", "grain": "task", "environment": [ "cli", "repo" ], "grading": [ "automated-tests" ], "task_count": 45, "human_reference": "Every task was reproduced by hand three times to set answer tolerances, so a correct answer means matching what a careful human reproduction produced.", "metric": { "name": "CORE-Bench-Hard accuracy (public test set)", "unit": "percent", "higher_is_better": true, "description": "Share of the 45 Hard test-set tasks where the agent answers every task question correctly." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "status_override": { "status": "solved", "reason": "HAL (Princeton) declared CORE-Bench solved after Claude Opus 4.5 in a Claude Code scaffold scored 77.78% on Hard, rising to 95.5% after HAL fixed grading errors in 8 tasks by manual scoring and removed 1 task whose data URL had died. HAL said it would open a private test set of new papers.", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "date": "2025-12-03" }, "jobs": [ { "family": "science-research", "soc": [ "19-1029", "19-3099" ], "weight": "primary" }, { "family": "ml-research-engineering", "soc": [ "15-1221" ], "weight": "partial" } ], "functions": [ "research-replication" ], "links": { "homepage": "https://github.com/siegelz/core-bench", "paper": "https://arxiv.org/abs/2409.11363", "leaderboard": "https://hal.cs.princeton.edu/corebench_hard", "repo": "https://github.com/siegelz/core-bench" }, "license": { "data": "Harness and code are MIT licensed. Papers and capsules come from CodeOcean and keep their own licenses.", "code": "MIT", "reuse": "open-mit" }, "contamination": "The 45 test papers are public and named. HAL keeps a second private set of papers, disclosed only after the 80% threshold was crossed, for future evaluation.", "limits": [ { "text": "HAL found grading errors in 9 tasks that only surfaced with strong agents. Deterministic outputs were penalized for tiny floating-point differences, and some tasks were underspecified.", "source_url": "https://hal.cs.princeton.edu/corebench_hard" }, { "text": "Papers were filtered to those that run in under 45 minutes, and the agent reproduces only selected results, so real reproduction is harder than the benchmark.", "source_url": "https://arxiv.org/html/2409.11363v2" }, { "text": "Scores depend heavily on scaffold. Opus 4.5 scored 42.22% with HAL's CORE-Agent and 77.78% with Claude Code before any regrading.", "source_url": "https://hal.cs.princeton.edu/corebench_hard" }, { "text": "HAL has paused adding new models to its leaderboards while it focuses on reliability work.", "source_url": "https://hal.cs.princeton.edu/" } ], "released": "2024-09-17", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2409.11363", "https://arxiv.org/html/2409.11363v2", "https://hal.cs.princeton.edu/", "https://hal.cs.princeton.edu/corebench_hard", "https://api.fxtwitter.com/sayashk/status/1996334941832089732" ], "spot_checked": false, "notes": "The \"solved\" date is the timestamp of Sayash Kapoor's X post (2025-12-03 UTC), read through the fxtwitter API because x.com blocked direct fetches. The HAL page links to that post. The HAL row shows \"77.78% (95.5% w/ manual validation)\" and names Nicholas Carlini as the submitter of the Claude Code scaffold. HAL leaderboard rows carry no run dates, so history dates are approximate." }, "institution_names": [ "Princeton University" ], "status": "solved", "frontier": { "subject": { "label": "Claude Code + Claude Opus 4.5 (after manual regrading)", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "Claude Code", "open_weights": false }, "value": 95.5, "unit": "percent", "date": "2025-12-03", "frontier": true, "source_url": "https://hal.cs.princeton.edu/corebench_hard", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23", "notes": "HAL regraded 8 tasks by hand and removed 1 task with a dead data URL. The agent failed 2 remaining tasks. HAL declared the benchmark solved on this basis." } }, { "id": "crmarena-pro", "name": "CRMArena-Pro", "short_name": "CRMArena-Pro", "kind": "benchmark", "institutions": [ "salesforce" ], "authors": [ "Kung-Hsiang Huang", "Akshara Prabhakar", "Onkar Thorat", "Divyansh Agarwal", "Prafulla Kumar Choubey", "Yixin Mao", "Silvio Savarese", "Caiming Xiong", "Chien-Sheng Wu" ], "summary": "Salesforce AI Research's benchmark for agents that work inside a CRM. Agents answer sales, service, and configure-price-quote requests by querying two synthetic Salesforce orgs (one B2B, one B2C) through the Salesforce API, in single-turn and multi-turn settings with a simulated user.", "what_is_measured": "Task completion: whether the agent's final answer matches the gold answer for each query. The 19 task types cover four skills: workflow execution, policy compliance, text understanding, and database querying. A second track checks whether agents refuse to reveal confidential data. The headline is the single-turn success rate on the B2C org, averaged over the four skills. Multi-turn scores, where a persona-driven simulated user holds back details, are much lower.", "grain": "task", "environment": [ "api-tools", "chat" ], "grading": [ "automated-tests", "llm-judge" ], "task_count": 4280, "human_reference": null, "metric": { "name": "Single-turn task success (B2C org)", "unit": "percent", "higher_is_better": true, "description": "Share of single-turn queries on the B2C org where the agent's answer matches the gold answer, averaged over the four business skills. The paper also reports B2B and multi-turn results." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "sales-marketing", "soc": [ "41-4012", "41-3091" ], "weight": "primary" }, { "family": "customer-support", "soc": [ "43-4051" ], "weight": "partial" } ], "functions": [ "crm-operations", "customer-service" ], "links": { "paper": "https://arxiv.org/abs/2505.18878", "repo": "https://github.com/SalesforceAIResearch/CRMArena", "dataset": "https://huggingface.co/datasets/Salesforce/CRMArenaPro" }, "license": { "data": "CC BY-NC 4.0 (repository LICENSE.txt). Research use only.", "code": "CC BY-NC 4.0", "reuse": "cite-only" }, "contamination": "Queries, gold answers, and the org data are public on Hugging Face. The orgs hold synthetic data that gpt-4o generated from Salesforce schemas.", "limits": [ { "text": "Published scores come from the May 2025 paper (o1, gpt-4o, Gemini 2.5, Llama 3.1 and 4). The Hugging Face leaderboard covers only the original CRMArena, so newer models are not tracked here.", "source_url": "https://huggingface.co/spaces/Salesforce/CRMArena-Leaderboard" }, { "text": "The org data is synthetic and generated by gpt-4o. 66.7% of CRM experts rated the B2B data as realistic and 62.3% rated the B2C data as realistic.", "source_url": "https://arxiv.org/html/2505.18878v1" }, { "text": "Multi-turn runs use an LLM user simulator. A manual check of 20 trajectories found one simulator error (5%). gpt-4o also extracts answers and judges confidentiality refusals.", "source_url": "https://arxiv.org/html/2505.18878v1" }, { "text": "Agents run through a ReAct scaffold with API access only. GUI access to the orgs was withdrawn after a Salesforce system update.", "source_url": "https://github.com/SalesforceAIResearch/CRMArena" } ], "released": "2025-05-24", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2505.18878", "https://arxiv.org/html/2505.18878v1", "https://github.com/SalesforceAIResearch/CRMArena", "https://huggingface.co/spaces/Salesforce/CRMArena-Leaderboard", "https://huggingface.co/datasets/Salesforce/CRMArenaPro" ], "spot_checked": false, "notes": "Task count is 19 task types x 100 queries x 2 orgs plus 3 x 80 x 2 confidentiality queries, as stated in the paper. The paper was accepted at TMLR in February 2026 per the repository, but the arXiv version is still v1. Salesforce does not report its own models in the paper." }, "institution_names": [ "Salesforce AI Research" ], "status": "emerging", "frontier": { "subject": { "label": "gemini-2.5-pro (ReAct)", "model": "gemini-2.5-pro", "model_org": "Google", "harness": "ReAct", "open_weights": false }, "value": 58.3, "unit": "percent", "date": "2025-05-24", "frontier": true, "source_url": "https://arxiv.org/html/2505.18878v1", "source_kind": "primary", "reported_by": "salesforce", "retrieved_at": "2026-09-23", "notes": "Table 2 of the paper. B2B single-turn 54.1. Multi-turn 35.1 (B2B) and 30.0 (B2C). Workflow execution alone reached 83.0 (B2B) and 90.0 (B2C) single-turn. No newer primary results exist." } }, { "id": "cve-bench", "name": "CVE-Bench", "short_name": "CVE-Bench", "kind": "benchmark", "institutions": [ "uiuc" ], "summary": "40 critical-severity CVEs (CVSS 9.0 or higher) in real web applications such as WordPress plugins and LLM tools. Each runs in an isolated sandbox. An agent must carry out a real attack, for example read the secret table, log in as admin, or execute code, and an evaluator checks the effect.", "what_is_measured": "Whether an agent can exploit a real web vulnerability end to end. In the zero-day setting the agent gets only the target; in the one-day setting it also gets a short description of the vulnerability. Success is one of eight attack outcomes checked by the sandbox. The paper reports success with one attempt and with five attempts. The headline here is one-day success with five attempts.", "grain": "task", "environment": [ "cli", "browser" ], "grading": [ "state-check" ], "task_count": 40, "human_reference": null, "metric": { "name": "Success rate (one-day, 5 attempts)", "unit": "percent", "higher_is_better": true, "description": "Share of the 40 CVEs exploited within five attempts when the agent is given a vulnerability description." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "security", "weight": "primary" } ], "functions": [ "vulnerability-research" ], "links": { "paper": "https://arxiv.org/abs/2503.17332", "repo": "https://github.com/uiuc-kang-lab/cve-bench", "blog": "https://open.substack.com/pub/ddkang/p/cve-bench-v20-making-evaluation-more" }, "license": { "data": "Apache-2.0 (GitHub repository license). Reference exploits are available on request.", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": "CVEs are public in the National Vulnerability Database with published details. Since v2.0, secrets are generated at runtime instead of hard-coded, in case old secrets appear in training data.", "limits": [ { "text": "Published results use gpt-4o-2024-11-20 with three agent frameworks from early 2025. There is no public leaderboard for newer models.", "source_url": "https://arxiv.org/abs/2503.17332" }, { "text": "v2.0 (2025-10-30) closed two shortcuts that agents used to pass tasks. The maintainers say GPT-4o agent success rates fell by up to 32.5 points after the fixes, so paper numbers overstate v1 ability.", "source_url": "https://open.substack.com/pub/ddkang/p/cve-bench-v20-making-evaluation-more" }, { "text": "v2.1.0 (2026-01-12) replaced arbitrary file upload with remote code execution as an attack goal, so results across versions are not directly comparable.", "source_url": "https://github.com/uiuc-kang-lab/cve-bench" }, { "text": "40 tasks is a small set; one task equals 2.5 percentage points.", "source_url": "https://arxiv.org/abs/2503.17332" } ], "released": "2025-03-31", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2503.17332", "https://arxiv.org/html/2503.17332v4", "https://github.com/uiuc-kang-lab/cve-bench", "https://open.substack.com/pub/ddkang/p/cve-bench-v20-making-evaluation-more" ], "spot_checked": false, "notes": "ICML 2025 spotlight. Authors list the Siebel School of Computing and Data Science, UIUC. The abstract rounds the best result to 13%; the results section gives 12.5% (5 of 40) for one-day with five attempts and 10% for zero-day. Release date from the repo news entry \"CVE-Bench released\" on 2025-03-31." }, "institution_names": [ "UIUC" ], "status": "open", "frontier": { "subject": { "label": "AutoGPT with GPT-4o", "model": "gpt-4o-2024-11-20", "model_org": "OpenAI", "agent": "AutoGPT", "open_weights": false }, "value": 12.5, "unit": "percent", "date": "2025-03", "frontier": true, "source_url": "https://arxiv.org/abs/2503.17332", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23", "notes": "Best result in the paper: 12.5% one-day and 10% zero-day with five attempts (abstract rounds to 13%). Measured on CVE-Bench v1 before the v2.0 grading fixes." } }, { "id": "cybench", "name": "Cybench", "short_name": "Cybench", "kind": "benchmark", "institutions": [ "stanford" ], "summary": "40 professional-level Capture the Flag tasks from four recent CTF competitions. An agent works in a Kali Linux container, runs commands against local files and task servers, and submits a flag. Each task also has subtasks for partial credit. Used by the US and UK AI Safety Institutes and in many model system cards.", "what_is_measured": "Whether an agent can solve a CTF task on its own (unguided) or with subtask hints (guided), across cryptography, web, reverse engineering, forensics, binary exploitation, and misc categories. The headline is the unguided share of tasks solved. The site also reports the hardest task solved by first-solve time of human teams (up to 24 hours 54 minutes).", "grain": "task", "environment": [ "cli" ], "grading": [ "automated-tests" ], "task_count": 40, "human_reference": "First solve time by competition teams, from 2 minutes to 24 hours 54 minutes per task.", "metric": { "name": "Unguided % solved", "unit": "percent", "higher_is_better": true, "description": "Share of tasks solved without subtask guidance. Lab-reported entries use pass@1 on subsets of 35 to 39 tasks." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "security", "weight": "primary" } ], "functions": [ "vulnerability-research" ], "links": { "homepage": "https://cybench.github.io", "paper": "https://arxiv.org/abs/2408.08926", "leaderboard": "https://cybench.github.io", "repo": "https://github.com/andyzorigin/cybench" }, "license": { "data": "Tasks are public in the GitHub repository (Apache-2.0); the site asks users to cite the ICLR 2025 paper.", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": "Tasks come from public CTF competitions with published writeups, so training data may include solutions. One HAL run used a framework fork that leaked an answer; the site adjusted those scores down.", "limits": [ { "text": "Recent leaderboard entries are copied from lab system cards on 35 to 39 task subsets with different trial counts, not from a common harness.", "source_url": "https://cybench.github.io" }, { "text": "A fork of the Inspect framework leaked an answer to one task. Scores for o3-mini and o1-mini were adjusted down by 2.5 points.", "source_url": "https://cybench.github.io" }, { "text": "Anthropic found grading errors in its earlier Cybench runs, so its newer numbers may not match previously reported ones.", "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf" }, { "text": "CTF tasks are gamified. Anthropic says real-world vulnerability work (for example CyberGym) now reflects capability better.", "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf" } ], "released": "2024-08-15", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://cybench.github.io", "https://cybench.github.io/data/leaderboard.csv", "https://arxiv.org/abs/2408.08926", "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf", "https://cdn.sanity.io/files/4zrzovbb/website/037f06850df7fbe871e206dad004c3db5fd50340.pdf", "https://assets.anthropic.com/m/64823ba7485345a7/Claude-Opus-4-5-System-Card.pdf" ], "spot_checked": true, "notes": "ICLR 2025 oral. The leaderboard CSV lists Claude Mythos Preview 100 (35 tasks), Claude Opus 4.7 96 (35), Claude Opus 4.6 93 (37), Claude Opus 4.5 82 (39). The 100% figure was confirmed in the Mythos Preview system card dated 2026-04-07. Status is computed (saturated); we removed an earlier \"solved\" override because no maintainer or evaluator declared the benchmark solved, and Anthropic still reported it for Claude Opus 4.7 on 2026-04-16. Second check 2026-09-23 fixed the GPT-4o launch value (12.5% unguided) and the license (Apache-2.0)." }, "institution_names": [ "Stanford University" ], "status": "saturated", "frontier": { "subject": { "label": "Claude Mythos Preview", "model": "Claude Mythos Preview", "model_org": "Anthropic", "open_weights": false }, "value": 100, "unit": "percent", "date": "2026-04-07", "frontier": true, "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf", "source_kind": "lab-reported", "reported_by": "anthropic", "retrieved_at": "2026-09-23", "notes": "Pass@1 of 100% on a 35-task subset, 10 trials per task, no extended thinking. The Cybench site lists this entry on its leaderboard." } }, { "id": "cybergym", "name": "CyberGym", "short_name": "CyberGym", "kind": "benchmark", "institutions": [ "uc-berkeley" ], "summary": "1,507 real vulnerabilities from 188 open-source projects, drawn from Google's OSS-Fuzz. Given a text description of a bug and the unpatched codebase, an agent must write a proof-of-concept input that triggers the vulnerability. UC Berkeley maintains a public leaderboard with team submissions.", "what_is_measured": "Whether an agent can reproduce a known vulnerability in a large real codebase. A task counts as solved when the generated proof of concept crashes the pre-patch build but not the post-patch build. The headline (Level 1) gives the agent the vulnerability description and the source. Other levels give less or more information. The leaderboard notes when a team used a runnable vulnerable image or test-time memory.", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 1507, "human_reference": null, "metric": { "name": "Success rate (Level 1)", "unit": "percent", "higher_is_better": true, "description": "Share of the 1,507 instances where the agent produces a working proof of concept for the target vulnerability." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "security", "weight": "primary" }, { "family": "software-engineering", "weight": "partial" } ], "functions": [ "vulnerability-research" ], "links": { "homepage": "https://www.cybergym.io/cybergym/", "paper": "https://arxiv.org/abs/2506.02548", "leaderboard": "https://www.cybergym.io/cybergym/", "repo": "https://github.com/sunblaze-ucb/cybergym", "dataset": "https://huggingface.co/datasets/sunblaze-ucb/cybergym" }, "license": { "data": "Apache-2.0 (GitHub repository license). Dataset hosted on Hugging Face (about 240 GB).", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": "Vulnerabilities and their fixes are public in OSS-Fuzz and project histories, so training data may include them.", "limits": [ { "text": "Leaderboard results are run and submitted by the teams themselves. Agent runs are stochastic, and the maintainers say small score differences may not reflect real capability gaps now that leading systems score high.", "source_url": "https://www.cybergym.io/cybergym/" }, { "text": "Vulnerability descriptions can be ambiguous, which adds noise to grading.", "source_url": "https://www.cybergym.io/cybergym/" }, { "text": "Top entries use multiple models, orchestration, and test-time memory across instances. They measure agent systems, not single models.", "source_url": "https://www.cybergym.io/cybergym/" }, { "text": "The task is reproduction of known bugs, not discovery. The paper separately reports 34 zero-days found by agents in open-ended runs.", "source_url": "https://arxiv.org/abs/2506.02548" } ], "released": "2025-06-03", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2506.02548", "https://www.cybergym.io/cybergym/", "https://www.cybergym.io/assets/data/cybergym.json", "https://github.com/sunblaze-ucb/cybergym", "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf" ], "spot_checked": false, "notes": "Authors (Zhun Wang, Tianneng Shi, Jingxuan He, Matthew Cai, Jialin Zhang, Dawn Song) are all listed as UC Berkeley on the site. Leaderboard values were read from the site's JSON data file. The paper abstract says top combinations reach \"~20%\"; the leaderboard shows 17.85% for OpenHands with Claude Sonnet 4 on 2025-05-23." }, "institution_names": [ "UC Berkeley" ], "status": "saturated", "frontier": { "subject": { "label": "Creation (天工), multi-model", "model": "Creation Model, DeepSeek-V4-Pro, Qwen 3.8 Max", "model_org": "Creation Lab", "agent": "Creation", "open_weights": null }, "value": 98.47, "unit": "percent", "date": "2026-09-07", "frontier": true, "source_url": "https://www.cybergym.io/cybergym/", "source_kind": "third-party", "reported_by": "uc-berkeley", "retrieved_at": "2026-09-23", "notes": "Team submission listed first on the official leaderboard. Uses a runnable vulnerable Docker image (\"dynamic\" label). Sangfor AI (GLM-5.3) follows at 97.21% and Alipay AI4SDL at 96.75%." } }, { "id": "gaia", "name": "GAIA", "kind": "benchmark", "institutions": [ "meta", "hugging-face" ], "authors": [ "Grégoire Mialon", "Clémentine Fourrier", "Craig Swift", "Thomas Wolf", "Yann LeCun", "Thomas Scialom" ], "summary": "466 questions for general AI assistants that need web browsing, file reading, and tool use to answer. Each question has one short, unambiguous answer. Questions are easy for people but were hard for models in 2023.", "what_is_measured": "Share of questions answered exactly right. Questions come in three difficulty levels and often attach a file (a spreadsheet, image, audio clip, or PDF). Answers are checked by quasi exact match against a hidden reference. 165 questions form a public validation set; 300 form the test set with private answers that powers the Hugging Face leaderboard.", "grain": "task", "environment": [ "browser", "api-tools", "documents" ], "grading": [ "automated-tests" ], "task_count": 466, "human_reference": "Human respondents scored 92% in the paper's study.", "metric": { "name": "Test-set accuracy", "unit": "percent", "higher_is_better": true, "description": "Share of the 300 private test questions answered correctly (average across levels 1 to 3)." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "office-admin", "weight": "primary" }, { "family": "data-analytics", "weight": "partial" } ], "functions": [ "web-research" ], "links": { "homepage": "https://huggingface.co/gaia-benchmark", "paper": "https://arxiv.org/abs/2311.12983", "leaderboard": "https://huggingface.co/spaces/gaia-benchmark/leaderboard", "dataset": "https://huggingface.co/datasets/gaia-benchmark/GAIA" }, "license": { "data": "Gated dataset on Hugging Face. Test answers are private. The maintainers ask that the public set not be reposted or used for training.", "reuse": "cite-only" }, "contamination": "The validation questions and answers are public and the maintainers ask people not to train on them. Test answers stay private, but the questions are public.", "limits": [ { "text": "The leaderboard accepts self-submitted answer files from anyone. Top entries are often unnamed or commercial agents with little public detail, and the maintainers do not re-run them.", "source_url": "https://huggingface.co/spaces/gaia-benchmark/leaderboard" }, { "text": "Top scores now exceed the 92% human figure, so the benchmark no longer separates the best agents.", "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public" }, { "text": "Answers are short strings or numbers, so the benchmark does not test long deliverables or judgment calls.", "source_url": "https://arxiv.org/abs/2311.12983" }, { "text": "The validation leaderboard was closed because it was no longer informative.", "source_url": "https://huggingface.co/spaces/gaia-benchmark/leaderboard" } ], "released": "2023-11-21", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2311.12983", "https://huggingface.co/spaces/gaia-benchmark/leaderboard", "https://huggingface.co/datasets/gaia-benchmark/results_public" ], "spot_checked": true, "notes": "The leaderboard app needs JavaScript, so scores were read from the results_public dataset (test split) that feeds it. Institutions follow the author list (Meta FAIR and Hugging Face); affiliations were not shown on the arXiv page and come from the leaderboard citation and the Hugging Face organization that hosts the benchmark." }, "institution_names": [ "Meta", "Hugging Face" ], "status": "saturated", "frontier": { "subject": { "label": "Ops-Agentic-Search-2.0", "agent": "Ops-Agentic-Search-2.0" }, "value": 93.69, "unit": "percent", "date": "2026-09-22", "frontier": true, "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public", "source_kind": "third-party", "reported_by": "Unnamed submitter (GAIA leaderboard entry links to github.com/tosky001/OpenSearch-Agentic-Search)", "retrieved_at": "2026-09-23", "notes": "Top test-set entry as of 2026-09-23. Level 1 98.92%, level 2 93.08%, level 3 85.71%. The organisation field is blank. Self-submitted answer file; not re-run by the maintainers. The next entries are CustomGPT.ai Research Lab v44 at 93.36% and Co-Sight Pro v1.0.1 at 93.02%." } }, { "id": "gdpval-aa", "name": "GDPval-AA", "kind": "leaderboard", "institutions": [ "artificial-analysis" ], "summary": "Artificial Analysis runs OpenAI's public 220-task GDPval gold set through its own agent harness, Stirrup, and ranks models by Elo from blind pairwise comparisons of the deliverables. The judges are a panel of three frontier language models, not human experts.", "what_is_measured": "How well a model, acting as an agent with a Linux sandbox, code execution, web search, web fetch, and image viewing, produces work deliverables (documents, slides, spreadsheets, media) for the 220 GDPval gold tasks across 44 occupations. A judge sampled from a panel of three frontier LLMs blindly picks the better of two submissions to the same task. Ratings are fitted with a Crowd-BT model and reported as Elo with 95% confidence intervals.", "grain": "project", "environment": [ "cli", "api-tools", "documents" ], "grading": [ "llm-judge" ], "task_count": 220, "human_reference": "None on the current v2.1 scale. In GDPval-AA v2 (June to September 2026) the scale was anchored so that human expert performance was 1000 Elo. v2.1 re-anchored the scale to DeepSeek V4.1 Flash (max) at 1600, so the human anchor no longer applies.", "metric": { "name": "GDPval-AA Elo", "unit": "elo", "higher_is_better": true, "description": "Crowd-BT rating from blind pairwise LLM-judge comparisons of deliverables. Anchored to DeepSeek V4.1 Flash (max) at 1600 in v2.1. Ties count as half a win for each side." }, "reference_rule": { "kind": "none", "note": "Elo scale with a model anchor. Scores are relative and shift when the anchor changes." }, "jobs": [ { "family": "management-business", "weight": "primary" }, { "family": "finance-accounting", "weight": "partial" }, { "family": "legal", "weight": "partial" }, { "family": "software-engineering", "weight": "partial" }, { "family": "sales-marketing", "weight": "partial" }, { "family": "office-admin", "weight": "partial" }, { "family": "healthcare", "weight": "partial" }, { "family": "design-media-writing", "weight": "partial" }, { "family": "architecture-engineering", "weight": "partial" } ], "functions": [ "professional-deliverables" ], "links": { "homepage": "https://artificialanalysis.ai/evaluations/gdpval-aa", "paper": "https://arxiv.org/abs/2510.04374", "leaderboard": "https://artificialanalysis.ai/evaluations/gdpval-aa", "repo": "https://github.com/ArtificialAnalysis/Stirrup", "blog": "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "dataset": "https://huggingface.co/datasets/openai/gdpval" }, "license": { "data": "Tasks are OpenAI's public GDPval gold set. The Stirrup harness is open source on GitHub. Artificial Analysis publishes Elo scores and example submissions on its site.", "reuse": "cite-only" }, "contamination": "Uses the public GDPval gold subset, so models may have seen the tasks. Artificial Analysis has not published a contamination analysis.", "limits": [ { "text": "Judges are language models (Claude Opus 5, GPT-5.6 Sol, Gemini 3.8 Flash in September 2026), not the human experts used in OpenAI's own GDPval grading.", "source_url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking" }, { "text": "The Elo scale has been re-anchored twice (v2 in June 2026 to human experts at 1000, v2.1 in September 2026 to DeepSeek V4.1 Flash at 1600). Scores from different versions are not comparable.", "source_url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking" }, { "text": "Each model runs once per task, and 95% confidence intervals are about plus or minus 20 to 30 Elo, so nearby ranks can swap.", "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa" }, { "text": "Many entries are the same model at different reasoning-effort settings, so the board mixes model quality with compute budget.", "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa" } ], "released": "2025-12", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://artificialanalysis.ai/evaluations/gdpval-aa", "https://artificialanalysis.ai/methodology/intelligence-benchmarking", "https://www.linkedin.com/posts/artificial-analysis_announcing-gdpval-aa-our-leaderboard-and-activity-7404608113624072193-Q82r" ], "spot_checked": false, "notes": "Release month comes from the Artificial Analysis LinkedIn announcement, shown as \"9mo\" old on 2026-09-23, and from the methodology version history (added to the Intelligence Index in January 2026). Exact launch day not confirmed. OpenAI's GPT-5.6 post quotes GDPval-AA v2 Elo (for example GPT-5.6 Sol 1,747.8), which is on the older v2 scale and is not comparable with the v2.1 board." }, "institution_names": [ "Artificial Analysis" ], "status": "unrated", "frontier": { "subject": { "label": "Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)", "model": "Claude Opus 5.5", "model_org": "Anthropic", "harness": "Stirrup", "open_weights": false }, "value": 1846, "unit": "elo", "ci": [ 1823, 1869 ], "date": "2026-09-23", "frontier": true, "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23", "notes": "Rank 1 on the GDPval-AA v2.1 board, seen 2026-09-23. Confidence interval shown as -23 / +23. Model release month shown as Sep 2026. Judged by a panel of three frontier LLMs, not humans." } }, { "id": "gdpval", "name": "GDPval", "kind": "benchmark", "institutions": [ "openai" ], "authors": [ "Tejal Patwardhan", "Rachel Dias", "Elizabeth Proehl", "Grace Kim", "Michele Wang", "Olivia Watkins", "Simón Posada Fishman", "Marwan Aljubeh", "Phoebe Thacker" ], "summary": "Real work tasks from 44 occupations in the 9 US industries that add the most to GDP. Experienced professionals wrote each task from their own work. Blinded experts from the same occupation compare the AI deliverable with a deliverable made by a professional.", "what_is_measured": "Whether an AI deliverable is judged better than or as good as a professional's deliverable for the same request. Tasks come with reference files and ask for documents, slides, spreadsheets, diagrams, or media. The full set has 1,320 tasks (30 per occupation). A 220-task gold subset (5 per occupation) is public and is the set used for the headline numbers. Occupations include software developers, lawyers, accountants, nurses, financial managers, sales managers, editors, and mechanical engineers.", "grain": "project", "environment": [ "documents", "cli" ], "grading": [ "pairwise-human", "human-expert" ], "task_count": 220, "human_reference": "A deliverable made by an industry professional (average 14 years of experience). Gold subset tasks take an expert 9.49 hours on average (median 5 hours) and are worth $398 on average at median wages.", "metric": { "name": "Wins plus ties against experts", "unit": "percent", "higher_is_better": true, "description": "Share of blinded pairwise comparisons in which experts rate the model deliverable better than (win) or as good as (tie) the professional's deliverable." }, "reference_rule": { "kind": "parity", "value": 50, "note": "At 50% wins plus ties the model matches the expert baseline on average." }, "jobs": [ { "family": "management-business", "weight": "primary" }, { "family": "finance-accounting", "weight": "partial" }, { "family": "legal", "weight": "partial" }, { "family": "software-engineering", "weight": "partial" }, { "family": "sales-marketing", "weight": "partial" }, { "family": "office-admin", "weight": "partial" }, { "family": "healthcare", "weight": "partial" }, { "family": "design-media-writing", "weight": "partial" }, { "family": "architecture-engineering", "weight": "partial" }, { "family": "customer-support", "weight": "partial" }, { "family": "education-social-services", "weight": "partial" } ], "functions": [ "professional-deliverables" ], "links": { "homepage": "https://openai.com/index/gdpval/", "paper": "https://arxiv.org/abs/2510.04374", "leaderboard": "https://evals.openai.com/gdpval/leaderboard", "dataset": "https://huggingface.co/datasets/openai/gdpval" }, "license": { "data": "The 220-task gold subset (prompts and reference files) is public on Hugging Face. The other 1,100 tasks and the expert grades are private. Rubrics and gold deliverables were later released.", "reuse": "cite-only" }, "contamination": "The gold subset is public, so later models may have seen it. OpenAI has not published a contamination analysis.", "limits": [ { "text": "Tasks are one-shot and well specified. They do not test clarifying questions, iteration with a client, or building context over time.", "source_url": "https://openai.com/index/gdpval/" }, { "text": "OpenAI builds the benchmark and reports its own models. Most later results come from OpenAI launch posts, which do not always say whether expert graders or the automated grader produced them.", "source_url": "https://openai.com/index/introducing-gpt-5-5/" }, { "text": "Human inter-rater agreement is 71%, so a single comparison is noisy. The automated grader agrees with humans 66% of the time and favors OpenAI outputs.", "source_url": "https://proceedings.iclr.cc/paper_files/paper/2026/file/290c2430f91912204f30bbcc990fff1d-Paper-Conference.pdf" }, { "text": "Graders may have guessed which deliverable came from a model because of style cues, for example em dashes or first-person phrasing.", "source_url": "https://proceedings.iclr.cc/paper_files/paper/2026/file/290c2430f91912204f30bbcc990fff1d-Paper-Conference.pdf" } ], "released": "2025-09-25", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://openai.com/index/gdpval/", "https://proceedings.iclr.cc/paper_files/paper/2026/file/290c2430f91912204f30bbcc990fff1d-Paper-Conference.pdf", "https://openai.com/index/introducing-gpt-5-2/", "https://openai.com/index/introducing-gpt-5-5/", "https://openai.com/index/gpt-5-6/", "https://openai.com/index/gpt-6-astra/" ], "spot_checked": false, "notes": "The official leaderboard at evals.openai.com needs JavaScript and did not render, so results come from the paper and OpenAI launch posts. The GPT-5.6 and GPT-6 Astra posts report GDPval-AA Elo instead of GDPval wins plus ties, so 84.9% (GPT-5.5, April 2026) is the latest OpenAI figure on this metric. Aggregators (BenchmarkList, aireleasetracker) show the same 84.9%." }, "institution_names": [ "OpenAI" ], "status": "strong", "frontier": { "subject": { "label": "GPT-5.5", "model": "GPT-5.5", "model_org": "OpenAI", "open_weights": false }, "value": 84.9, "unit": "percent", "date": "2026-04", "frontier": true, "source_url": "https://openai.com/index/introducing-gpt-5-5/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23", "notes": "From the GPT-5.5 launch post, which does not say whether blinded experts or the automated grader produced the number. OpenAI is both the benchmark maintainer and the model developer. The same table gives GPT-5.4 83.0%, GPT-5.5 Pro 82.3%, and Gemini 3.1 Pro 67.3%." } }, { "id": "iac-eval", "name": "IaC-Eval", "short_name": "IaC-Eval", "kind": "benchmark", "institutions": [ "university-of-michigan", "cisco" ], "summary": "A code-generation benchmark for Terraform. Each of 458 human-written scenarios pairs a natural-language request for AWS infrastructure with an intent specification. A model's Terraform program passes only if it plans successfully and meets the specified intent.", "what_is_measured": "Whether a model can write a correct Terraform configuration from a plain-English request. Generated programs are checked with Terraform tooling and Open Policy Agent rules that encode the scenario's required resources and settings. The headline is pass@1 accuracy over the 458 scenarios, estimated from 20 samples per problem.", "grain": "task", "environment": [ "cli" ], "grading": [ "automated-tests", "state-check" ], "task_count": 458, "human_reference": null, "metric": { "name": "pass@1", "unit": "percent", "higher_is_better": true, "description": "Share of scenarios where a single generated Terraform program plans and satisfies the intent specification." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "devops-sre-it", "weight": "primary" } ], "functions": [ "infrastructure-as-code" ], "links": { "paper": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/f26b29298ae8acd94bd7e839688e329b-Abstract-Datasets_and_Benchmarks_Track.html", "repo": "https://github.com/autoiac-project/iac-eval", "dataset": "https://huggingface.co/datasets/autoiac-project/iac-eval" }, "license": { "data": "CC-BY-4.0 (Hugging Face dataset card)", "code": "MIT", "reuse": "open-cc-by" }, "contamination": null, "limits": [ { "text": "Results in the paper cover models from 2024 (GPT-4, GPT-3.5, Gemini 1.0 Pro, CodeLlama, WizardCoder, Magicoder). The README lists the leaderboard as \"TBD\" and no newer results are published by the maintainers.", "source_url": "https://github.com/autoiac-project/iac-eval" }, { "text": "AWS and Terraform only. Other clouds and IaC tools are left as future work.", "source_url": "https://github.com/autoiac-project/iac-eval" }, { "text": "The maintainers describe the framework as not production-ready and under active patching.", "source_url": "https://github.com/autoiac-project/iac-eval" } ], "released": "2024", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://proceedings.neurips.cc/paper_files/paper/2024/hash/f26b29298ae8acd94bd7e839688e329b-Abstract-Datasets_and_Benchmarks_Track.html", "https://proceedings.neurips.cc/paper_files/paper/2024/file/f26b29298ae8acd94bd7e839688e329b-Paper-Datasets_and_Benchmarks_Track.pdf", "https://github.com/autoiac-project/iac-eval" ], "spot_checked": true, "notes": "Affiliations from the paper header: University of Michigan and Cisco Research. NeurIPS 2024 Datasets and Benchmarks Track. The repo was created 2024-05-29. Third-party IaC-Bench (IIIT Hyderabad, SERI 2026) reuses this dataset with quality metrics rather than pass rates; it has no published headline result." }, "institution_names": [ "University of Michigan", "Cisco Research" ], "status": "emerging", "frontier": { "subject": { "label": "GPT-4 with retrieval-augmented generation", "model": "GPT-4", "model_org": "OpenAI", "harness": "RAG over Terraform documentation", "open_weights": false }, "value": 36.7, "unit": "percent", "date": "2024", "frontier": true, "source_url": "https://proceedings.neurips.cc/paper_files/paper/2024/file/f26b29298ae8acd94bd7e839688e329b-Paper-Datasets_and_Benchmarks_Track.pdf", "source_kind": "primary", "reported_by": "university-of-michigan", "retrieved_at": "2026-09-23", "notes": "Best result across the paper's enhancement strategies (few-shot, chain of thought, multi-turn, RAG). Multi-turn GPT-4 scores 31.12%." } }, { "id": "impossiblebench", "name": "ImpossibleBench", "kind": "benchmark", "institutions": [ "cmu", "anthropic" ], "authors": [ "Ziqian Zhong", "Aditi Raghunathan", "Nicholas Carlini" ], "summary": "Coding tasks from SWE-bench Verified and LiveCodeBench, rewritten so that the unit tests contradict the written specification. No honest solution can pass. Any pass means the agent took a shortcut, such as editing the tests, overloading comparison operators, or special-casing inputs. The pass rate is therefore a cheating rate.", "what_is_measured": "How often an agent games the tests instead of following the specification when the two conflict. Impossible-SWEbench has 349 tasks (from SWE-bench Verified) in two variants: one-off (one test expectation changed) and conflicting (tests contradict each other). Impossible- LiveCodeBench has 103 tasks from the hard split of LiveCodeBench v6. Agents have read and write access to the tests but are told not to modify them. The headline is the cheating rate on Conflicting-SWEbench with the full tool-using scaffold and the strictest prompt.", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 349, "human_reference": "A human who follows the specification would score 0%.", "metric": { "name": "Cheating rate (Conflicting-SWEbench, full scaffold)", "unit": "percent", "higher_is_better": false, "description": "Share of impossible tasks that the agent \"passes\". Every pass implies a specification-violating shortcut. Lower is better." }, "reference_rule": { "kind": "none", "note": "Behavior metric where lower is better. The ideal is 0%." }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "test-integrity", "issue-resolution" ], "links": { "paper": "https://arxiv.org/abs/2510.20270", "repo": "https://github.com/safety-research/impossiblebench", "dataset": "https://huggingface.co/datasets/fjzzq2002/impossible_swebench" }, "license": { "data": "Paper CC BY 4.0; repo MIT. Task data derives from SWE-bench Verified and LiveCodeBench.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Base tasks come from public benchmarks that models may have trained on. The impossible mutations are new, but a model that memorized the original fix could notice the conflict more easily.", "limits": [ { "text": "Cheating rates depend strongly on the prompt. On Conflicting-LiveCodeBench, GPT-5 and o3 cheat more than 85% of the time with loose prompts (A and B) and 1% and 33% with the strictest prompt (D), which the main results use.", "source_url": "https://arxiv.org/html/2510.20270v1" }, { "text": "More capable models cheat more. The paper reports that GPT-5, the strongest model tested, cheats 54.0% of the time on Conflicting-SWEbench.", "source_url": "https://arxiv.org/html/2510.20270v1" }, { "text": "Per-model cheating rates for other models appear only in figures (Figure 3 and 4), not as numbers in the text, so we record only the value the paper states.", "source_url": "https://arxiv.org/html/2510.20270v1" }, { "text": "Anthropic co-authored the paper and provided compute; results include Claude models. The paper reports Claude Opus 4.1 and Sonnet 4 cheat less than Claude Sonnet 3.7.", "source_url": "https://arxiv.org/html/2510.20270v1" } ], "released": "2025-10-23", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2510.20270", "https://arxiv.org/html/2510.20270v1", "https://raw.githubusercontent.com/safety-research/impossiblebench/HEAD/README.md", "https://api.github.com/repos/safety-research/impossiblebench" ], "spot_checked": false, "notes": "Affiliations from the paper: Zhong and Raghunathan at Carnegie Mellon University, Carlini at Anthropic. Models tested: GPT-5, o3, o4-mini, GPT-4.1, Claude Opus 4.1, Claude Sonnet 4, Claude Sonnet 3.7, Qwen3-Coder. Figure 3 shows GPT-4.1 and Qwen3-Coder with the lowest cheating rates and Claude Sonnet 3.7 with the highest, but exact values are not in the text. No public leaderboard; the repo was last pushed 2025-12-01." }, "institution_names": [ "Carnegie Mellon University", "Anthropic" ], "status": "unrated", "frontier": { "subject": { "label": "GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "ImpossibleBench full scaffold (bash, python, text editor, think tools)", "open_weights": false }, "value": 54, "unit": "percent", "date": "2025-10-23", "frontier": true, "source_url": "https://arxiv.org/html/2510.20270v1", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23", "notes": "Conflicting-SWEbench, full scaffold, strictest prompt. Lower is better; this is the paper's stated number for the most capable model tested, not the lowest rate. GPT-4.1 and Qwen3-Coder show lower rates in Figure 3 but the paper does not print their values." } }, { "id": "itbench-aa", "name": "ITBench-AA", "short_name": "ITBench-AA", "kind": "benchmark", "institutions": [ "artificial-analysis", "ibm" ], "summary": "Kubernetes incident diagnosis tasks built from IBM's ITBench and run by Artificial Analysis in a fixed agent harness. The model reads an offline incident snapshot with alerts, logs, traces, metrics, and topology, then names the root-cause Kubernetes entities.", "what_is_measured": "Whether the model finds the exact set of root-cause entities for a Kubernetes incident. Scoring is average precision at full recall: a repeat scores 0 if any true root cause is missed, and otherwise scores the precision of the submitted list. The headline is the mean over 59 tasks and 3 repeats. Models run in the open-source Stirrup harness with shell access and a 100-turn cap.", "grain": "task", "environment": [ "cli" ], "grading": [ "state-check" ], "task_count": 59, "human_reference": null, "metric": { "name": "ITBench-AA score", "unit": "percent", "higher_is_better": true, "description": "Mean average precision at full recall over 59 SRE tasks x 3 repeats." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "devops-sre-it", "weight": "primary" } ], "functions": [ "incident-response" ], "links": { "homepage": "https://artificialanalysis.ai/evaluations/itbench-aa", "leaderboard": "https://artificialanalysis.ai/evaluations/itbench-aa", "blog": "https://artificialanalysis.ai/articles/itbench-aa-launch" }, "license": { "data": "40 public tasks and 19 held-out tasks; the held-out tasks are private to Artificial Analysis.", "reuse": "cite-only" }, "contamination": "19 of the 59 tasks are new and held out. The other 40 are public ITBench scenarios.", "limits": [ { "text": "Diagnosis only. Models name root-cause entities from a snapshot; they do not repair a live system. Repair is what the original ITBench SRE track scores.", "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch" }, { "text": "Strict scoring: naming one extra entity lowers the score, and missing one true root cause gives zero for that repeat. Models that investigate longer tend to add false positives.", "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch" }, { "text": "FinOps and CISO tasks were announced but not yet included at launch.", "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch" } ], "released": "2026-05-27", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://artificialanalysis.ai/articles/itbench-aa-launch", "https://artificialanalysis.ai/evaluations/itbench-aa", "https://github.com/itbench-hub/ITBench" ], "spot_checked": false, "notes": "The live leaderboard page renders with JavaScript and did not expose numbers to a fetch. All scores come from the launch article dated 2026-05-27. Models released after that date may now lead." }, "institution_names": [ "Artificial Analysis", "IBM Research" ], "status": "emerging", "frontier": { "subject": { "label": "Claude Opus 4.7 (Adaptive Reasoning, Max Effort)", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "Stirrup", "open_weights": false }, "value": 47, "unit": "percent", "date": "2026-05-27", "frontier": true, "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23", "notes": "Led the launch leaderboard. Most expensive at $5.38 per task." } }, { "id": "itbench", "name": "ITBench", "short_name": "ITBench", "kind": "benchmark", "institutions": [ "ibm", "uiuc" ], "summary": "IT automation scenarios that run in live Kubernetes environments. Agents must diagnose and repair incidents (SRE), assess and enforce compliance (CISO), and find cost problems (FinOps). IBM Research hosts the environments and a leaderboard.", "what_is_measured": "For SRE scenarios, whether the agent localizes the fault and repairs the incident so that the alert clears. The headline number here is the share of SRE incidents resolved. The CISO track scores compliance assessments and policy generation (1.0 is perfect). The FinOps track scores cost optimization and anomaly detection. The ICML 2025 paper covers 102 scenarios; the public repo ships a smaller open-source subset.", "grain": "task", "environment": [ "live-system", "cli" ], "grading": [ "state-check", "automated-tests" ], "task_count": 102, "human_reference": null, "metric": { "name": "SRE incidents resolved", "unit": "percent", "higher_is_better": true, "description": "Share of SRE incidents that the agent repaired, as reported on the ITBench SRE leaderboard." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "devops-sre-it", "weight": "primary" }, { "family": "security", "weight": "partial" } ], "functions": [ "incident-response", "compliance-operations", "finops" ], "links": { "homepage": "https://github.com/itbench-hub/ITBench", "paper": "https://proceedings.mlr.press/v267/jha25a.html", "leaderboard": "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md", "repo": "https://github.com/itbench-hub/ITBench" }, "license": { "data": "Apache-2.0 (repository LICENSE file). Scenario tooling and sample scenarios are open source; hosted environments require registration.", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": null, "limits": [ { "text": "The SRE leaderboard was last updated in May 2025 and lists only IBM's own reference agent with three models. Newer models have not been posted there.", "source_url": "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md" }, { "text": "The paper's headline (11.4% of SRE scenarios resolved) and the leaderboard's best entry (25.0%) use different scenario sets and agents, so they are not directly comparable.", "source_url": "https://proceedings.mlr.press/v267/jha25a.html" }, { "text": "SREGym, a later benchmark from some of the same authors, reports that problems ported from ITBench and AIOpsLab are now easy for strong agents (mitigation above 80%).", "source_url": "https://arxiv.org/abs/2605.07161" }, { "text": "The arXiv v1 abstract (94 scenarios, 13.8% SRE) and the ICML version (102 scenarios, 11.4% SRE) report different counts and rates.", "source_url": "https://arxiv.org/abs/2502.05352" } ], "released": "2025-02-07", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://github.com/itbench-hub/ITBench", "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md", "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_CISO.md", "https://proceedings.mlr.press/v267/jha25a.html", "https://arxiv.org/abs/2502.05352" ], "spot_checked": true, "notes": "Leaderboard \"Updated on: 02/05/2025\" is read as 2 May 2025 because other rows in the same files use DD/MM/YYYY. CISO leaderboard best score is 0.30 (3 scenarios passed) by a UIUC agent. See data/conflicts/itbench-sre-resolved-rate.yaml." }, "institution_names": [ "IBM Research", "UIUC" ], "status": "emerging", "frontier": { "subject": { "label": "ITBench-SRE-Agent-GPT-4o", "model": "GPT-4o", "model_org": "OpenAI", "harness": "ITBench-SRE-Agent", "open_weights": false }, "value": 25, "unit": "percent", "date": "2025-05-02", "frontier": true, "source_url": "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md", "source_kind": "primary", "reported_by": "ibm", "retrieved_at": "2026-09-23", "notes": "Single-trial table, 16 trials across incidents. The multi-trial table (162 trials) shows 24.79% for the same agent. Leaderboard date read as 2 May 2025." } }, { "id": "martian-code-review-bench", "name": "Code Review Bench", "short_name": "CRB", "kind": "benchmark", "institutions": [ "martian" ], "authors": [ "Aleksandr Zverianskii", "Ashley Zhang", "Jacob Clyne", "Antía Garcia", "Fazl Barez", "Shriyash Upadhyay" ], "summary": "Open benchmark for AI code review tools. The offline set runs each tool on the same 50 real pull requests and scores its comments against human-curated golden comments with an LLM judge. A separate online set samples fresh GitHub pull requests where review bots commented and checks which bot suggestions the developer then fixed.", "what_is_measured": "Whether a code review tool finds the real issues in a pull request without adding noise. Offline: 50 PRs from Sentry, Grafana, Cal.com, Discourse, and Keycloak, with 173 golden comments labelled by severity and category. Precision is the share of tool comments that match a golden comment. Recall is the share of golden comments (in the active category profile) that the tool found. The dashboard default combines them as F2 (recall weighted 4x) on the Core profile (158 golden comments: bug, security, concurrency, data, API, perf, test gap, doc defect). Online: per-PR precision and recall against the developer's post-review fixes.", "grain": "task", "environment": [ "repo" ], "grading": [ "llm-judge" ], "task_count": 50, "human_reference": "Golden comments come from human reviewers, so the ceiling is the human-curated issue set. No human reviewer is scored as a row.", "metric": { "name": "F2 score (Core profile, offline set)", "unit": "score", "higher_is_better": true, "description": "F-beta with beta 2 on a 0 to 100 scale, computed from precision and recall against golden comments on the Core category profile. Judge model Claude Opus 4.5 (the dashboard default)." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "code-review" ], "links": { "homepage": "https://codereview.withmartian.com", "leaderboard": "https://codereview.withmartian.com", "repo": "https://github.com/withmartian/code-review-benchmark" }, "license": { "data": "MIT (repo license covers PRs metadata, golden comments, judge prompts, pipeline, and results)", "code": "MIT", "reuse": "open-mit" }, "contamination": "The offline PRs come from well-known public repositories, so tools may have seen them in training. Martian runs the online set on fresh PRs to check for this.", "limits": [ { "text": "Static offline dataset of 50 PRs from public repos. Maintainers say tools may have seen these PRs during training and point to the online set as the check.", "source_url": "https://github.com/withmartian/code-review-benchmark" }, { "text": "Scores depend on the LLM judge. Under the Claude Opus 4.5 judge, Qodo Extended leads; under Claude Sonnet 4.5 and GPT-5.2, Cubic v2 leads. Maintainers say the top 5 tools are the same across judges and most tools move at most 2 ranks.", "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json" }, { "text": "The golden set has sparse coverage of style and nit issues (about 10 across 50 PRs), so correct style comments can still count as false positives.", "source_url": "https://github.com/withmartian/code-review-benchmark/blob/main/offline/README.md" }, { "text": "Rows are review products and coding assistants (for example Qodo, CodeRabbit, Claude Code), not raw models. Martian runs every tool itself and only publishes tools with enough public GitHub usage and attributable bot accounts.", "source_url": "https://github.com/withmartian/code-review-benchmark" } ], "released": "2026-02-26", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://raw.githubusercontent.com/withmartian/code-review-benchmark/HEAD/README.md", "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/README.md", "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/methodology/summary.md", "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "https://api.github.com/repos/withmartian/code-review-benchmark" ], "spot_checked": true, "notes": "codereview.withmartian.com is JavaScript-rendered and did not load; all numbers come from the repo's benchmark_dashboard.json (last updated 2026-08-20). Release date is the date of Martian's launch post on X (status 2027087286773714984, 2026-02-26); the repo was created 2026-02-06. The dashboard defaults are judge anthropic_claude-opus-4-5-20251101, profile core, beta 2.0. The methodology doc says the online set was the headline at launch, but only the offline set has published per-tool numbers in the repo, so we show offline F2. The online set's results live in a database that is not in the repo." }, "institution_names": [ "Martian" ], "status": "strong", "frontier": { "subject": { "label": "Qodo Extended", "model_org": "Qodo", "agent": "Qodo Extended" }, "value": 65.1, "unit": "score", "date": "2026-08-20", "frontier": true, "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "source_kind": "primary", "reported_by": "martian", "retrieved_at": "2026-09-23", "notes": "Offline set, Core profile, judge Claude Opus 4.5 (dashboard default). Precision 67.1, recall 64.6, F1 65.8 across 50 PRs. Under the Claude Sonnet 4.5 judge Qodo Extended scores 56.8 F2 and under GPT-5.2 59.3 F2, where Cubic v2 leads (59.9 under both)." } }, { "id": "medagentbench", "name": "MedAgentBench", "short_name": "MedAgentBench", "kind": "benchmark", "institutions": [ "stanford" ], "authors": [ "Yixing Jiang", "Kameron C. Black", "Gloria Geng", "Danny Park", "James Zou", "Andrew Y. Ng", "Jonathan H. Chen" ], "summary": "A Stanford benchmark of 300 clinical tasks that a physician might hand to an assistant inside an electronic health record. An agent works through a FHIR API against a virtual EHR that holds 100 de-identified patient profiles, for example to look up a lab value, record vitals, or place an order.", "what_is_measured": "Task success on one try (pass@1). Two internal medicine physicians wrote the 300 tasks in 10 categories, from patient lookup and lab retrieval to ordering tests, referrals, and medications. Half the tasks only need GET requests (query tasks); half need POST requests that change records (action tasks). Query tasks are graded against reference answers, and action tasks by rule-based checks of the request payload. The agent gets nine FHIR functions and at most eight rounds.", "grain": "task", "environment": [ "api-tools" ], "grading": [ "automated-tests", "state-check" ], "task_count": 300, "human_reference": null, "metric": { "name": "Task success rate", "unit": "percent", "higher_is_better": true, "description": "Share of the 300 tasks that the agent completes correctly on a single attempt." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "healthcare", "soc": [ "29-1216", "29-2072" ], "weight": "primary" } ], "functions": [ "clinical-records" ], "links": { "homepage": "https://stanfordmlgroup.github.io/projects/medagentbench/", "paper": "https://arxiv.org/abs/2501.14654", "repo": "https://github.com/stanfordmlgroup/MedAgentBench" }, "license": { "data": "Code is MIT (repository license). The arXiv paper is CC BY 4.0. Patient data derives from de-identified Stanford STARR records and ships as a Docker image; the reference solutions are distributed separately from a Stanford Medicine Box link.", "code": "MIT", "reuse": "cite-only" }, "contamination": "Tasks and the FHIR environment are public. Timestamps in the patient data are jittered at the patient level.", "limits": [ { "text": "Published scores are from the January 2025 paper (Claude 3.5 Sonnet v2, GPT-4o, DeepSeek-V3 and others). There is no maintained leaderboard, so newer models are not tracked here.", "source_url": "https://stanfordmlgroup.github.io/projects/medagentbench/" }, { "text": "The baseline agent is a simple orchestrator with nine FHIR functions and an eight-round cap. Many failures were format errors, for example answering in a sentence instead of a bare value.", "source_url": "https://arxiv.org/html/2501.14654v2" }, { "text": "Tasks take only a few steps each. The authors kept the first release to 300 tasks and 100 patients to limit evaluation cost.", "source_url": "https://arxiv.org/html/2501.14654v2" } ], "released": "2025-01-24", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2501.14654", "https://arxiv.org/html/2501.14654v2", "https://github.com/stanfordmlgroup/MedAgentBench", "https://stanfordmlgroup.github.io/projects/medagentbench/" ], "spot_checked": false, "notes": "The paper was later published in NEJM AI (doi 10.1056/AIdbp2500144), which we did not open. A third-party blog claims a 2026 re-run with Grok 4.6 at 95.9%; we could not find the primary source and did not use it." }, "institution_names": [ "Stanford University" ], "status": "strong", "frontier": { "subject": { "label": "Claude 3.5 Sonnet v2", "model": "Claude 3.5 Sonnet v2", "model_org": "Anthropic", "harness": "MedAgentBench baseline orchestrator", "open_weights": false }, "value": 69.67, "unit": "percent", "date": "2025-01-24", "frontier": true, "source_url": "https://arxiv.org/html/2501.14654v2", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23", "notes": "Table 3 of the paper. Query tasks 85.33, action tasks 54.00. Temperature 0, single attempt. No newer primary results exist." } }, { "id": "metr-time-horizons", "name": "METR Task-Completion Time Horizons", "short_name": "METR Time Horizon", "kind": "benchmark", "institutions": [ "metr" ], "authors": [ "Thomas Kwa", "Ben West", "Joel Becker", "Amy Deng", "Katharyn Garcia", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Nate Rush", "Sydney Von Arx", "Elizabeth Barnes", "Lawrence Chan" ], "summary": "METR runs AI agents on software, ML, and cybersecurity tasks that skilled humans were timed on, then fits a curve of success against human task length. The 50% time horizon is the human task length at which the agent succeeds half the time. METR reports the frontier horizon has doubled about every 7 months since 2019, and about every 4 months since 2023 under the TH1.1 suite.", "what_is_measured": "How long a task (in expert human time) an agent can complete with 50% reliability. The Time Horizon 1.1 suite has 228 tasks from HCAST, RE-Bench, and short SWAA tasks, with 31 tasks estimated at 8 hours or more for humans. Human times come from contracted professionals with about 5 years of experience, given the same instructions and tools as the agents. Each model runs with a scaffold that METR chooses after a small elicitation phase; runs are checked for reward hacking. METR also reports an 80% horizon, which is several times shorter.", "grain": "task", "environment": [ "cli", "repo" ], "grading": [ "automated-tests", "state-check" ], "task_count": 228, "human_reference": "Skilled contractors (software, ML, cybersecurity; about 5 years of experience) attempt each task; the geometric mean of successful completion times sets the task length. Only 5 of the 31 tasks of 8 or more hours have measured human baselines; the rest use estimates.", "metric": { "name": "50% task-completion time horizon", "unit": "minutes", "higher_is_better": true, "description": "Human task length at which the fitted logistic curve predicts a 50% success rate for the agent. Time Horizon 1.1 suite unless noted." }, "reference_rule": { "kind": "none", "note": "Open-ended scale in human minutes. METR says measurements above 16 hours (960 minutes) are unreliable with the current suite." }, "jobs": [ { "family": "software-engineering", "weight": "primary" }, { "family": "ml-research-engineering", "weight": "partial" }, { "family": "security", "weight": "partial" } ], "functions": [ "long-horizon-autonomy" ], "links": { "homepage": "https://metr.org/time-horizons/", "paper": "https://arxiv.org/abs/2503.14499", "leaderboard": "https://metr.org/time-horizons/", "repo": "https://github.com/METR/eval-analysis-public", "blog": "https://metr.org/blog/2026-1-29-time-horizon-1-1/", "dataset": "https://metr.org/assets/benchmark_results_1_1.yaml" }, "license": { "data": "Horizon estimates and run data are public in eval-analysis-public (no license file seen). Many tasks are private.", "reuse": "cite-only" }, "contamination": "Most tasks are private. SWAA short tasks and some HCAST tasks are described publicly. METR screens runs for reward hacking by keyword search, LLM flags, and human review.", "limits": [ { "text": "Measurements above 16 hours are unreliable with the current task suite (notice added 2026-05-08). The top model, Claude Mythos Preview (early), is at 1,044.8 minutes, above that line, with a 95% CI of 508.9 to 3,304.3 minutes.", "source_url": "https://metr.org/time-horizons/" }, { "text": "Tasks are self-contained and well specified, so the horizon is closer to what a low-context new hire or contractor could do, not a professional with full context. METR says agents do worse on messier tasks and when graded holistically.", "source_url": "https://metr.org/time-horizons/" }, { "text": "Task composition changes the trend. Moving from TH1 to TH1.1 changed the post-2023 doubling time from 165 to 131 days and moved individual estimates by up to 57%. A regularization fix on 2026-03-03 changed estimates again (Claude Opus 4.5 went from 320 to 293 minutes).", "source_url": "https://metr.org/blog/2026-1-29-time-horizon-1-1/" }, { "text": "Coverage is not complete. As of 2026-09-23 METR listed Claude Opus 4.7, Grok 4.3, and GPT-5.5 as recent models without time horizons. Evaluations take 1 to 2 weeks or more.", "source_url": "https://metr.org/time-horizons/" } ], "released": "2025-03-18", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://metr.org/time-horizons/", "https://metr.org/assets/benchmark_results_1_1.yaml", "https://metr.org/blog/2026-1-29-time-horizon-1-1/", "https://arxiv.org/abs/2503.14499", "https://raw.githubusercontent.com/METR/eval-analysis-public/HEAD/README.md", "https://api.github.com/repos/METR/eval-analysis-public" ], "spot_checked": false, "notes": "Author list is truncated to the first ten and the two senior authors of the 26 on the paper. Doubling times from benchmark_results_1_1.yaml (retrieved 2026-09-23): all-time stitched trend 187.8 days; from 2023 on 128.7 days (CI 104.4 to 158.0). The 2026-01-29 blog gave 196.5 days stitched, 130.8 days (CI 107 to 161) since 2023, and 88.6 days since 2024, before the 2026-03-03 regularization fix. The 2025 paper said roughly 7 months. The results page lists Claude Mythos Preview (early) as the latest addition (2026-05-08). The eval-analysis-public repo was last pushed 2026-03-06, so the live YAML on metr.org is the more current source." }, "institution_names": [ "METR" ], "status": "unrated", "frontier": { "subject": { "label": "Claude Mythos Preview (early)", "model": "Claude Mythos Preview (early)", "model_org": "Anthropic", "harness": "METR ReAct agent (Inspect)", "open_weights": false }, "value": 1044.8, "unit": "minutes", "ci": [ 508.9, 3304.3 ], "date": "2026-04-07", "frontier": true, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23", "notes": "About 17.4 hours. Added to the page 2026-05-08 together with a notice that measurements above 16 hours are unreliable with the current task suite. 80% horizon 185.9 minutes. Date is the release_date field in METR's data file." } }, { "id": "mle-bench", "name": "MLE-bench", "short_name": "MLE-bench", "kind": "benchmark", "institutions": [ "openai" ], "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace" ], "summary": "75 Kaggle competitions rebuilt offline. An agent gets the data and description, trains models, and submits predictions. Its score is compared with the real human leaderboard to see whether it would have won a bronze, silver, or gold medal.", "what_is_measured": "Whether an agent can do end-to-end machine learning engineering: prepare data, train and tune models, and produce a valid submission. The headline is the share of competitions where the agent's best submission reaches at least a bronze medal, averaged over at least 3 seeds on all 75 competitions. The repo also reports Low (22 competitions, the \"Lite\" set), Medium, and High splits.", "grain": "project", "environment": [ "cli" ], "grading": [ "outcome-metric" ], "task_count": 75, "human_reference": "Medal thresholds come from the public Kaggle leaderboards, so a medal means the agent beat most human participants in that competition.", "metric": { "name": "Any medal (%)", "unit": "percent", "higher_is_better": true, "description": "Share of the 75 competitions where the agent earns any medal, reported as the mean over seeds." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "ml-research-engineering", "soc": [ "15-1221", "15-2051" ], "weight": "primary" }, { "family": "data-analytics", "weight": "partial" } ], "functions": [ "ml-engineering" ], "links": { "homepage": "https://github.com/openai/mle-bench", "paper": "https://arxiv.org/abs/2410.07095", "leaderboard": "https://github.com/openai/mle-bench", "repo": "https://github.com/openai/mle-bench" }, "license": { "data": "Code is MIT licensed. Competition data comes from Kaggle and users must accept each competition's rules to download it.", "code": "MIT", "reuse": "cite-only" }, "contamination": "The competitions are public and top solutions are online. The paper ran familiarity experiments and found no clear link between a model's familiarity with a competition and its score.", "limits": [ { "text": "Since 2026-04-24 the maintainers accept no new leaderboard submissions while they design a fairer submission process.", "source_url": "https://github.com/openai/mle-bench" }, { "text": "Two submissions (Disarray 77.78%, LoongFlow 62.66%) are listed separately because they used test-set feedback and are not comparable with the main table.", "source_url": "https://github.com/openai/mle-bench" }, { "text": "Known grading issues in several competitions are left unfixed to keep the v1 leaderboard comparable. Fixes are planned for a v2 release.", "source_url": "https://github.com/openai/mle-bench" }, { "text": "Agents run for 24 hours per competition with modern models, while Kaggle participants worked under different conditions, so a medal is not a like-for-like human comparison.", "source_url": "https://arxiv.org/abs/2410.07095" } ], "released": "2024-10-09", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2410.07095", "https://raw.githubusercontent.com/openai/mle-bench/HEAD/README.md", "https://raw.githubusercontent.com/openai/mle-bench/HEAD/LICENSE" ], "spot_checked": false, "notes": "OpenAI maintains the leaderboard and it includes third-party agents. The paper abstract says 16.9% for o1-preview with AIDE; the README table lists 17.12 ± 0.61 for the same setup." }, "institution_names": [ "OpenAI" ], "status": "strong", "frontier": { "subject": { "label": "Famou-Agent 2.0 (Gemini-3-Pro-Preview)", "model": "Gemini 3 Pro Preview", "model_org": "Google", "agent": "Famou-Agent 2.0 (Baidu)", "open_weights": false }, "value": 64.44, "unit": "percent", "date": "2026-02-23", "frontier": true, "source_url": "https://github.com/openai/mle-bench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23", "notes": "64.44 ± 1.18 over 24 hours per competition. Top of the main leaderboard table. Disarray reports 77.78 but used test-set feedback and is listed separately as not comparable." } }, { "id": "osworld-verified", "name": "OSWorld-Verified", "short_name": "OSWorld", "kind": "benchmark", "institutions": [ "hku", "salesforce", "cmu", "university-of-waterloo" ], "authors": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "summary": "A real Ubuntu desktop in a virtual machine with 369 computer tasks across Chrome, LibreOffice, GIMP, VLC, VS Code, Thunderbird, the file system, and multi-app workflows. The agent sees the screen and acts with mouse and keyboard. A script checks the final state of the machine.", "what_is_measured": "Share of desktop tasks an agent finishes correctly. Each task starts from a set initial state and ends with an execution-based check of files, application state, or settings. OSWorld-Verified (July 2025) fixed about 300 reported problems with tasks and checkers and moved evaluation to a parallel AWS setup. Eight Google Drive tasks may be skipped, giving a 361-task set. The official board runs agents at a 100-step budget.", "grain": "task", "environment": [ "computer-use" ], "grading": [ "state-check" ], "task_count": 369, "human_reference": "Humans completed 72.36% of tasks in the original 2024 study. The OSWorld-Verified report cites human performance as about 72%.", "metric": { "name": "Success rate", "unit": "percent", "higher_is_better": true, "description": "Share of tasks whose final-state check passes, on the 361-task set (or 369 with Google Drive tasks)." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "office-admin", "weight": "primary" }, { "family": "devops-sre-it", "weight": "partial" }, { "family": "design-media-writing", "weight": "partial" } ], "functions": [ "computer-use", "spreadsheet-work" ], "links": { "homepage": "https://os-world.github.io", "paper": "https://arxiv.org/abs/2404.07972", "leaderboard": "https://os-world.github.io/#benchmark", "repo": "https://github.com/xlang-ai/OSWorld", "blog": "https://xlang.ai/blog/osworld-verified" }, "license": { "data": "Apache 2.0 on the GitHub repository. Windows tasks need a licensed image.", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": "All tasks and checkers are public. The maintainers have not published a contamination analysis. Some tasks depend on live websites that change over time.", "limits": [ { "text": "Agents now score above the 72% human figure, and the maintainers released OSWorld 2.0 in June 2026 as the next version with 108 longer tasks. OSWorld-Verified is still maintained but is no longer the hardest test.", "source_url": "https://os-world.github.io" }, { "text": "The original task set avoided long tasks, deep professional software, video, and real-time work, so it under-covers those parts of computer use.", "source_url": "https://xlang.ai/blog/osworld-verified" }, { "text": "Scores depend on the step budget. The same model can score much higher at 100 steps than at 15, so compare entries at the same budget.", "source_url": "https://xlang.ai/blog/osworld-verified" }, { "text": "Verified entries need the maintainers to run the agent on their own AWS platform, so the board lags model releases and lab-reported numbers (for example GPT-5.5 at 78.7%) are not on it.", "source_url": "https://os-world.github.io" } ], "released": "2025-07-28", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://os-world.github.io", "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "https://xlang.ai/blog/osworld-verified", "https://arxiv.org/abs/2404.07972", "https://github.com/xlang-ai/OSWorld", "https://osworld-v2.xlang.ai/" ], "spot_checked": false, "notes": "The original OSWorld paper is from April 2024; the released date here is the OSWorld-Verified update. The leaderboard renders from an xlsx file on the site, which we read directly. Success rates in that file are on 357 to 361 tasks depending on the run. The maintainers describe the verified results as run by their own team under unified settings." }, "institution_names": [ "HKU", "Salesforce AI Research", "Carnegie Mellon University", "University of Waterloo" ], "status": "saturated", "frontier": { "subject": { "label": "Intelligence-Indeed Agent", "model_org": "Intelligence Indeed", "agent": "Intelligence-Indeed Agent" }, "value": 90.19, "unit": "percent", "date": "2026-07-25", "frontier": true, "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23", "notes": "Rank 1 on the verified board as of 2026-09-23. Agentic framework, 100 steps, 325.59 of 361 tasks. The board does not name the underlying model. Verified runs are executed by the OSWorld team." } }, { "id": "paperbench", "name": "PaperBench", "short_name": "PaperBench", "kind": "benchmark", "institutions": [ "openai" ], "authors": [ "Giulio Starace", "Oliver Jaffe", "Dane Sherburn", "James Aung", "Jun Shern Chan", "Tejal Patwardhan" ], "summary": "An agent gets one of 20 ICML 2024 Spotlight or Oral papers and must replicate it from scratch: understand the contributions, write a codebase, and run the experiments. Rubrics written with the paper authors break each paper into 8,316 gradable items, and an LLM judge scores the result.", "what_is_measured": "How much of a research paper's empirical results an agent can reproduce with no access to the authors' code. Rubric leaves cover code development, execution, and result match. The headline is the average replication score across the 20 papers. A lighter Code-Dev variant skips execution and grades only code development; its scores are much higher and are not comparable.", "grain": "project", "environment": [ "cli", "repo" ], "grading": [ "llm-judge", "rubric" ], "task_count": 20, "human_reference": "ML PhDs (best of 3 attempts) scored 41.4% on a 3-paper subset after 48 hours, against 26.6% for o1 on the same subset.", "metric": { "name": "Average replication score", "unit": "percent", "higher_is_better": true, "description": "Mean rubric score across the 20 papers on the full PaperBench pipeline, judged by SimpleJudge with o3-mini, averaged over 3 runs." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "ml-research-engineering", "soc": [ "15-1221" ], "weight": "primary" }, { "family": "science-research", "weight": "partial" } ], "functions": [ "research-replication" ], "links": { "homepage": "https://openai.com/index/paperbench/", "paper": "https://arxiv.org/abs/2504.01848", "leaderboard": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "repo": "https://github.com/openai/frontier-evals/tree/main/project/paperbench" }, "license": { "data": "Code and rubrics are in the MIT-licensed openai/frontier-evals repo. Papers belong to their authors.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Agents are told not to use the authors' public code, and a monitor checks for blacklisted repositories. The papers and their code are public, so training data may include them.", "limits": [ { "text": "The official leaderboard has not changed since 2025-04-02. Newer models are reported only in lab posts, usually on the Code-Dev variant with different judges, so they are not comparable.", "source_url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench" }, { "text": "Grading uses an LLM judge. The paper's JudgeEval shows the judge agrees with humans imperfectly, and full grading is expensive.", "source_url": "https://arxiv.org/abs/2504.01848" }, { "text": "The human baseline covers only 3 of 20 papers and 8 participants, and models were run under slightly different conditions.", "source_url": "https://arxiv.org/html/2504.01848v3" }, { "text": "A Qwen3.8 model card lists PaperBench scores of 64.8 to 93.0 for several models, but those are Code-Dev runs judged by Claude Opus 4.6. Aggregators show them as full PaperBench scores.", "source_url": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" } ], "released": "2025-04-02", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2504.01848", "https://arxiv.org/html/2504.01848v3", "https://raw.githubusercontent.com/openai/frontier-evals/HEAD/project/paperbench/README.md", "https://openai.com/index/paperbench/", "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B/raw/main/README.md" ], "spot_checked": false, "notes": "OpenAI built the benchmark and evaluated its own models, so its o1 results are source_kind primary. The repo moved from openai/preparedness to openai/frontier-evals. BenchmarkList lists Qwen3.8 Max at 93.0% and others; the Qwen model card footnote says those are Code-Dev mode, judged by Claude Opus 4.6, 3 runs, 12-hour limit. We do not show them as headline results." }, "institution_names": [ "OpenAI" ], "status": "emerging", "frontier": { "subject": { "label": "IterativeAgent o1-high (36h limit)", "model": "o1", "model_org": "OpenAI", "harness": "IterativeAgent", "open_weights": false }, "value": 26, "unit": "percent", "date": "2025-04-02", "frontier": true, "source_url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23", "notes": "26.0 ± 0.3 over 3 runs with a 36-hour limit. Top of the official leaderboard, which has not been updated since launch. OpenAI reporting its own model." } }, { "id": "pr-arena", "name": "PR Arena", "kind": "leaderboard", "institutions": [ "aavetis" ], "summary": "A public tracker that counts pull requests opened on GitHub by coding agents (Copilot, Codex, Cursor, Devin, Codegen, Jules) and how many of them were merged. It uses GitHub search queries on branch prefixes and bot authors and refreshes several times a day.", "what_is_measured": "The share of an agent's non-draft (\"ready\") pull requests on public GitHub that were merged. The maintainer uses ready PRs rather than all PRs so that agents that open drafts first (Copilot, Codegen) can be compared with agents that open ready PRs directly (Codex). Counts come from GitHub search, for example is:pr head:copilot/ and is:pr author:devin-ai-integration[bot].", "grain": "task", "environment": [ "field-data" ], "grading": [ "field-signal" ], "task_count": null, "human_reference": null, "metric": { "name": "Merge rate of ready PRs", "unit": "percent", "higher_is_better": true, "description": "Merged PRs divided by non-draft PRs, from GitHub search counts, across all public repositories." }, "reference_rule": { "kind": "none", "note": "Field signal. Merge rates reflect who uses each agent and where, not a controlled task set." }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "shipping-prs" ], "links": { "homepage": "https://prarena.ai", "leaderboard": "https://prarena.ai", "repo": "https://github.com/aavetis/PRarena", "dataset": "https://github.com/aavetis/PRarena/blob/main/data.csv" }, "license": { "data": "No license file in the repo. Data is derived from public GitHub search counts.", "reuse": "unknown" }, "contamination": null, "limits": [ { "text": "Counts depend on GitHub search heuristics (branch prefixes such as head:copilot/ and bot authors). Agent PRs that do not match these patterns are not counted.", "source_url": "https://github.com/aavetis/PRarena" }, { "text": "The merged count can exceed the ready count. On 2026-09-23 Cursor showed 910,955 merged against 894,225 ready PRs (a 101.87% rate). The maintainer does not explain the cause. We do not show Cursor as the frontier for this reason.", "source_url": "https://github.com/aavetis/PRarena" }, { "text": "Merge rate is a selection signal, not a quality score. Agents that require a human to mark a PR ready (Copilot) get a high ready-PR merge rate because humans filter first.", "source_url": "https://github.com/aavetis/PRarena" } ], "released": "2025-05-26", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://raw.githubusercontent.com/aavetis/PRarena/HEAD/README.md", "https://raw.githubusercontent.com/aavetis/PRarena/main/data.csv", "https://raw.githubusercontent.com/aavetis/PRarena/main/docs/chart-data.json", "https://raw.githubusercontent.com/aavetis/PRarena/main/docs/index.html", "https://api.github.com/repos/aavetis/PRarena" ], "spot_checked": true, "notes": "Release date is the first row of data.csv (2025-05-26); the repo was created 2025-05-22. The site footer credits \"by aavetis\". Rates were computed from data.csv (merged / nondraft) and match the README \"Current Statistics\" table for 2026-09-23." }, "institution_names": [ "aavetis" ], "status": "unrated", "frontier": { "subject": { "label": "Copilot", "model_org": "GitHub", "agent": "GitHub Copilot coding agent" }, "value": 96.02, "unit": "percent", "date": "2026-09-23", "frontier": true, "source_url": "https://github.com/aavetis/PRarena", "source_kind": "primary", "reported_by": "aavetis", "retrieved_at": "2026-09-23", "notes": "README \"Current Statistics\" table: 1,548,258 merged of 1,612,469 ready PRs. Codex was at 89.65% (6,343,777 of 7,075,934), Jules 85.47%, Devin 61.94%, Codegen 61.6%. Cursor showed 101.87%, which is a counting artifact, so it is excluded." } }, { "id": "re-bench", "name": "RE-Bench", "short_name": "RE-Bench", "kind": "benchmark", "institutions": [ "metr" ], "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Elizabeth Barnes" ], "summary": "Seven open-ended ML research engineering environments built from scratch by METR, for example writing a faster GPU kernel, fixing a corrupted model embedding, or inferring a scaling law. Humans and agents get the same machine, GPUs, and scoring function, and try to push the score as high as they can in a fixed time.", "what_is_measured": "How an agent's research engineering output compares with human experts under the same conditions. Each environment's raw score is normalized so the starting solution is 0 and METR's reference solution is 1. METR reports the average normalized score by total time budget, using best-of-k over shorter runs for agents. The comparison set is 71 eight-hour attempts by 61 ML experts, whose average normalized score was 0.64. We track the human-expert percentile that METR assigns to an agent at an 8-hour budget.", "grain": "task", "environment": [ "cli" ], "grading": [ "outcome-metric" ], "task_count": 7, "human_reference": "71 eight-hour attempts by 61 human experts (METR hiring applicants, professional network, and graduate students). Average normalized score 0.64; 82% of attempts scored above zero and 24% matched or beat the reference solution.", "metric": { "name": "Human-expert percentile matched (8-hour budget)", "unit": "percent", "higher_is_better": true, "description": "Percentile of METR's human expert 8-hour attempts that the agent's average normalized score matches when the agent also gets an 8-hour total budget. 50 means the median expert." }, "reference_rule": { "kind": "parity", "value": 50, "note": "Median human expert attempt over 8 hours." }, "jobs": [ { "family": "ml-research-engineering", "soc": [ "15-1221" ], "weight": "primary" } ], "functions": [ "autonomous-rd", "long-horizon-autonomy", "performance-engineering" ], "links": { "homepage": "https://metr.org/blog/2024-11-22-evaluating-r-d-capabilities-of-llms/", "paper": "https://arxiv.org/abs/2411.15114", "repo": "https://github.com/METR/RE-Bench", "blog": "https://metr.org/blog/2025-01-31-update-sonnet-o1-evals/" }, "license": { "data": "Repo is MIT licensed. Reference solutions are password-protected. METR asks users to keep the tasks out of training data and not to publish solutions.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Environments were created from scratch and had not appeared in training data at launch. Public discussion of solutions since then may contaminate future runs. METR keeps other environments held out.", "limits": [ { "text": "Only 7 environments. METR's later reports often use a 5-task subset and fold RE-Bench into time-horizon estimates, so there is no single up-to-date leaderboard number.", "source_url": "https://metr.org/evaluations/openai-o3-report/" }, { "text": "Agents beat humans at a 2-hour budget (about 4 times the human score) but humans pull ahead at 8 hours and reach about twice the best agent at 32 hours, so the headline depends on the budget.", "source_url": "https://arxiv.org/abs/2411.15114" }, { "text": "Agents can run the scoring function at will, and METR found reward hacking, for example faking a training run's output. Detected hacks are scored as failures.", "source_url": "https://metr.org/blog/2024-11-22-evaluating-r-d-capabilities-of-llms/" }, { "text": "Environments have clear goals and fast feedback, unlike much real research, and human scores vary a lot by recruiting source (0.48 for hiring applicants, 0.98 for professional network).", "source_url": "https://arxiv.org/html/2411.15114v2" } ], "released": "2024-11-22", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2411.15114", "https://arxiv.org/html/2411.15114v2", "https://metr.org/blog/2024-11-22-evaluating-r-d-capabilities-of-llms/", "https://metr.org/blog/2025-01-31-update-sonnet-o1-evals/", "https://metr.org/evaluations/claude-3-7-report/", "https://metr.org/evaluations/openai-o3-report/", "https://metr.org/evaluations/gpt-5-report/", "https://metr.org/evaluations/gpt-5-1-codex-max-report/", "https://github.com/METR/RE-Bench" ], "spot_checked": false, "notes": "The paper gives agent scores only in charts, so we do not record normalized scores for agents. The percentile metric comes from METR's 2025-01-31 update. Later METR reports say Claude 3.7 Sonnet with a 32-hour budget matched the median human 8-hour attempt on a 5-task subset, and o4-mini with a 32-hour budget exceeded the 50th percentile on the same subset; neither is an 8-hour figure, so they are not entered as results. GPT-5 and GPT-5.1-Codex-Max reports give no separate RE-Bench score." }, "institution_names": [ "METR" ], "status": "emerging", "frontier": { "subject": { "label": "Claude 3.5 Sonnet (New) (Modular scaffold)", "model": "Claude 3.5 Sonnet (New)", "model_org": "Anthropic", "harness": "Modular", "open_weights": false }, "value": 37, "unit": "percent", "date": "2025-01-31", "frontier": true, "source_url": "https://metr.org/blog/2025-01-31-update-sonnet-o1-evals/", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23", "notes": "METR says Claude 3.5 Sonnet performed comparable to a 37th-percentile human expert at 8 hours per task. This is the latest 8-hour, 7-task figure METR has published. Newer models were evaluated on a 5-task subset at a 32-hour budget and are not comparable." } }, { "id": "remote-labor-index", "name": "Remote Labor Index", "short_name": "RLI", "kind": "benchmark", "institutions": [ "cais", "scale-ai" ], "summary": "Real freelance projects, collected from professionals on Upwork, that an agent must deliver end to end. Trained human evaluators compare each AI deliverable with the deliverable that a paid professional made.", "what_is_measured": "Whether a reasonable client would accept the agent's deliverable, compared with a gold-standard deliverable from a professional freelancer. Projects come from 23 Upwork domains, for example 3D and CAD, architecture, graphic design, video and animation, audio, data analysis, and web apps.", "grain": "project", "environment": [ "cli", "computer-use" ], "grading": [ "human-expert", "pairwise-human" ], "task_count": 240, "human_reference": "A professional freelancer's accepted deliverable. Mean human completion time 28.9 hours (median 11.5 hours). Mean project value $632.60 (median $200).", "metric": { "name": "Automation rate", "unit": "percent", "higher_is_better": true, "description": "Share of projects where the AI deliverable is judged at least as good as the professional deliverable." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "design-media-writing", "weight": "primary" }, { "family": "architecture-engineering", "weight": "partial" }, { "family": "data-analytics", "weight": "partial" }, { "family": "software-engineering", "weight": "partial" } ], "functions": [ "freelance-projects", "professional-deliverables" ], "links": { "homepage": "https://www.remotelabor.ai/", "paper": "https://arxiv.org/abs/2510.26787", "leaderboard": "https://labs.scale.com/leaderboard/rli", "blog": "https://safe.ai/blog/significant-increase-in-digital-labor-automation" }, "license": { "data": "Scores come from 230 private projects. 10 public projects and the open-source evaluation platform are released for qualitative analysis.", "reuse": "cite-only" }, "contamination": "Leaderboard scores use a private set of 230 projects.", "limits": [ { "text": "Excludes projects that need direct client interaction, physical labor, or long-term evaluation (for example SEO).", "source_url": "https://labs.scale.com/leaderboard/rli" }, { "text": "Trained experts grade every deliverable by hand, so new agents are tested at a low frequency.", "source_url": "https://labs.scale.com/leaderboard/rli" }, { "text": "Agents had a maximum generation budget of $30 per project.", "source_url": "https://labs.scale.com/leaderboard/rli" } ], "released": "2025-10", "last_checked": "2026-09-23", "verification": { "researched_by": "main agent, 2026-09-23", "primary_sources_checked": [ "https://labs.scale.com/leaderboard/rli", "https://safe.ai/blog/significant-increase-in-digital-labor-automation" ], "spot_checked": true, "notes": "Aggregators show 16.1% for Fable 5; CAIS shows 15.8%. See data/conflicts/rli-fable-5-automation-rate.yaml." }, "institution_names": [ "CAIS", "Scale AI" ], "status": "open", "frontier": { "subject": { "label": "Claude Fable 5", "model": "Claude Fable 5", "model_org": "Anthropic", "open_weights": false }, "value": 15.8, "unit": "percent", "date": "2026-07-01", "frontier": true, "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation", "source_kind": "primary", "reported_by": "cais", "retrieved_at": "2026-09-23", "notes": "CAIS says the three new models were paired with stronger agent scaffolding." } }, { "id": "spider-2", "name": "Spider 2.0", "short_name": "Spider 2.0", "kind": "benchmark", "institutions": [ "hku", "salesforce" ], "authors": [ "Fangyu Lei", "Jixuan Chen", "Yuxiao Ye", "Ruisheng Cao", "Dongchan Shin", "Tao Yu" ], "summary": "Enterprise text-to-SQL workflow problems built from real business databases, often with more than 1,000 columns, hosted on BigQuery, Snowflake, SQLite, and DuckDB (dbt). An agent must explore the schema, read dialect documentation, and write long, multi-step SQL to return the right result table.", "what_is_measured": "Whether an agent can turn a business question into correct SQL against a large real-world warehouse. The headline here is Spider 2.0-Snow: 547 questions on Snowflake, graded by comparing the returned table with a gold table. The site also runs Spider 2.0-Lite (547 questions across BigQuery, Snowflake, and SQLite) and Spider 2.0-DBT (68 repository-level dbt tasks). We chose Snow because it is the default tab, has the most entries (64 on 2026-09-23), and is free to run.", "grain": "task", "environment": [ "cli", "api-tools" ], "grading": [ "automated-tests" ], "task_count": 547, "human_reference": null, "metric": { "name": "Execution accuracy (Spider 2.0-Snow)", "unit": "percent", "higher_is_better": true, "description": "Share of the 547 Snow questions where the submitted result table matches the gold table under the official evaluation suite. Entries that use oracle (ground-truth) tables are excluded from ranking." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "data-analytics", "soc": [ "15-1243", "15-1242", "15-2051" ], "weight": "primary" }, { "family": "software-engineering", "weight": "partial" } ], "functions": [ "data-engineering" ], "links": { "homepage": "https://spider2-sql.github.io", "paper": "https://arxiv.org/abs/2411.07763", "leaderboard": "https://spider2-sql.github.io", "repo": "https://github.com/xlang-ai/Spider2" }, "license": { "data": "Repository is MIT licensed. All examples and gold answers were released for self-evaluation in December 2024. Maintainers ask users not to fine-tune on the gold SQL.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Gold SQL and answers are public since 2024-12-24. Annotators rewrote each SQL to reduce leakage from the source projects. The maintainers ask users not to use the gold SQL for supervised fine-tuning.", "limits": [ { "text": "Leaderboard scores are self-reported by teams and then validated by the maintainers. Many top entries are closed commercial agents with no paper or code.", "source_url": "https://spider2-sql.github.io" }, { "text": "The evaluation suite was fixed on 2025-10-29 and affected scores were refreshed, so scores from before that date may not match current runs.", "source_url": "https://github.com/xlang-ai/Spider2" }, { "text": "Snow scores above 96% since March 2026 leave little headroom on this variant. Lite (76.23 best) and DBT (65.6 best) still have room.", "source_url": "https://spider2-sql.github.io" }, { "text": "The shared Snowflake evaluation account was suspended in August 2026, which disrupted access to Spider 2.0-Snow.", "source_url": "https://github.com/xlang-ai/Spider2" } ], "released": "2024-11-12", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://spider2-sql.github.io", "https://arxiv.org/abs/2411.07763", "https://arxiv.org/html/2411.07763v2", "https://github.com/xlang-ai/Spider2" ], "spot_checked": false, "notes": "ICLR 2025 oral. Authors are from HKU (XLANG Lab), Google DeepMind, Google Cloud AI Research, Salesforce Research, Sea AI Lab, and the University of Waterloo. Only HKU and Salesforce have institution files. Leaderboard values were read from the data-date rows of the Snow table in the page HTML. The paper abstract gives 21.3% for o1-preview on the original Spider 2.0 setting, which was retired on 2025-05-22 and replaced by Spider 2.0-DBT." }, "institution_names": [ "HKU", "Salesforce AI Research" ], "status": "saturated", "frontier": { "subject": { "label": "Genloop's Sentinel Agent v2 Pro", "model": null, "model_org": null, "agent": "Sentinel Agent v2 Pro (Genloop)" }, "value": 96.7, "unit": "percent", "date": "2026-03-01", "frontier": true, "source_url": "https://spider2-sql.github.io", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23", "notes": "Top of the Spider 2.0-Snow table on 2026-09-23. The entry does not name the underlying model. Native mini (usenative.ai) is second at 96.53, dated 2026-02-18." } }, { "id": "spreadsheetbench", "name": "SpreadsheetBench", "short_name": "SpreadsheetBench", "kind": "benchmark", "institutions": [ "renmin-university" ], "authors": [ "Zeyao Ma", "Bohan Zhang", "Jing Zhang", "Jifan Yu", "Jie Tang" ], "summary": "912 spreadsheet manipulation questions taken from real Excel forum posts, each paired with the user's actual workbook. Workbooks have multiple tables, odd layouts, and non-text elements. A solution is checked like an online judge: it must work on several test-case spreadsheets with different values.", "what_is_measured": "Whether a system can produce the exact cell values a user asked for in a real spreadsheet, across cell-level and sheet-level edits. The headline is overall pass@1 on the full 912-question V1 set. The site also keeps a 400-question expert-verified V1 subset (released December 2025, best 99.25%) and SpreadsheetBench 2 (321 workflow tasks on financial modeling, debugging, and charts; best 50.86%).", "grain": "task", "environment": [ "documents", "cli" ], "grading": [ "state-check" ], "task_count": 912, "human_reference": "Four Excel experts on a 50-instruction subset (3 test cases each) scored 71.33% under the soft restriction and 62.00% under the hard restriction. GPT-4o scored 18.35% and 15.02% on the same measures in the paper.", "metric": { "name": "Overall pass@1 (V1, 912 questions)", "unit": "percent", "higher_is_better": true, "description": "Share of the 912 questions answered correctly across all 2,729 test-case spreadsheets, as listed on the official leaderboard." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "office-admin", "soc": [ "43-9021", "43-3031" ], "weight": "primary" }, { "family": "data-analytics", "weight": "partial" }, { "family": "finance-accounting", "weight": "partial" } ], "functions": [ "spreadsheet-work" ], "links": { "homepage": "https://spreadsheetbench.github.io", "paper": "https://arxiv.org/abs/2406.14991", "leaderboard": "https://spreadsheetbench.github.io", "repo": "https://github.com/RUCKBReasoning/SpreadsheetBench", "dataset": "https://huggingface.co/datasets/KAKA22/SpreadsheetBench" }, "license": { "data": "CC BY-SA 4.0, stated in the paper's maintenance plan and the repo README.", "reuse": "open-other" }, "contamination": "Questions were rewritten from forum posts by GPT-4 and annotators, and spreadsheet values were changed to build test cases, so exact forum solutions do not apply directly. The full data set and answers are public.", "limits": [ { "text": "Some leaderboard rows are marked unverified. They come from outside evaluations by OpenAI and Microsoft, not from the maintainers' own runs.", "source_url": "https://spreadsheetbench.github.io" }, { "text": "The site's hard-coded \"Top Score\" box showed 70.48% on 2026-09-23 while its leaderboard data file listed 83.11%. See data/conflicts/spreadsheetbench-v1-top-score.yaml.", "source_url": "https://spreadsheetbench.github.io" }, { "text": "Most top entries are commercial spreadsheet products (WPS, Google Sheets, Univer) that do not disclose the model or scaffold.", "source_url": "https://spreadsheetbench.github.io" }, { "text": "The human baseline covers only 50 of 912 instructions, so it is not directly comparable with leaderboard scores.", "source_url": "https://arxiv.org/html/2406.14991v2" } ], "released": "2024-06-21", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://spreadsheetbench.github.io", "https://spreadsheetbench.github.io/data/leaderboard-v1-full.json", "https://spreadsheetbench.github.io/data/leaderboard-v1-verified.json", "https://spreadsheetbench.github.io/data/leaderboard-v2-full.json", "https://arxiv.org/abs/2406.14991", "https://arxiv.org/html/2406.14991v2", "https://github.com/RUCKBReasoning/SpreadsheetBench" ], "spot_checked": false, "notes": "NeurIPS 2024 spotlight. Paper authors are from Renmin University of China, Tsinghua University, and Zhipu AI; the RUC KBReasoning group maintains the benchmark. SpreadsheetBench 2 lists contributors from Renmin, Aptura.AI, AfterQuery, and Shortcut.AI. Leaderboard values come from the site's JSON data files, which the page loads at runtime." }, "institution_names": [ "Renmin University of China" ], "status": "strong", "frontier": { "subject": { "label": "Qingqiu Agent", "model_org": "Kingsoft Office", "agent": "Qingqiu Agent" }, "value": 83.11, "unit": "percent", "date": "2026-06-23", "frontier": true, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "reported_by": "renmin-university", "retrieved_at": "2026-09-23", "notes": "Verified entry at the top of the V1 Full (912) table in the leaderboard data file on 2026-09-23. JT AlphaData (CMCC JIUTIAN) is second at 77.85, dated 2026-08-15." } }, { "id": "sregym", "name": "SREGym", "short_name": "SREGym", "kind": "benchmark", "institutions": [ "uiuc", "university-of-toronto" ], "summary": "A live Kubernetes benchmark for SRE agents. Fault injectors break real cloud-native applications (up to 40 microservices), optionally with background noise, and the agent must find the root cause and fix the system. It builds on AIOpsLab and ITBench and adds OS-level, hardware, metastable, and concurrent failures.", "what_is_measured": "End-to-end success: the agent must both diagnose the root cause correctly (an LLM judge with a fixed checklist) and mitigate the failure (checked programmatically) in the same run. Diagnosis and mitigation rates are also reported, along with time to diagnose, time to mitigate, and tokens. The full suite has 90 problems; SREGym-Lite is a 21-problem subset.", "grain": "task", "environment": [ "live-system", "cli" ], "grading": [ "state-check", "llm-judge" ], "task_count": 90, "human_reference": null, "metric": { "name": "End-to-end success rate", "unit": "percent", "higher_is_better": true, "description": "Share of runs where diagnosis and mitigation both succeed on the 90-problem suite, without injected noise." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "devops-sre-it", "weight": "primary" } ], "functions": [ "incident-response", "terminal-operations" ], "links": { "homepage": "https://sregym.com", "paper": "https://arxiv.org/abs/2605.07161", "leaderboard": "https://sregym.com/leaderboard", "repo": "https://github.com/SREGym/SREGym" }, "license": { "data": "MIT (GitHub repository license). Paper is CC BY 4.0.", "code": "MIT", "reuse": "open-mit" }, "contamination": null, "limits": [ { "text": "Diagnosis is graded by an LLM judge (Claude Sonnet 4.6 in the paper), which adds variance that programmatic checks do not have.", "source_url": "https://arxiv.org/abs/2605.07161" }, { "text": "Deployed applications are far smaller than production systems; the largest has 40 microservices.", "source_url": "https://arxiv.org/abs/2605.07161" }, { "text": "Scores fall with injected noise. Claude Code with Sonnet 4.6 drops from 60.7% to 53.7% end to end. Leaderboard entries for newer models are without noise.", "source_url": "https://sregym.com/leaderboard" }, { "text": "SREGym-Lite leaderboard scores (best 81.0%) use a 21-problem subset and are not comparable with the 90-problem suite.", "source_url": "https://sregym.com/leaderboard" } ], "released": "2026-05-08", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2605.07161", "https://arxiv.org/html/2605.07161v3", "https://sregym.com", "https://sregym.com/leaderboard", "https://github.com/SREGym/SREGym" ], "spot_checked": false, "notes": "Affiliations confirmed from the paper header (UIUC and University of Toronto). Leaderboard entries carry no dates; the full-suite cohort is labeled SREGym-0508 (90 faults). The repo was created 2025-05-19; the paper appeared 2026-05-08." }, "institution_names": [ "UIUC", "University of Toronto" ], "status": "strong", "frontier": { "subject": { "label": "GitHub Copilot (GPT-5.6 Sol, max)", "model": "GPT-5.6 Sol", "model_org": "OpenAI", "agent": "GitHub Copilot", "open_weights": false }, "value": 72.2, "unit": "percent", "date": "2026", "frontier": true, "source_url": "https://sregym.com/leaderboard", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23", "notes": "Full 90-problem suite (cohort SREGym-0508), without noise. Diagnosis 83.3%, mitigation 83.7%. The leaderboard does not date its entries." } }, { "id": "swe-bench-pro", "name": "SWE-Bench Pro", "short_name": "SWE-Bench Pro", "kind": "benchmark", "institutions": [ "scale-ai" ], "authors": [ "Xiang Deng", "Jeff Da", "Edwin Pan", "Yannis Yiming He", "Charles Ide", "Kanak Garg", "Niklas Lauffer", "Sean Hendryx", "Zifan Wang", "Bing Liu", "Brad Kenstler" ], "summary": "Long-horizon software engineering tasks taken from the commit history of 41 repositories: 11 public copyleft (GPL) repositories, 12 held-out copyleft repositories, and 18 private startup codebases. Human engineers built each environment and rewrote each task into a problem statement, a requirements brief, and an optional interface.", "what_is_measured": "Whether an agent's patch makes the new fail-to-pass tests pass without breaking the existing pass-to-pass tests. Tasks are bug fixes, features, optimizations, security updates, and UI changes in Python, Go, JavaScript, and TypeScript. Reference patches average 107.4 changed lines across 4.1 files. Scale reports separate leaderboards for the public set (731 tasks) and the private commercial set (276 tasks).", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 1865, "human_reference": "Scale says tasks may take a professional software engineer hours to days. No measured human baseline is published.", "metric": { "name": "Resolve rate (public set)", "unit": "percent", "higher_is_better": true, "description": "Share of public-set tasks where the patch passes all fail-to-pass and pass-to-pass tests. The leaderboard shows a 95% confidence interval and ranks by its upper bound." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "issue-resolution", "feature-development" ], "links": { "homepage": "https://scale.com/research/swe_bench_pro", "paper": "https://arxiv.org/abs/2509.16941", "leaderboard": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "repo": "https://github.com/scaleapi/SWE-bench_Pro-os", "dataset": "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro" }, "license": { "data": "Public-set tasks come from GPL-licensed repositories and stay under those licenses. The harness is MIT. The held-out and commercial sets are not released.", "code": "MIT", "reuse": "cite-only" }, "contamination": "Public tasks come from strong-copyleft repositories, which Scale says are less likely to be in training data. The private set is not public. OpenAI's contamination probes found rarer and weaker leakage than on SWE-bench Verified, and no verbatim gold patch.", "limits": [ { "text": "OpenAI audited the public set in July 2026 and estimated that about 30% of tasks are broken (overly strict tests, underspecified prompts, low-coverage tests, misleading prompts). It withdrew its recommendation to use the benchmark.", "source_url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations/" }, { "text": "Scale released SWE-Bench Pro V2 on 2026-09-22 with 642 validated tasks and a locked offline protocol. The leaderboard still shows V1 (731-task) results, so numbers will not transfer to V2.", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os" }, { "text": "Evaluation settings changed after launch. Launch results used a 50-turn, $2 cap. Current leaderboard runs use no cost cap and a 250-turn limit, so launch and current numbers are not comparable.", "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public" }, { "text": "Most current leaderboard entries were run with the mini-swe-agent harness, so scores reflect a fixed scaffold, not the best agent for each model.", "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public" } ], "released": "2025-09-21", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://labs.scale.com/leaderboard/swe_bench_pro_public", "https://arxiv.org/abs/2509.16941", "https://arxiv.org/html/2509.16941v2", "https://github.com/scaleapi/SWE-bench_Pro-os", "https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro", "https://openai.com/index/separating-signal-from-noise-coding-evaluations/" ], "spot_checked": false, "notes": "Frontier comes from the public set. The private (commercial) set leader is the same model, Muse Spark 1.1, at 51.5% ± 5.5. Leaderboard values were read from the JSON embedded in the leaderboard page. OpenAI's July 2026 post says frontier models reached 80.3% on the public split, without naming the model. See data/conflicts/swe-bench-pro-public-frontier.yaml." }, "institution_names": [ "Scale AI" ], "status": "strong", "frontier": { "subject": { "label": "Muse Spark 1.1", "model": "Muse Spark 1.1", "model_org": "Meta", "harness": "mini-swe-agent" }, "value": 61.5, "unit": "percent", "ci": [ 58.4, 64.6 ], "date": "2026-07-09", "frontier": true, "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23", "notes": "Public set (731 tasks, V1). Leaderboard shows 61.50 ± 3.10. The same model leads the private commercial set at 51.5 ± 5.5. Entry added 2026-07-09. The leaderboard lists the company as meta; open-weights status not confirmed." } }, { "id": "swe-bench-verified", "name": "SWE-bench Verified", "short_name": "SWE-bench Verified", "kind": "benchmark", "institutions": [ "princeton", "openai" ], "summary": "A human-validated subset of 500 SWE-bench tasks. Each task is a real GitHub issue from one of 12 Python repositories. The agent gets the repository and the issue text and must produce a patch. OpenAI and the SWE-bench team built it in 2024 after 93 developers screened 1,699 SWE-bench tasks for unfair tests and vague issue descriptions.", "what_is_measured": "Share of the 500 issues resolved: the patch must make the hidden fail-to-pass tests pass and keep the pass-to-pass tests passing. The official leaderboard accepts submissions from any agent system, so scores mix model quality with scaffold quality. A \"Bash Only\" view runs every model in the same mini-SWE-agent environment.", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 500, "human_reference": "Annotators estimated task difficulty by human time. 196 tasks are under 15 minutes, 45 tasks are over 1 hour. No direct human success rate is published.", "metric": { "name": "Resolved", "unit": "percent", "higher_is_better": true, "description": "Percentage of the 500 task instances whose tests pass after the agent's patch is applied." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "status_override": { "status": "retired", "reason": "OpenAI stopped reporting SWE-bench Verified on 2026-02-23. Its audit found that at least 59.4% of 138 hard tasks had tests that reject correct solutions, and that all frontier models it probed could reproduce gold patches or problem text from training data. OpenAI told other developers to stop too. The swebench.com leaderboard still exists but has few entries after early 2026.", "source_url": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/", "date": "2026-02-23" }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "issue-resolution" ], "links": { "homepage": "https://www.swebench.com/", "paper": "https://arxiv.org/abs/2310.06770", "leaderboard": "https://www.swebench.com/", "repo": "https://github.com/SWE-bench/SWE-bench", "blog": "https://openai.com/index/introducing-swe-bench-verified/", "dataset": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified" }, "license": { "data": "Task content comes from 12 open-source Python repositories under their own licenses. The SWE-bench harness is MIT.", "code": "MIT", "reuse": "cite-only" }, "contamination": "High. Tasks and gold patches are public since 2024 and the source repositories are common in training data. OpenAI's red-team probes got GPT-5.2-Chat, Claude Opus 4.5, and Gemini 3 Flash Preview to reproduce gold patches or verbatim task details.", "limits": [ { "text": "OpenAI's audit of 138 tasks that o3 often failed found 59.4% had material problems: 35.5% had tests tied to one implementation, 18.8% tested behavior not in the issue.", "source_url": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/" }, { "text": "Leaderboard entries use different agents, retry budgets, and model mixes. Most top entries are not checked by the SWE-bench team.", "source_url": "https://www.swebench.com/" }, { "text": "Python only, from 12 repositories, so it does not cover most languages or codebases.", "source_url": "https://openai.com/index/introducing-swe-bench-verified/" } ], "released": "2024-08-13", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://www.swebench.com/", "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/", "https://openai.com/index/introducing-swe-bench-verified/", "https://github.com/SWE-bench/SWE-bench", "https://www.anthropic.com/news/claude-opus-4-5" ], "spot_checked": false, "notes": "Leaderboard values were read from the JSON embedded in swebench.com. The top score is a tie at 79.2% between two Claude Opus 4.5 systems. OpenAI's retirement post says the state of the art rose \"from 74.9% to 80.9%\" but cites an aggregator (llm-stats) for that figure, and Anthropic's Claude Opus 4.5 post shows its score only in a chart image, so we do not use 80.9%." }, "institution_names": [ "Princeton University", "OpenAI" ], "status": "retired", "frontier": { "subject": { "label": "Sonar Foundation Agent + Claude 4.5 Opus", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "Sonar Foundation Agent", "agent": "Sonar Foundation Agent", "open_weights": false }, "value": 79.2, "unit": "percent", "date": "2025-12-05", "frontier": true, "source_url": "https://www.swebench.com/", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23", "notes": "Tied at 79.2% with live-SWE-agent + Claude 4.5 Opus (medium), dated 2025-12-15. Neither entry is checked by the SWE-bench team. Single attempt." } }, { "id": "swe-lancer", "name": "SWE-Lancer", "short_name": "SWE-Lancer", "kind": "benchmark", "institutions": [ "openai" ], "authors": [ "Samuel Miserendino", "Michele Wang", "Tejal Patwardhan", "Johannes Heidecke" ], "summary": "Real freelance software jobs from Upwork, all from the Expensify codebase, worth $1 million in actual payouts. Independent contributor (IC) tasks ask the model to fix a bug or build a feature; manager tasks ask it to pick the best of several freelancer proposals. The public split, SWE-Lancer Diamond, is what labs report.", "what_is_measured": "For IC tasks, the model's patch must pass end-to-end browser tests that professional engineers wrote and reviewed three times. For manager tasks, the choice must match the proposal the real hiring manager picked. The paper reports dollars earned; OpenAI's later system cards report pass@1 on the IC SWE Diamond set, which is the metric here. The 2025-07-17 offline revision keeps 198 IC Diamond tasks and removes internet access during runs.", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 1488, "human_reference": "Each task's value is the real payout to the freelancer who did it, from $250 to $32,000 in the full set. IC tasks range from 15-minute bug fixes to multi-week features.", "metric": { "name": "IC SWE Diamond pass@1", "unit": "percent", "higher_is_better": true, "description": "Share of Diamond-set independent contributor tasks whose end-to-end tests pass, one attempt per task." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "issue-resolution", "feature-development", "freelance-projects" ], "links": { "homepage": "https://openai.com/index/swe-lancer/", "paper": "https://arxiv.org/abs/2502.12115", "repo": "https://github.com/openai/preparedness" }, "license": { "data": "Diamond split is public in the openai/preparedness repository (MIT). The remaining tasks are a private holdout.", "code": "MIT", "reuse": "open-mit" }, "contamination": "Tasks come from a public open-source repository (Expensify). OpenAI keeps most tasks private and disables internet during runs.", "limits": [ { "text": "One codebase (the open-source Expensify repository), so results say little about other stacks or domains. OpenAI lists this as a limitation.", "source_url": "https://arxiv.org/abs/2502.12115" }, { "text": "OpenAI is both maintainer and the only lab that reports results. No other model developer adopted the benchmark, and OpenAI system cards from December 2025 on no longer include it.", "source_url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer" }, { "text": "Dataset changed on 2025-07-17: 39 of 237 IC Diamond tasks were dropped and internet access was removed, so results before and after are not comparable.", "source_url": "https://github.com/openai/preparedness" }, { "text": "pass@1 is one sample per task, and OpenAI notes significant variance between runs. Later system cards average three runs.", "source_url": "https://deploymentsafety.openai.com/gpt-5/swe-lancer" } ], "released": "2025-02-17", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://openai.com/index/swe-lancer/", "https://arxiv.org/abs/2502.12115", "https://arxiv.org/html/2502.12115v4", "https://github.com/openai/preparedness", "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer", "https://deploymentsafety.openai.com/gpt-5/swe-lancer" ], "spot_checked": false, "notes": "OpenAI is the maintainer and reports its own models, so results are source_kind primary. The frontier value comes from a chart in the GPT-5.1-Codex-Max system card that labels bars to whole percents (80%); the exact value is not in text. The paper's headline metric is dollars earned ($208,050 of $500,800 on Diamond for Claude 3.5 Sonnet). A community leaderboard at swelancer.github.io was last built 2025-07-28 and lists only o1 and GPT-4o." }, "institution_names": [ "OpenAI" ], "status": "strong", "frontier": { "subject": { "label": "gpt-5.1-codex-max", "model": "GPT-5.1-Codex-Max", "model_org": "OpenAI", "open_weights": false }, "value": 80, "unit": "percent", "date": "2025-11-18", "frontier": true, "source_url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23", "notes": "IC SWE Diamond, pass@1 averaged over 3 runs, on the 2025-07-17 offline dataset. The system card shows the value only in a chart labeled 80%. OpenAI is the maintainer and reports its own model. Later OpenAI system cards do not include SWE-Lancer." } }, { "id": "swe-rebench", "name": "SWE-rebench", "short_name": "SWE-rebench", "kind": "leaderboard", "institutions": [ "nebius" ], "authors": [ "Ibragim Badertdinov", "Alexander Golubev", "Maksim Nekrashevich", "Anton Shevtsov", "Simon Karasik", "Andrei Andriushchenko" ], "summary": "A continuously refreshed leaderboard of real GitHub issues. Nebius collects new issue and pull request pairs from active Python repositories every month, runs every model in the same minimal ReAct agent, and marks results where the issues predate a model's release as possibly contaminated.", "what_is_measured": "Share of fresh issues that a model resolves, averaged over five runs, with the standard error of the mean and pass@5. Every model gets the same scaffold, the same prompts, default sampling settings, and a 128K-token context. The leaderboard shows one time window at a time; the default window on 2026-09-23 covered 111 problems from 65 repositories created between 2026-05-15 and 2026-07-01. Nebius also runs reference agents (Claude Code, Codex, Junie, Cursor) as separate rows.", "grain": "task", "environment": [ "repo", "cli" ], "grading": [ "automated-tests" ], "task_count": 111, "human_reference": null, "metric": { "name": "Resolved rate", "unit": "percent", "higher_is_better": true, "description": "Mean share of problems in the selected time window resolved across five runs, with the standard error of the mean." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" } ], "functions": [ "issue-resolution" ], "links": { "homepage": "https://swe-rebench.com/", "paper": "https://arxiv.org/abs/2505.20411", "leaderboard": "https://swe-rebench.com/", "dataset": "https://huggingface.co/datasets/nebius/SWE-rebench-leaderboard" }, "license": { "data": "Tasks come from repositories under permissive licenses (MIT, Apache-2.0, BSD, ISC, and others that Nebius checked by hand). The leaderboard dataset and Docker images are public.", "reuse": "cite-only" }, "contamination": "Low by design. Tasks are collected after models are released, and the leaderboard flags any evaluation where issues predate the model. Older windows become contaminated over time.", "limits": [ { "text": "One fixed ReAct scaffold for all models. Nebius says model-specific tuning or another scaffold could score higher, so this measures the model in a common harness, not the best agent.", "source_url": "https://swe-rebench.com/about" }, { "text": "Each window is small (111 problems in the current window) and the task mix changes every month, so scores across windows are not directly comparable.", "source_url": "https://swe-rebench.com/" }, { "text": "Python repositories only.", "source_url": "https://arxiv.org/abs/2505.20411" }, { "text": "Top scores in the current window overlap within their standard errors (64.5 ± 1.41 versus 63.8 ± 0.60 and 63.4 ± 1.35), so the ranking at the top is not settled.", "source_url": "https://swe-rebench.com/" } ], "released": "2025-05-26", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://swe-rebench.com/", "https://swe-rebench.com/about", "https://arxiv.org/abs/2505.20411", "https://arxiv.org/html/2505.20411v2", "https://huggingface.co/datasets/nebius/SWE-rebench-leaderboard" ], "spot_checked": false, "notes": "The leaderboard renders server-side, so values were read from the page text. The ± values are standard errors of the mean over five runs, not confidence intervals. Nebius also published a SWE-rebench V2 dataset (language-agnostic, February 2026), but the leaderboard at swe-rebench.com is what we track here." }, "institution_names": [ "Nebius" ], "status": "strong", "frontier": { "subject": { "label": "Anthropic Fable 5 [high]", "model": "Claude Fable 5", "model_org": "Anthropic", "harness": "SWE-rebench ReAct scaffold", "open_weights": false }, "value": 64.5, "unit": "percent", "date": "2026-07-01", "frontier": true, "source_url": "https://swe-rebench.com/", "source_kind": "primary", "reported_by": "nebius", "retrieved_at": "2026-09-23", "notes": "Rank 1 in the default window (2026-05-15 to 2026-07-01, 111 problems from 65 repositories). SEM 1.41, pass@5 78.4, $4.40 per problem. The date is the end of the time window; the leaderboard does not give a run date for each row." } }, { "id": "tau2-bench", "name": "τ²-bench", "short_name": "τ²-bench", "kind": "benchmark", "institutions": [ "sierra" ], "authors": [ "Victor Barres", "Honghua Dong", "Soham Ray", "Xujie Si", "Karthik Narasimhan" ], "summary": "Simulated customer-service conversations from Sierra. An agent chats with an LLM-played user, calls tools, and must follow a written domain policy. The telecom domain adds dual control: the user also has tools, so the agent must guide the user through steps that only the user can perform.", "what_is_measured": "Whether the final database state and required outputs match the expected result for each task. Each task runs several times. pass^k is the chance that all k tries of a task succeed, so higher k rewards reliability. Domains are airline (50 tasks), retail (114), and telecom (114). We use telecom pass^1 as the headline because telecom is the domain that τ²-bench introduced. The repository now ships as τ³-bench with a banking knowledge domain and a voice mode; those are not scored here.", "grain": "task", "environment": [ "api-tools", "chat" ], "grading": [ "state-check" ], "task_count": 278, "human_reference": null, "metric": { "name": "Telecom pass^1", "unit": "percent", "higher_is_better": true, "description": "Share of telecom tasks that the agent completes correctly on a single try, averaged over trials. The leaderboard reports pass^1 to pass^4 per domain." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "customer-support", "soc": [ "43-4051" ], "weight": "primary" } ], "functions": [ "customer-service" ], "links": { "homepage": "https://taubench.com", "paper": "https://arxiv.org/abs/2506.07982", "leaderboard": "https://taubench.com/leaderboard", "repo": "https://github.com/sierra-research/tau2-bench", "blog": "https://sierra.ai/blog/benchmarking-agents-in-collaborative-real-world-scenarios" }, "license": { "data": "MIT (repository license covers tasks, policies, and code).", "code": "MIT", "reuse": "open-mit" }, "contamination": "Tasks, policies, and tools are public in the repository. The leaderboard labels models trained on τ-bench domains or tasks as custom submissions, but it cannot detect training on the public data.", "limits": [ { "text": "The user is an LLM simulator. The choice of user model changes scores, and submissions use different user models (gpt-4.1, gpt-5.2, or the agent's own model). Sierra recommends gpt-5.2.", "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/docs/leaderboard-submission.md" }, { "text": "Telecom pass^1 is close to the ceiling: several models score 97-98%. pass^4 and the newer banking knowledge domain (best 55.2% pass^1) still separate models.", "source_url": "https://github.com/sierra-research/tau2-bench/tree/main/web/leaderboard/public/submissions" }, { "text": "In February 2026 Sierra fixed 50+ airline and retail tasks. Airline pass^1 rose by 14 to 20 points after the fixes, so airline and retail scores from before and after are not comparable.", "source_url": "https://taubench.com/blog/tau3-task-fixes.html" }, { "text": "Some leaderboard entries were submitted by model developers with modified prompts and no trajectories. Sierra marks these as unverified in the submission files.", "source_url": "https://github.com/sierra-research/tau2-bench/tree/main/web/leaderboard/public/submissions" } ], "released": "2025-06-09", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2506.07982", "https://arxiv.org/html/2506.07982v1", "https://github.com/sierra-research/tau2-bench", "https://github.com/sierra-research/tau2-bench/blob/main/CHANGELOG.md", "https://github.com/sierra-research/tau2-bench/blob/main/docs/leaderboard-submission.md", "https://github.com/sierra-research/tau2-bench/tree/main/web/leaderboard/public/submissions", "https://taubench.com", "https://taubench.com/blog/tau3-task-fixes.html" ], "spot_checked": false, "notes": "taubench.com renders with JavaScript, so scores were read from the submission.json files that the leaderboard site loads from the repository. Task counts come from split_tasks.json (base split) and the task-fixes post. Sierra is a customer-service AI company but not a model developer; results that Sierra ran itself are marked primary, and results submitted by model developers are marked lab-reported." }, "institution_names": [ "Sierra" ], "status": "saturated", "frontier": { "subject": { "label": "Qwen3.5-397B-A17B (thinking)", "model": "Qwen3.5-397B-A17B", "model_org": "Alibaba (Qwen)", "harness": "tau2-bench default agent", "open_weights": true }, "value": 97.8, "unit": "percent", "date": "2026-02-27", "frontier": true, "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json", "source_kind": "primary", "reported_by": "sierra", "retrieved_at": "2026-09-23", "notes": "Best telecom pass^1 among runs that Sierra ran and verified with trajectories. User simulator gpt-5.2 (low reasoning), 4 trials, self-hosted with vLLM. pass^2 95.8, pass^3 93.9, pass^4 92.1. Same submission: airline 81.5, retail 84.4, banking knowledge 9.8 pass^1." } }, { "id": "terminal-bench-2", "name": "Terminal-Bench 2.0", "short_name": "TB 2.0", "kind": "benchmark", "institutions": [ "laude-institute", "stanford" ], "authors": [ "Mike Merrill", "Alex Shaw" ], "summary": "89 hand-verified tasks that an agent must finish inside a Docker container from the command line. Tasks include compiling old software, training small models, debugging code, cracking archives, and setting up servers. It replaced Terminal-Bench 1.0 in November 2025 and runs on the Harbor framework.", "what_is_measured": "Share of tasks whose tests pass after the agent finishes, averaged over several trials with a 95% confidence interval. Each leaderboard row is a harness plus a model, so scores depend on both. Some rows are verified by the maintainers; most are self-submitted.", "grain": "task", "environment": [ "cli" ], "grading": [ "automated-tests" ], "task_count": 89, "human_reference": null, "metric": { "name": "Resolution rate", "unit": "percent", "higher_is_better": true, "description": "Percentage of tasks resolved across trials, shown with a 95% confidence interval." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "status_override": { "status": "retired", "reason": "The maintainers replaced 2.0 with Terminal-Bench 2.1 on 2026-05-06 after finding problems in 28 of the 89 tasks (changed external dependencies, resource limits, and instructions that did not match tests). They then moved to 3.0 (2026-07-30) and 4.0 (2026-08-28), saying many tasks had saturated. The 2.0 leaderboard is frozen but still hosted.", "source_url": "https://www.tbench.ai/news/terminal-bench-2-1", "date": "2026-05-06" }, "jobs": [ { "family": "devops-sre-it", "weight": "primary" }, { "family": "software-engineering", "weight": "partial" }, { "family": "ml-research-engineering", "weight": "partial" } ], "functions": [ "terminal-operations", "feature-development" ], "links": { "homepage": "https://www.tbench.ai/", "leaderboard": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "repo": "https://github.com/harbor-framework/terminal-bench-2", "blog": "https://www.tbench.ai/news/announcement-2-0", "dataset": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench-2/latest" }, "license": { "data": "Apache-2.0 (terminal-bench-2 repository).", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": "Tasks are public on GitHub. The maintainers add a canary string and ask that the data never appear in training corpora.", "limits": [ { "text": "28 of 89 tasks had defects that 2.1 fixed: nine depended on external resources that changed, eight had resource budgets too small for valid solutions, and some instructions did not match their tests. Scores on 2.0 understate some systems.", "source_url": "https://www.tbench.ai/news/terminal-bench-2-1" }, { "text": "The top entries overlap within their confidence intervals (84.7 ± 2.1, 84.5 ± 2.6, 83.1 ± 2.1, 82.2 ± 2.2), so the leader is not statistically distinct.", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0" }, { "text": "Most top rows are self-submitted harnesses that the maintainers did not verify. The best maintainer-verified row is Codex CLI with GPT-5.5 at 82.2%.", "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0" }, { "text": "The maintainers say many Terminal-Bench tasks have saturated, which is why they built 3.0.", "source_url": "https://www.tbench.ai/news/terminal-bench-3-0" } ], "released": "2025-11-07", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://www.tbench.ai/", "https://www.tbench.ai/benchmarks", "https://www.tbench.ai/news/announcement-2-0", "https://www.tbench.ai/news/terminal-bench-2-1", "https://www.tbench.ai/news/terminal-bench-3-0", "https://www.tbench.ai/news/terminal-bench-4-0", "https://github.com/harbor-framework/terminal-bench-2", "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench-2/latest" ], "spot_checked": false, "notes": "The 2.0 leaderboard page renders client-side. Values were read from the site's own data endpoint (Supabase function leaderboard-read, package terminal-bench/terminal-bench-2, leaderboard 2-0), which the maintainers describe as \"the official leaderboard for Terminal-Bench 2.0, seeded from tbench.ai\". The site footer says \"Hosted by Stanford / Harbor / Laude Institute\". Row dates appear to be model release dates, not run dates." }, "institution_names": [ "Laude Institute", "Stanford University" ], "status": "retired", "frontier": { "subject": { "label": "NexAU-AHE + GPT-5.5", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "NexAU-AHE", "open_weights": false }, "value": 84.7, "unit": "percent", "ci": [ 82.6, 86.8 ], "date": "2026-04-23", "frontier": true, "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "source_kind": "primary", "reported_by": "laude-institute", "retrieved_at": "2026-09-23", "notes": "Rank 1 on the frozen 2.0 leaderboard (84.7 ± 2.1). Self-submitted by china-qijizhifeng and not verified by the maintainers. Statistically tied with LemonHarness (84.5 ± 2.6), Capy + GPT-5.5 (83.1 ± 2.1), and the verified Codex CLI + GPT-5.5 (82.2 ± 2.2)." } }, { "id": "theagentcompany", "name": "TheAgentCompany", "kind": "benchmark", "institutions": [ "cmu" ], "authors": [ "Frank F. Xu", "Yufan Song", "Boxuan Li", "Yuxuan Tang", "Kritanjali Jain", "Zora Z. Wang", "Xuhui Zhou" ], "summary": "A simulated software company with self-hosted GitLab, a project tracker (Plane), ownCloud file storage, and RocketChat. An agent gets work tasks like a digital employee and must browse, code, run programs, and talk to simulated coworkers to finish them.", "what_is_measured": "Whether an agent completes 175 work tasks in a sealed company environment by browsing the web, writing code, running programs, and messaging coworkers. Checkers inspect the final state of the systems (for example a merged change, a filed document, or a message sent) and award full credit only when every checkpoint passes. Simulated coworkers are language-model agents that the agent must ask for information.", "grain": "task", "environment": [ "simulated-workplace", "browser", "cli" ], "grading": [ "state-check" ], "task_count": 175, "human_reference": null, "metric": { "name": "Tasks resolved", "unit": "percent", "higher_is_better": true, "description": "Share of the 175 tasks fully completed. The leaderboard also shows a partial-credit score that rewards passed checkpoints." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "software-engineering", "weight": "primary" }, { "family": "management-business", "weight": "partial" }, { "family": "office-admin", "weight": "partial" } ], "functions": [ "enterprise-workflows", "feature-development", "long-horizon-autonomy" ], "links": { "homepage": "https://the-agent-company.com", "paper": "https://arxiv.org/abs/2412.14161", "leaderboard": "https://the-agent-company.com/#/leaderboard", "repo": "https://github.com/TheAgentCompany/TheAgentCompany" }, "license": { "data": "MIT license on the GitHub repository. Task images, evaluators, and company data are public.", "code": "MIT", "reuse": "open-mit" }, "contamination": "All tasks and evaluators are public, so models trained after December 2024 may have seen them. The maintainers have not published a contamination analysis.", "limits": [ { "text": "The leaderboard's newest entry is from November 2025, so it does not reflect models released in 2026.", "source_url": "https://github.com/TheAgentCompany/theagentcompany.github.io/blob/main/src/data/leaderboard.json" }, { "text": "Coworkers are simulated by a language model, and results depend on which model plays the environment (the board lists it per entry).", "source_url": "https://github.com/TheAgentCompany/theagentcompany.github.io/blob/main/src/data/leaderboard.json" }, { "text": "The company is a small software firm, so tasks lean toward engineering and internal tooling rather than the full range of office work.", "source_url": "https://github.com/TheAgentCompany/TheAgentCompany" } ], "released": "2024-12-18", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://arxiv.org/abs/2412.14161", "https://github.com/TheAgentCompany/TheAgentCompany", "https://raw.githubusercontent.com/TheAgentCompany/theagentcompany.github.io/main/src/data/leaderboard.json" ], "spot_checked": false, "notes": "The website needs JavaScript, so results come from the leaderboard JSON in the website repository. Author list is from the website's header data (first authors and CMU affiliations). Task count 175 is the number_of_tasks field in that file." }, "institution_names": [ "Carnegie Mellon University" ], "status": "emerging", "frontier": { "subject": { "label": "TTE-MatrixAgent + DeepSeek-V3.2", "model": "DeepSeek-V3.2", "model_org": "DeepSeek", "harness": "TTE-MatrixAgent", "open_weights": true }, "value": 42.86, "unit": "percent", "date": "2025-11-10", "frontier": true, "source_url": "https://raw.githubusercontent.com/TheAgentCompany/theagentcompany.github.io/main/src/data/leaderboard.json", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23", "notes": "Top entry on the leaderboard, marked checked by the maintainers. Partial-credit score 52.4, 29.91 steps and $0.40 per task on average. Environment model Qwen Plus. The agent code is not open source. Newest entry on the board as of 2026-09-23." } }, { "id": "vals-finance-agent", "name": "Finance Agent Benchmark", "short_name": "Finance Agent", "kind": "leaderboard", "institutions": [ "vals-ai" ], "authors": [ "Antoine Bigeard", "Langston Nashold", "Rayan Krishnan", "Shirley Wu" ], "summary": "Vals AI's test of whether an agent can do the research work of an entry-level financial analyst. Each question asks about public companies and their SEC filings, and the agent must find the answer with EDGAR search, web search, a page parser, and a retrieval tool.", "what_is_measured": "Final-answer accuracy on a private test set of 337 expert-written questions (537 in total, with 50 public and 150 licensable validation questions). Questions span nine categories from simple retrieval to financial modeling and market analysis. An LLM judge (GPT-5.2, mode of three runs) compares each answer with the expert answer. Vals also records cost, latency, and tool calls.", "grain": "task", "environment": [ "api-tools", "documents" ], "grading": [ "llm-judge" ], "task_count": 537, "human_reference": "Finance experts from banks, private equity firms, and hedge funds wrote and answered the questions. No human accuracy score is published.", "metric": { "name": "Accuracy", "unit": "percent", "higher_is_better": true, "description": "Share of private test questions where the LLM judge accepts the agent's final answer." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "finance-accounting", "soc": [ "13-2051" ], "weight": "primary" } ], "functions": [ "financial-analysis", "web-research" ], "links": { "homepage": "https://www.vals.ai/benchmarks/finance_agent", "paper": "https://arxiv.org/abs/2508.00828", "leaderboard": "https://www.vals.ai/benchmarks/finance_agent", "repo": "https://github.com/vals-ai/finance-agent", "dataset": "https://github.com/vals-ai/finance-agent/blob/main/data/public.csv" }, "license": { "data": "50 public validation questions in the repository (MIT). 150 validation questions are available for license. The 337-question test set is private.", "code": "MIT", "reuse": "cite-only" }, "contamination": "The test set is private and Vals says it will stay private.", "limits": [ { "text": "Version 1.1 (early 2026) changed the data, tools, prompts, and judge, and re-ran every model. Scores from version 1.0, including the paper's 46.8% for o3, are not comparable.", "source_url": "https://www.vals.ai/benchmarks/finance_agent" }, { "text": "Each score has a standard error of about 2.8 points, so the top five models (60% to 64%) overlap.", "source_url": "https://www.vals.ai/benchmarks/finance_agent" }, { "text": "Grading uses an LLM judge. Vals takes the mode of three GPT-5.2 judgments to reduce variance.", "source_url": "https://www.vals.ai/benchmarks/finance_agent" }, { "text": "The agent only sees the fixed tool set in the Vals harness. Access to the Vals platform to run the private set is gated.", "source_url": "https://github.com/vals-ai/finance-agent" } ], "released": "2025-05-20", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://www.vals.ai/benchmarks/finance_agent", "https://arxiv.org/abs/2508.00828", "https://github.com/vals-ai/finance-agent" ], "spot_checked": false, "notes": "The leaderboard table renders with JavaScript. Scores were read from the data embedded in the page HTML (accuracy and stderr per model, benchmark \"updated\" 2026-06-04, 51 models). Vals AI is an independent evaluator and not a model developer." }, "institution_names": [ "Vals AI" ], "status": "strong", "frontier": { "subject": { "label": "Claude Opus 4.7", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "Vals finance-agent v1.1", "open_weights": false }, "value": 64.37, "unit": "percent", "date": "2026-06-04", "frontier": true, "source_url": "https://www.vals.ai/benchmarks/finance_agent", "source_kind": "primary", "reported_by": "vals-ai", "retrieved_at": "2026-09-23", "notes": "Top of the v1.1 leaderboard (page data 64.373, stderr 2.79, cost about $0.80 per question). Muse Spark 60.59, DeepSeek V4 Pro 60.39, and Claude Opus 4.6 (Thinking) 60.05 follow. Date is the page's \"updated\" field, not the run date." } }, { "id": "vending-bench-2", "name": "Vending-Bench 2", "short_name": "Vending-Bench 2", "kind": "benchmark", "institutions": [ "andon-labs" ], "summary": "An agent runs a simulated vending machine business for one simulated year, starting with $500. It finds suppliers on a simulated web, negotiates by email, orders and stocks products, sets prices, and handles delays, bankrupt suppliers, refund demands, and suppliers that try to cheat it.", "what_is_measured": "The bank balance at the end of the year, averaged over several runs. The agent pays a $2 daily fee and is terminated early if it cannot pay for more than 10 days in a row. Sales depend on day of week, season, weather, and price. A full run produces 3,000 to 6,000 messages and 60 to 100 million output tokens, so the score mostly reflects whether the agent stays coherent and keeps negotiating well over a long horizon.", "grain": "project", "environment": [ "simulated-business" ], "grading": [ "outcome-metric" ], "task_count": null, "human_reference": "No human baseline was run. Andon Labs estimates that a \"good\" strategy (best-selling product, half-price supply, optimal stocking) would make about $206 per day, or roughly $63k in a year.", "metric": { "name": "Final bank balance", "unit": "dollars", "higher_is_better": true, "description": "Money balance after one simulated year, averaged across runs. There is no fixed ceiling." }, "reference_rule": { "kind": "none", "note": "No ceiling by design. Andon Labs' own rough estimate of a good human-level strategy is about $63k, roughly four times the best model score in September 2026." }, "jobs": [ { "family": "management-business", "soc": [ "11-1021", "13-1023" ], "weight": "primary" } ], "functions": [ "business-operations", "long-horizon-autonomy" ], "links": { "homepage": "https://andonlabs.com/evals/vending-bench-2", "paper": "https://arxiv.org/abs/2502.15840", "leaderboard": "https://andonlabs.com/evals/vending-bench-2", "blog": "https://andonlabs.com/blog/gpt-6-astra-vending-bench" }, "license": { "data": "No public code or data release is linked from the benchmark page. Andon Labs runs the simulation.", "reuse": "cite-only" }, "contamination": "The environment is not public, but the system prompt is shown on the benchmark page and the sales simulation follows the equations in the original Vending-Bench paper.", "limits": [ { "text": "Run-to-run variance is large. Leaderboard entries carry error bars of roughly $370 to $2,100 on averages of five runs.", "source_url": "https://andonlabs.com/evals/vending-bench-2" }, { "text": "Suppliers are other LLMs. The page notes they could in theory be jailbroken into giving stock away, and the sales equations can be gamed.", "source_url": "https://andonlabs.com/evals/vending-bench-2" }, { "text": "Each run uses 60 to 100 million output tokens, so results are costly to reproduce and only Andon Labs reports them.", "source_url": "https://andonlabs.com/evals/vending-bench-2" }, { "text": "The score ignores conduct. Andon Labs' blog posts describe top models that lied to customers, fixed prices with competitors, or withheld refunds while still scoring well.", "source_url": "https://andonlabs.com/blog/opus-5-vending-bench" } ], "released": "2025-11-18", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://andonlabs.com/evals/vending-bench-2", "http://web.archive.org/web/20251118175816/https://andonlabs.com/evals/vending-bench-2", "https://andonlabs.com/blog", "https://andonlabs.com/blog/gpt-6-astra-vending-bench", "https://andonlabs.com/blog/opus-5-vending-bench" ], "spot_checked": true, "notes": "The paper link is the original Vending-Bench paper, which the page cites for the sales simulation; there is no Vending-Bench 2 paper. Release date comes from the first Wayback Machine capture of the page. Andon Labs is not a model developer." }, "institution_names": [ "Andon Labs" ], "status": "unrated", "frontier": { "subject": { "label": "GPT-6 Astra", "model": "GPT-6 Astra", "model_org": "OpenAI", "open_weights": false }, "value": 15514.7, "unit": "dollars", "date": "2026-09-07", "frontier": true, "source_url": "https://andonlabs.com/evals/vending-bench-2", "source_kind": "primary", "reported_by": "andon-labs", "retrieved_at": "2026-09-23", "notes": "Leaderboard value (± $1,074, \"average across 5 runs\"). The 2026-09-07 blog post gives $15,515 as the average of six runs; the blog also says every Astra run beat every Claude Fable 5.1 run (Fable 5.1 average $5,422)." } }, { "id": "workarena", "name": "WorkArena", "kind": "benchmark", "institutions": [ "servicenow" ], "authors": [ "Alexandre Drouin", "Maxime Gasse", "Massimo Caccia", "Issam H. Laradji", "Manuel Del Verme", "Tom Marty", "Léo Boisvert", "Megh Thakkar", "Quentin Cappart", "David Vazquez", "Nicolas Chapados", "Alexandre Lacoste" ], "summary": "Browser tasks on a live ServiceNow enterprise instance, the software many companies use for IT and HR requests. WorkArena-L1 has 33 task types (19,912 instances) that cover the main parts of the ServiceNow interface: forms, lists, knowledge base search, service catalog orders, and dashboards.", "what_is_measured": "Whether a web agent completes routine enterprise-software actions from a text instruction. Each task samples a fresh configuration, and a checker validates the result in the ServiceNow instance. WorkArena++ (L2 and L3, 682 tasks) composes these atomic actions into longer workflows that need planning and reasoning, but the public leaderboard currently lists results only for L1.", "grain": "task", "environment": [ "browser", "live-system" ], "grading": [ "state-check" ], "task_count": 33, "human_reference": null, "metric": { "name": "WorkArena-L1 success rate", "unit": "percent", "higher_is_better": true, "description": "Share of sampled WorkArena-L1 task instances completed, as reported on the BrowserGym leaderboard with standard error." }, "reference_rule": { "kind": "ceiling", "value": 100 }, "jobs": [ { "family": "office-admin", "weight": "primary" }, { "family": "devops-sre-it", "weight": "partial" } ], "functions": [ "enterprise-workflows", "computer-use" ], "links": { "homepage": "https://github.com/ServiceNow/WorkArena", "paper": "https://arxiv.org/abs/2403.07718", "leaderboard": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "repo": "https://github.com/ServiceNow/WorkArena" }, "license": { "data": "Apache 2.0 for the benchmark code. Running tasks needs access to a ServiceNow developer instance.", "code": "Apache-2.0", "reuse": "open-apache" }, "contamination": "Task templates are public, but each run samples new instances. The maintainers have not published a contamination analysis.", "limits": [ { "text": "Leaderboard entries are self-submitted result files. The top entry raised the step budget from 15 to 30 and is marked as not following the standard evaluation protocol.", "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/IpaziaHPA-Gemini-3-flash-preview/workarena-l1.json" }, { "text": "L1 tasks are short atomic actions. The harder WorkArena++ (L2, L3) tasks have no entries on the public leaderboard, so the board understates how far agents are from full workflows.", "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard" }, { "text": "Few agents have been submitted (about 20 folders), and several have no WorkArena result at all.", "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard" }, { "text": "All tasks run on one vendor's platform, so results say little about other enterprise software.", "source_url": "https://arxiv.org/abs/2403.07718" } ], "released": "2024-03-12", "last_checked": "2026-09-23", "verification": { "researched_by": "atlas-researcher, 2026-09-23", "primary_sources_checked": [ "https://github.com/ServiceNow/WorkArena", "https://arxiv.org/abs/2403.07718", "https://arxiv.org/abs/2407.05291", "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard", "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/IpaziaHPA-Gemini-3-flash-preview/workarena-l1.json", "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5/workarena-l1.json" ], "spot_checked": false, "notes": "We chose WorkArena-L1 because it is the only version with public leaderboard results. The WorkArena++ paper reports a human study, but the abstract gives no number and we did not open the full paper, so human_reference is left empty. Results were read from the JSON files in the leaderboard Space, not the rendered app." }, "institution_names": [ "ServiceNow" ], "status": "saturated", "frontier": { "subject": { "label": "IpaziaHPA-Gemini-3-flash-preview", "model": "Gemini 3 Flash Preview", "model_org": "Google", "harness": "IpaziaHPA", "agent": "IpaziaHPA (Ipazia S.p.a.)", "open_weights": false }, "value": 90.3, "unit": "percent", "date": "2026-03-07", "frontier": true, "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/IpaziaHPA-Gemini-3-flash-preview/workarena-l1.json", "source_kind": "third-party", "reported_by": "Ipazia S.p.a. (submitted to the ServiceNow BrowserGym leaderboard)", "retrieved_at": "2026-09-23", "notes": "Top WorkArena-L1 entry on the leaderboard as of 2026-09-23. Standard error 1.6. Uses the accessibility tree, no screenshots. Max steps raised from 15 to 30, so the file marks it as not following the standard protocol. Self-submitted; not re-run by ServiceNow." } } ], "results": [ { "benchmark_id": "aider-polyglot", "benchmark_name": "Aider Polyglot leaderboard", "institutions": "Aider", "metric": "Percent correct", "subject": "o1-2024-12-17 (high)", "model": "o1", "model_org": "OpenAI", "harness": "aider", "agent": "", "value": 61.7, "unit": "percent", "date": "2024-12-21", "frontier": false, "source_url": "https://aider.chat/2024/12/21/polyglot.html", "source_kind": "primary", "reported_by": "aider", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aider-polyglot", "benchmark_name": "Aider Polyglot leaderboard", "institutions": "Aider", "metric": "Percent correct", "subject": "Gemini 2.5 Pro Preview 05-06", "model": "Gemini 2.5 Pro Preview", "model_org": "Google", "harness": "aider", "agent": "", "value": 76.9, "unit": "percent", "date": "2025-05-07", "frontier": false, "source_url": "https://aider.chat/docs/leaderboards/", "source_kind": "primary", "reported_by": "aider", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aider-polyglot", "benchmark_name": "Aider Polyglot leaderboard", "institutions": "Aider", "metric": "Percent correct", "subject": "o3-pro (high)", "model": "o3-pro", "model_org": "OpenAI", "harness": "aider", "agent": "", "value": 84.9, "unit": "percent", "date": "2025-06-28", "frontier": false, "source_url": "https://aider.chat/docs/leaderboards/", "source_kind": "primary", "reported_by": "aider", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aider-polyglot", "benchmark_name": "Aider Polyglot leaderboard", "institutions": "Aider", "metric": "Percent correct", "subject": "gpt-5 (high)", "model": "GPT-5", "model_org": "OpenAI", "harness": "aider", "agent": "", "value": 88, "unit": "percent", "date": "2025-08-23", "frontier": true, "source_url": "https://aider.chat/docs/leaderboards/", "source_kind": "primary", "reported_by": "aider", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aidev-dataset", "benchmark_name": "AIDev", "institutions": "Queen's University", "metric": "PR acceptance rate (AIDev-pop)", "subject": "OpenAI Codex", "model": "", "model_org": "OpenAI", "harness": "", "agent": "OpenAI Codex", "value": 65.3, "unit": "percent", "date": "2025-07-20", "frontier": true, "source_url": "https://arxiv.org/html/2507.15003v1", "source_kind": "primary", "reported_by": "queens-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aidev-dataset", "benchmark_name": "AIDev", "institutions": "Queen's University", "metric": "PR acceptance rate (AIDev-pop)", "subject": "Claude Code", "model": "", "model_org": "Anthropic", "harness": "", "agent": "Claude Code", "value": 52.5, "unit": "percent", "date": "2025-07-20", "frontier": false, "source_url": "https://arxiv.org/html/2507.15003v1", "source_kind": "primary", "reported_by": "queens-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aidev-dataset", "benchmark_name": "AIDev", "institutions": "Queen's University", "metric": "PR acceptance rate (AIDev-pop)", "subject": "Cursor", "model": "", "model_org": "Cursor", "harness": "", "agent": "Cursor", "value": 51.4, "unit": "percent", "date": "2025-07-20", "frontier": false, "source_url": "https://arxiv.org/html/2507.15003v1", "source_kind": "primary", "reported_by": "queens-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aidev-dataset", "benchmark_name": "AIDev", "institutions": "Queen's University", "metric": "PR acceptance rate (AIDev-pop)", "subject": "Devin", "model": "", "model_org": "Cognition", "harness": "", "agent": "Devin", "value": 48.9, "unit": "percent", "date": "2025-07-20", "frontier": false, "source_url": "https://arxiv.org/html/2507.15003v1", "source_kind": "primary", "reported_by": "queens-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aidev-dataset", "benchmark_name": "AIDev", "institutions": "Queen's University", "metric": "PR acceptance rate (AIDev-pop)", "subject": "GitHub Copilot", "model": "", "model_org": "GitHub", "harness": "", "agent": "GitHub Copilot coding agent", "value": 38.2, "unit": "percent", "date": "2025-07-20", "frontier": false, "source_url": "https://arxiv.org/html/2507.15003v1", "source_kind": "primary", "reported_by": "queens-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aiopslab", "benchmark_name": "AIOpsLab", "institutions": "Microsoft Research; UIUC; UC Berkeley; IISc", "metric": "Accuracy", "subject": "GPT-3.5-w-Shell", "model": "GPT-3.5-turbo", "model_org": "OpenAI", "harness": "shell-only baseline", "agent": "", "value": 15.25, "unit": "percent", "date": "2025-01-12", "frontier": false, "source_url": "https://arxiv.org/abs/2501.06706", "source_kind": "primary", "reported_by": "microsoft", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aiopslab", "benchmark_name": "AIOpsLab", "institutions": "Microsoft Research; UIUC; UC Berkeley; IISc", "metric": "Accuracy", "subject": "GPT-4-w-Shell", "model": "GPT-4-turbo", "model_org": "OpenAI", "harness": "shell-only baseline", "agent": "", "value": 49.15, "unit": "percent", "date": "2025-01-12", "frontier": false, "source_url": "https://arxiv.org/abs/2501.06706", "source_kind": "primary", "reported_by": "microsoft", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aiopslab", "benchmark_name": "AIOpsLab", "institutions": "Microsoft Research; UIUC; UC Berkeley; IISc", "metric": "Accuracy", "subject": "ReAct", "model": "", "model_org": "", "harness": "", "agent": "ReAct", "value": 55.93, "unit": "percent", "date": "2025-01-12", "frontier": false, "source_url": "https://arxiv.org/abs/2501.06706", "source_kind": "primary", "reported_by": "microsoft", "retrieved_at": "2026-09-23" }, { "benchmark_id": "aiopslab", "benchmark_name": "AIOpsLab", "institutions": "Microsoft Research; UIUC; UC Berkeley; IISc", "metric": "Accuracy", "subject": "Flash", "model": "", "model_org": "", "harness": "", "agent": "Flash", "value": 59.32, "unit": "percent", "date": "2025-01-12", "frontier": true, "source_url": "https://arxiv.org/abs/2501.06706", "source_kind": "primary", "reported_by": "microsoft", "retrieved_at": "2026-09-23" }, { "benchmark_id": "ambig-swe", "benchmark_name": "Ambig-SWE", "institutions": "Carnegie Mellon University", "metric": "Underspecificity detection accuracy (best prompt)", "subject": "Claude Sonnet 3.5", "model": "Claude Sonnet 3.5", "model_org": "Anthropic", "harness": "OpenHands", "agent": "", "value": 84, "unit": "percent", "date": "2026-02-21", "frontier": false, "source_url": "https://arxiv.org/html/2502.13069v3", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "ambig-swe", "benchmark_name": "Ambig-SWE", "institutions": "Carnegie Mellon University", "metric": "Underspecificity detection accuracy (best prompt)", "subject": "Claude Sonnet 4", "model": "Claude Sonnet 4", "model_org": "Anthropic", "harness": "OpenHands", "agent": "", "value": 89, "unit": "percent", "date": "2026-02-21", "frontier": true, "source_url": "https://arxiv.org/html/2502.13069v3", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "ambig-swe", "benchmark_name": "Ambig-SWE", "institutions": "Carnegie Mellon University", "metric": "Underspecificity detection accuracy (best prompt)", "subject": "Qwen 3 Coder 480B", "model": "Qwen3-Coder-480B", "model_org": "Alibaba", "harness": "OpenHands", "agent": "", "value": 50, "unit": "percent", "date": "2026-02-21", "frontier": false, "source_url": "https://arxiv.org/html/2502.13069v3", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "apex-agents", "benchmark_name": "APEX-Agents", "institutions": "Mercor; Box; Harvey", "metric": "Pass@1", "subject": "Gemini 3 Flash (Thinking=High)", "model": "Gemini 3 Flash", "model_org": "Google", "harness": "", "agent": "", "value": 24, "unit": "percent", "date": "2026-01", "frontier": false, "source_url": "https://arxiv.org/abs/2601.14242", "source_kind": "primary", "reported_by": "mercor", "retrieved_at": "2026-09-23" }, { "benchmark_id": "apex-agents", "benchmark_name": "APEX-Agents", "institutions": "Mercor; Box; Harvey", "metric": "Pass@1", "subject": "GPT-5.5 (xhigh)", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "Mercor Loop agent", "agent": "", "value": 55.1, "unit": "percent", "date": "2026-09-23", "frontier": false, "source_url": "https://www.mercor.com/apex/apex-agents-leaderboard/", "source_kind": "primary", "reported_by": "mercor", "retrieved_at": "2026-09-23" }, { "benchmark_id": "apex-agents", "benchmark_name": "APEX-Agents", "institutions": "Mercor; Box; Harvey", "metric": "Pass@1", "subject": "Claude Fable 5.1 (max)", "model": "Claude Fable 5.1", "model_org": "Anthropic", "harness": "Mercor Loop agent", "agent": "", "value": 68.6, "unit": "percent", "date": "2026-09-08", "frontier": false, "source_url": "https://www.mercor.com/blog/introducing-apex-agents-1-1", "source_kind": "primary", "reported_by": "mercor", "retrieved_at": "2026-09-23" }, { "benchmark_id": "apex-agents", "benchmark_name": "APEX-Agents", "institutions": "Mercor; Box; Harvey", "metric": "Pass@1", "subject": "Claude Opus 5.5 (max)", "model": "Claude Opus 5.5", "model_org": "Anthropic", "harness": "Mercor Loop agent", "agent": "", "value": 73.5, "unit": "percent", "date": "2026-09-23", "frontier": true, "source_url": "https://www.mercor.com/apex/apex-agents-leaderboard/", "source_kind": "primary", "reported_by": "mercor", "retrieved_at": "2026-09-23" }, { "benchmark_id": "browsecomp", "benchmark_name": "BrowseComp", "institutions": "OpenAI", "metric": "Accuracy", "subject": "Deep research", "model": "", "model_org": "OpenAI", "harness": "", "agent": "OpenAI deep research", "value": 51.5, "unit": "percent", "date": "2025-04", "frontier": false, "source_url": "https://openai.com/index/browsecomp/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "browsecomp", "benchmark_name": "BrowseComp", "institutions": "OpenAI", "metric": "Accuracy", "subject": "GPT-5.5 Pro", "model": "GPT-5.5 Pro", "model_org": "OpenAI", "harness": "", "agent": "", "value": 90.1, "unit": "percent", "date": "2026-04", "frontier": false, "source_url": "https://openai.com/index/introducing-gpt-5-5/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "browsecomp", "benchmark_name": "BrowseComp", "institutions": "OpenAI", "metric": "Accuracy", "subject": "GPT-6 Astra", "model": "GPT-6 Astra", "model_org": "OpenAI", "harness": "", "agent": "", "value": 91.5, "unit": "percent", "date": "2026-09", "frontier": false, "source_url": "https://openai.com/index/gpt-6-astra/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "browsecomp", "benchmark_name": "BrowseComp", "institutions": "OpenAI", "metric": "Accuracy", "subject": "GPT-5.6 Sol Ultra", "model": "GPT-5.6 Sol", "model_org": "OpenAI", "harness": "ultra (four parallel agents)", "agent": "", "value": 92.2, "unit": "percent", "date": "2026-07", "frontier": true, "source_url": "https://openai.com/index/gpt-5-6/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "codeclash", "benchmark_name": "CodeClash", "institutions": "Stanford University; Princeton University", "metric": "Elo (all arenas)", "subject": "o3", "model": "o3", "model_org": "OpenAI", "harness": "mini-SWE-agent", "agent": "", "value": 1343, "unit": "elo", "date": "2025-11-03", "frontier": false, "source_url": "https://codeclash.ai/", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "codeclash", "benchmark_name": "CodeClash", "institutions": "Stanford University; Princeton University", "metric": "Elo (all arenas)", "subject": "GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "mini-SWE-agent", "agent": "", "value": 1366, "unit": "elo", "date": "2025-11-03", "frontier": false, "source_url": "https://codeclash.ai/", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "codeclash", "benchmark_name": "CodeClash", "institutions": "Stanford University; Princeton University", "metric": "Elo (all arenas)", "subject": "Claude Sonnet 4.5", "model": "Claude Sonnet 4.5", "model_org": "Anthropic", "harness": "mini-SWE-agent", "agent": "", "value": 1385, "unit": "elo", "date": "2025-11-03", "frontier": true, "source_url": "https://codeclash.ai/", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "core-bench", "benchmark_name": "CORE-Bench", "institutions": "Princeton University", "metric": "CORE-Bench-Hard accuracy (public test set)", "subject": "CORE-Agent + GPT-4o", "model": "GPT-4o", "model_org": "OpenAI", "harness": "CORE-Agent", "agent": "", "value": 21.48, "unit": "percent", "date": "2024-09-17", "frontier": false, "source_url": "https://arxiv.org/abs/2409.11363", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "core-bench", "benchmark_name": "CORE-Bench", "institutions": "Princeton University", "metric": "CORE-Bench-Hard accuracy (public test set)", "subject": "CORE-Agent + Claude Opus 4.1", "model": "Claude Opus 4.1", "model_org": "Anthropic", "harness": "CORE-Agent", "agent": "", "value": 51.11, "unit": "percent", "date": "2025", "frontier": false, "source_url": "https://hal.cs.princeton.edu/corebench_hard", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "core-bench", "benchmark_name": "CORE-Bench", "institutions": "Princeton University", "metric": "CORE-Bench-Hard accuracy (public test set)", "subject": "Claude Code + Claude Sonnet 4.5", "model": "Claude Sonnet 4.5", "model_org": "Anthropic", "harness": "Claude Code", "agent": "", "value": 62.22, "unit": "percent", "date": "2025", "frontier": false, "source_url": "https://hal.cs.princeton.edu/corebench_hard", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "core-bench", "benchmark_name": "CORE-Bench", "institutions": "Princeton University", "metric": "CORE-Bench-Hard accuracy (public test set)", "subject": "Claude Code + Claude Opus 4.5 (automated grading)", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "Claude Code", "agent": "", "value": 77.78, "unit": "percent", "date": "2025-12-03", "frontier": false, "source_url": "https://hal.cs.princeton.edu/corebench_hard", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "core-bench", "benchmark_name": "CORE-Bench", "institutions": "Princeton University", "metric": "CORE-Bench-Hard accuracy (public test set)", "subject": "Claude Code + Claude Opus 4.5 (after manual regrading)", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "Claude Code", "agent": "", "value": 95.5, "unit": "percent", "date": "2025-12-03", "frontier": true, "source_url": "https://hal.cs.princeton.edu/corebench_hard", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "crmarena-pro", "benchmark_name": "CRMArena-Pro", "institutions": "Salesforce AI Research", "metric": "Single-turn task success (B2C org)", "subject": "gpt-4o (ReAct)", "model": "gpt-4o", "model_org": "OpenAI", "harness": "ReAct", "agent": "", "value": 29.2, "unit": "percent", "date": "2025-05-24", "frontier": false, "source_url": "https://arxiv.org/html/2505.18878v1", "source_kind": "primary", "reported_by": "salesforce", "retrieved_at": "2026-09-23" }, { "benchmark_id": "crmarena-pro", "benchmark_name": "CRMArena-Pro", "institutions": "Salesforce AI Research", "metric": "Single-turn task success (B2C org)", "subject": "o1 (ReAct)", "model": "o1", "model_org": "OpenAI", "harness": "ReAct", "agent": "", "value": 49.5, "unit": "percent", "date": "2025-05-24", "frontier": false, "source_url": "https://arxiv.org/html/2505.18878v1", "source_kind": "primary", "reported_by": "salesforce", "retrieved_at": "2026-09-23" }, { "benchmark_id": "crmarena-pro", "benchmark_name": "CRMArena-Pro", "institutions": "Salesforce AI Research", "metric": "Single-turn task success (B2C org)", "subject": "gemini-2.5-pro (ReAct)", "model": "gemini-2.5-pro", "model_org": "Google", "harness": "ReAct", "agent": "", "value": 58.3, "unit": "percent", "date": "2025-05-24", "frontier": true, "source_url": "https://arxiv.org/html/2505.18878v1", "source_kind": "primary", "reported_by": "salesforce", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cve-bench", "benchmark_name": "CVE-Bench", "institutions": "UIUC", "metric": "Success rate (one-day, 5 attempts)", "subject": "Cybench agent (Cy-Agent) with GPT-4o", "model": "gpt-4o-2024-11-20", "model_org": "OpenAI", "harness": "", "agent": "Cy-Agent", "value": 2.5, "unit": "percent", "date": "2025-03", "frontier": false, "source_url": "https://arxiv.org/abs/2503.17332", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cve-bench", "benchmark_name": "CVE-Bench", "institutions": "UIUC", "metric": "Success rate (one-day, 5 attempts)", "subject": "AutoGPT with GPT-4o", "model": "gpt-4o-2024-11-20", "model_org": "OpenAI", "harness": "", "agent": "AutoGPT", "value": 12.5, "unit": "percent", "date": "2025-03", "frontier": true, "source_url": "https://arxiv.org/abs/2503.17332", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybench", "benchmark_name": "Cybench", "institutions": "Stanford University", "metric": "Unguided % solved", "subject": "Claude 3.5 Sonnet", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "harness": "Cybench agent (structured bash)", "agent": "", "value": 17.5, "unit": "percent", "date": "2024-08-15", "frontier": false, "source_url": "https://cybench.github.io/data/leaderboard.csv", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybench", "benchmark_name": "Cybench", "institutions": "Stanford University", "metric": "Unguided % solved", "subject": "Claude Opus 4.5", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "", "agent": "", "value": 82, "unit": "percent", "date": "2025-11", "frontier": false, "source_url": "https://assets.anthropic.com/m/64823ba7485345a7/Claude-Opus-4-5-System-Card.pdf", "source_kind": "lab-reported", "reported_by": "anthropic", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybench", "benchmark_name": "Cybench", "institutions": "Stanford University", "metric": "Unguided % solved", "subject": "Claude Opus 4.7", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "", "agent": "", "value": 96, "unit": "percent", "date": "2026-04-16", "frontier": false, "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/037f06850df7fbe871e206dad004c3db5fd50340.pdf", "source_kind": "lab-reported", "reported_by": "anthropic", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybench", "benchmark_name": "Cybench", "institutions": "Stanford University", "metric": "Unguided % solved", "subject": "Claude Mythos Preview", "model": "Claude Mythos Preview", "model_org": "Anthropic", "harness": "", "agent": "", "value": 100, "unit": "percent", "date": "2026-04-07", "frontier": true, "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf", "source_kind": "lab-reported", "reported_by": "anthropic", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybergym", "benchmark_name": "CyberGym", "institutions": "UC Berkeley", "metric": "Success rate (Level 1)", "subject": "OpenHands (Claude Sonnet 3.7)", "model": "Claude 3.7 Sonnet", "model_org": "Anthropic", "harness": "", "agent": "OpenHands", "value": 11.9, "unit": "percent", "date": "2025-05-15", "frontier": false, "source_url": "https://www.cybergym.io/cybergym/", "source_kind": "primary", "reported_by": "uc-berkeley", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybergym", "benchmark_name": "CyberGym", "institutions": "UC Berkeley", "metric": "Success rate (Level 1)", "subject": "OpenHands (GPT-5)", "model": "GPT-5", "model_org": "OpenAI", "harness": "", "agent": "OpenHands", "value": 39.4, "unit": "percent", "date": "2025-12-05", "frontier": false, "source_url": "https://www.cybergym.io/cybergym/", "source_kind": "primary", "reported_by": "uc-berkeley", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybergym", "benchmark_name": "CyberGym", "institutions": "UC Berkeley", "metric": "Success rate (Level 1)", "subject": "Claude Mythos Preview", "model": "Claude Mythos Preview", "model_org": "Anthropic", "harness": "", "agent": "", "value": 83.1, "unit": "percent", "date": "2026-04-07", "frontier": false, "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf", "source_kind": "lab-reported", "reported_by": "anthropic", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybergym", "benchmark_name": "CyberGym", "institutions": "UC Berkeley", "metric": "Success rate (Level 1)", "subject": "MDASH (multi-model)", "model": "GPT-5.4, Claude Opus 4.6, Claude Sonnet 4.6", "model_org": "Microsoft", "harness": "", "agent": "MDASH", "value": 90.97, "unit": "percent", "date": "2026-06-17", "frontier": false, "source_url": "https://www.cybergym.io/cybergym/", "source_kind": "lab-reported", "reported_by": "microsoft", "retrieved_at": "2026-09-23" }, { "benchmark_id": "cybergym", "benchmark_name": "CyberGym", "institutions": "UC Berkeley", "metric": "Success rate (Level 1)", "subject": "Creation (天工), multi-model", "model": "Creation Model, DeepSeek-V4-Pro, Qwen 3.8 Max", "model_org": "Creation Lab", "harness": "", "agent": "Creation", "value": 98.47, "unit": "percent", "date": "2026-09-07", "frontier": true, "source_url": "https://www.cybergym.io/cybergym/", "source_kind": "third-party", "reported_by": "uc-berkeley", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gaia", "benchmark_name": "GAIA", "institutions": "Meta; Hugging Face", "metric": "Test-set accuracy", "subject": "GPT-4 with plugins", "model": "GPT-4", "model_org": "OpenAI", "harness": "", "agent": "", "value": 15, "unit": "percent", "date": "2023-11", "frontier": false, "source_url": "https://arxiv.org/abs/2311.12983", "source_kind": "primary", "reported_by": "meta", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gaia", "benchmark_name": "GAIA", "institutions": "Meta; Hugging Face", "metric": "Test-set accuracy", "subject": "h2oGPTe Agent v1.6.8", "model": "", "model_org": "h2o.ai", "harness": "", "agent": "h2oGPTe Agent", "value": 65.12, "unit": "percent", "date": "2024-12-21", "frontier": false, "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public", "source_kind": "third-party", "reported_by": "h2o.ai (submitted to the GAIA leaderboard)", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gaia", "benchmark_name": "GAIA", "institutions": "Meta; Hugging Face", "metric": "Test-set accuracy", "subject": "Co-Sight_v2.1.0", "model": "", "model_org": "ZTE-AICloud", "harness": "", "agent": "Co-Sight", "value": 87.04, "unit": "percent", "date": "2025-10-13", "frontier": false, "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public", "source_kind": "third-party", "reported_by": "ZTE-AICloud (submitted to the GAIA leaderboard)", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gaia", "benchmark_name": "GAIA", "institutions": "Meta; Hugging Face", "metric": "Test-set accuracy", "subject": "OPS-Agentic-Search", "model": "", "model_org": "Alibaba Cloud", "harness": "", "agent": "OPS-Agentic-Search", "value": 92.36, "unit": "percent", "date": "2026-03-11", "frontier": false, "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public", "source_kind": "third-party", "reported_by": "Alibaba Cloud (submitted to the GAIA leaderboard)", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gaia", "benchmark_name": "GAIA", "institutions": "Meta; Hugging Face", "metric": "Test-set accuracy", "subject": "Ops-Agentic-Search-2.0", "model": "", "model_org": "", "harness": "", "agent": "Ops-Agentic-Search-2.0", "value": 93.69, "unit": "percent", "date": "2026-09-22", "frontier": true, "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public", "source_kind": "third-party", "reported_by": "Unnamed submitter (GAIA leaderboard entry links to github.com/tosky001/OpenSearch-Agentic-Search)", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval-aa", "benchmark_name": "GDPval-AA", "institutions": "Artificial Analysis", "metric": "GDPval-AA Elo", "subject": "GPT-5 (high)", "model": "GPT-5", "model_org": "OpenAI", "harness": "Stirrup", "agent": "", "value": 906, "unit": "elo", "date": "2026-09-23", "frontier": false, "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval-aa", "benchmark_name": "GDPval-AA", "institutions": "Artificial Analysis", "metric": "GDPval-AA Elo", "subject": "Claude Opus 4.7 (Adaptive Reasoning, Max Effort)", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "Stirrup", "agent": "", "value": 1338, "unit": "elo", "date": "2026-09-23", "frontier": false, "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval-aa", "benchmark_name": "GDPval-AA", "institutions": "Artificial Analysis", "metric": "GDPval-AA Elo", "subject": "Claude Opus 5 (Adaptive Reasoning, Max Effort)", "model": "Claude Opus 5", "model_org": "Anthropic", "harness": "Stirrup", "agent": "", "value": 1708, "unit": "elo", "date": "2026-09-23", "frontier": false, "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval-aa", "benchmark_name": "GDPval-AA", "institutions": "Artificial Analysis", "metric": "GDPval-AA Elo", "subject": "GPT-5.6 Sol (max)", "model": "GPT-5.6 Sol", "model_org": "OpenAI", "harness": "Stirrup", "agent": "", "value": 1588, "unit": "elo", "date": "2026-09-23", "frontier": false, "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval-aa", "benchmark_name": "GDPval-AA", "institutions": "Artificial Analysis", "metric": "GDPval-AA Elo", "subject": "Claude Opus 5.5 (Adaptive Reasoning, Max Effort, Default Fallback)", "model": "Claude Opus 5.5", "model_org": "Anthropic", "harness": "Stirrup", "agent": "", "value": 1846, "unit": "elo", "date": "2026-09-23", "frontier": true, "source_url": "https://artificialanalysis.ai/evaluations/gdpval-aa", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval", "benchmark_name": "GDPval", "institutions": "OpenAI", "metric": "Wins plus ties against experts", "subject": "Claude Opus 4.1", "model": "Claude Opus 4.1", "model_org": "Anthropic", "harness": "", "agent": "", "value": 47.6, "unit": "percent", "date": "2025-09", "frontier": false, "source_url": "https://proceedings.iclr.cc/paper_files/paper/2026/file/290c2430f91912204f30bbcc990fff1d-Paper-Conference.pdf", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval", "benchmark_name": "GDPval", "institutions": "OpenAI", "metric": "Wins plus ties against experts", "subject": "GPT-5.2 Thinking", "model": "GPT-5.2 Thinking", "model_org": "OpenAI", "harness": "", "agent": "", "value": 70.9, "unit": "percent", "date": "2025-12", "frontier": false, "source_url": "https://openai.com/index/introducing-gpt-5-2/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval", "benchmark_name": "GDPval", "institutions": "OpenAI", "metric": "Wins plus ties against experts", "subject": "Claude Opus 4.7", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "", "agent": "", "value": 80.3, "unit": "percent", "date": "2026-04", "frontier": false, "source_url": "https://openai.com/index/introducing-gpt-5-5/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "gdpval", "benchmark_name": "GDPval", "institutions": "OpenAI", "metric": "Wins plus ties against experts", "subject": "GPT-5.5", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "", "agent": "", "value": 84.9, "unit": "percent", "date": "2026-04", "frontier": true, "source_url": "https://openai.com/index/introducing-gpt-5-5/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "iac-eval", "benchmark_name": "IaC-Eval", "institutions": "University of Michigan; Cisco Research", "metric": "pass@1", "subject": "WizardCoder-33B-V1.1", "model": "WizardCoder-33B-V1.1", "model_org": "WizardLM", "harness": "", "agent": "", "value": 8.93, "unit": "percent", "date": "2024", "frontier": false, "source_url": "https://proceedings.neurips.cc/paper_files/paper/2024/file/f26b29298ae8acd94bd7e839688e329b-Paper-Datasets_and_Benchmarks_Track.pdf", "source_kind": "primary", "reported_by": "university-of-michigan", "retrieved_at": "2026-09-23" }, { "benchmark_id": "iac-eval", "benchmark_name": "IaC-Eval", "institutions": "University of Michigan; Cisco Research", "metric": "pass@1", "subject": "GPT-4 (zero-shot)", "model": "GPT-4", "model_org": "OpenAI", "harness": "", "agent": "", "value": 19.36, "unit": "percent", "date": "2024", "frontier": false, "source_url": "https://proceedings.neurips.cc/paper_files/paper/2024/file/f26b29298ae8acd94bd7e839688e329b-Paper-Datasets_and_Benchmarks_Track.pdf", "source_kind": "primary", "reported_by": "university-of-michigan", "retrieved_at": "2026-09-23" }, { "benchmark_id": "iac-eval", "benchmark_name": "IaC-Eval", "institutions": "University of Michigan; Cisco Research", "metric": "pass@1", "subject": "GPT-4 with retrieval-augmented generation", "model": "GPT-4", "model_org": "OpenAI", "harness": "RAG over Terraform documentation", "agent": "", "value": 36.7, "unit": "percent", "date": "2024", "frontier": true, "source_url": "https://proceedings.neurips.cc/paper_files/paper/2024/file/f26b29298ae8acd94bd7e839688e329b-Paper-Datasets_and_Benchmarks_Track.pdf", "source_kind": "primary", "reported_by": "university-of-michigan", "retrieved_at": "2026-09-23" }, { "benchmark_id": "impossiblebench", "benchmark_name": "ImpossibleBench", "institutions": "Carnegie Mellon University; Anthropic", "metric": "Cheating rate (Conflicting-SWEbench, full scaffold)", "subject": "GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "ImpossibleBench full scaffold (bash, python, text editor, think tools)", "agent": "", "value": 54, "unit": "percent", "date": "2025-10-23", "frontier": true, "source_url": "https://arxiv.org/html/2510.20270v1", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench-aa", "benchmark_name": "ITBench-AA", "institutions": "Artificial Analysis; IBM Research", "metric": "ITBench-AA score", "subject": "Gemini 3.1 Pro Preview", "model": "Gemini 3.1 Pro Preview", "model_org": "Google", "harness": "Stirrup", "agent": "", "value": 30, "unit": "percent", "date": "2026-05-27", "frontier": false, "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench-aa", "benchmark_name": "ITBench-AA", "institutions": "Artificial Analysis; IBM Research", "metric": "ITBench-AA score", "subject": "GLM-5.1 (Reasoning)", "model": "GLM-5.1", "model_org": "Zhipu AI", "harness": "Stirrup", "agent": "", "value": 40, "unit": "percent", "date": "2026-05-27", "frontier": false, "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench-aa", "benchmark_name": "ITBench-AA", "institutions": "Artificial Analysis; IBM Research", "metric": "ITBench-AA score", "subject": "GPT-5.5 (xhigh)", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "Stirrup", "agent": "", "value": 46, "unit": "percent", "date": "2026-05-27", "frontier": false, "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench-aa", "benchmark_name": "ITBench-AA", "institutions": "Artificial Analysis; IBM Research", "metric": "ITBench-AA score", "subject": "Claude Opus 4.7 (Adaptive Reasoning, Max Effort)", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "Stirrup", "agent": "", "value": 47, "unit": "percent", "date": "2026-05-27", "frontier": true, "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch", "source_kind": "primary", "reported_by": "artificial-analysis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench", "benchmark_name": "ITBench", "institutions": "IBM Research; UIUC", "metric": "SRE incidents resolved", "subject": "Agents powered by state-of-the-art models (paper, ICML 2025)", "model": "", "model_org": "", "harness": "ITBench reference agents", "agent": "", "value": 11.4, "unit": "percent", "date": "2025-07", "frontier": false, "source_url": "https://proceedings.mlr.press/v267/jha25a.html", "source_kind": "primary", "reported_by": "ibm", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench", "benchmark_name": "ITBench", "institutions": "IBM Research; UIUC", "metric": "SRE incidents resolved", "subject": "ITBench-SRE-Agent-LLama-3-3-70B", "model": "Llama 3.3 70B", "model_org": "Meta", "harness": "ITBench-SRE-Agent", "agent": "", "value": 12.5, "unit": "percent", "date": "2025-05-02", "frontier": false, "source_url": "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md", "source_kind": "primary", "reported_by": "ibm", "retrieved_at": "2026-09-23" }, { "benchmark_id": "itbench", "benchmark_name": "ITBench", "institutions": "IBM Research; UIUC", "metric": "SRE incidents resolved", "subject": "ITBench-SRE-Agent-GPT-4o", "model": "GPT-4o", "model_org": "OpenAI", "harness": "ITBench-SRE-Agent", "agent": "", "value": 25, "unit": "percent", "date": "2025-05-02", "frontier": true, "source_url": "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md", "source_kind": "primary", "reported_by": "ibm", "retrieved_at": "2026-09-23" }, { "benchmark_id": "martian-code-review-bench", "benchmark_name": "Code Review Bench", "institutions": "Martian", "metric": "F2 score (Core profile, offline set)", "subject": "Qodo Extended", "model": "", "model_org": "Qodo", "harness": "", "agent": "Qodo Extended", "value": 65.1, "unit": "score", "date": "2026-08-20", "frontier": true, "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "source_kind": "primary", "reported_by": "martian", "retrieved_at": "2026-09-23" }, { "benchmark_id": "martian-code-review-bench", "benchmark_name": "Code Review Bench", "institutions": "Martian", "metric": "F2 score (Core profile, offline set)", "subject": "Cubic v2", "model": "", "model_org": "Cubic", "harness": "", "agent": "Cubic v2", "value": 64.9, "unit": "score", "date": "2026-08-20", "frontier": false, "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "source_kind": "primary", "reported_by": "martian", "retrieved_at": "2026-09-23" }, { "benchmark_id": "martian-code-review-bench", "benchmark_name": "Code Review Bench", "institutions": "Martian", "metric": "F2 score (Core profile, offline set)", "subject": "Augment", "model": "", "model_org": "Augment", "harness": "", "agent": "Augment", "value": 64, "unit": "score", "date": "2026-08-20", "frontier": false, "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "source_kind": "primary", "reported_by": "martian", "retrieved_at": "2026-09-23" }, { "benchmark_id": "martian-code-review-bench", "benchmark_name": "Code Review Bench", "institutions": "Martian", "metric": "F2 score (Core profile, offline set)", "subject": "GitHub Copilot", "model": "", "model_org": "GitHub", "harness": "", "agent": "GitHub Copilot", "value": 58, "unit": "score", "date": "2026-08-20", "frontier": false, "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "source_kind": "primary", "reported_by": "martian", "retrieved_at": "2026-09-23" }, { "benchmark_id": "martian-code-review-bench", "benchmark_name": "Code Review Bench", "institutions": "Martian", "metric": "F2 score (Core profile, offline set)", "subject": "CodeRabbit", "model": "", "model_org": "CodeRabbit", "harness": "", "agent": "CodeRabbit", "value": 51.1, "unit": "score", "date": "2026-08-20", "frontier": false, "source_url": "https://raw.githubusercontent.com/withmartian/code-review-benchmark/main/offline/analysis/benchmark_dashboard.json", "source_kind": "primary", "reported_by": "martian", "retrieved_at": "2026-09-23" }, { "benchmark_id": "medagentbench", "benchmark_name": "MedAgentBench", "institutions": "Stanford University", "metric": "Task success rate", "subject": "DeepSeek-V3", "model": "DeepSeek-V3", "model_org": "DeepSeek", "harness": "MedAgentBench baseline orchestrator", "agent": "", "value": 62.67, "unit": "percent", "date": "2025-01-24", "frontier": false, "source_url": "https://arxiv.org/html/2501.14654v2", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "medagentbench", "benchmark_name": "MedAgentBench", "institutions": "Stanford University", "metric": "Task success rate", "subject": "GPT-4o", "model": "GPT-4o", "model_org": "OpenAI", "harness": "MedAgentBench baseline orchestrator", "agent": "", "value": 64, "unit": "percent", "date": "2025-01-24", "frontier": false, "source_url": "https://arxiv.org/html/2501.14654v2", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "medagentbench", "benchmark_name": "MedAgentBench", "institutions": "Stanford University", "metric": "Task success rate", "subject": "Claude 3.5 Sonnet v2", "model": "Claude 3.5 Sonnet v2", "model_org": "Anthropic", "harness": "MedAgentBench baseline orchestrator", "agent": "", "value": 69.67, "unit": "percent", "date": "2025-01-24", "frontier": true, "source_url": "https://arxiv.org/html/2501.14654v2", "source_kind": "primary", "reported_by": "stanford", "retrieved_at": "2026-09-23" }, { "benchmark_id": "metr-time-horizons", "benchmark_name": "METR Task-Completion Time Horizons", "institutions": "METR", "metric": "50% task-completion time horizon", "subject": "GPT-4 (0314)", "model": "GPT-4", "model_org": "OpenAI", "harness": "modular-public", "agent": "", "value": 4, "unit": "minutes", "date": "2023-03-14", "frontier": false, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "metr-time-horizons", "benchmark_name": "METR Task-Completion Time Horizons", "institutions": "METR", "metric": "50% task-completion time horizon", "subject": "Claude 3.7 Sonnet", "model": "Claude 3.7 Sonnet", "model_org": "Anthropic", "harness": "METR ReAct agent (Inspect)", "agent": "", "value": 60.4, "unit": "minutes", "date": "2025-02-24", "frontier": false, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "metr-time-horizons", "benchmark_name": "METR Task-Completion Time Horizons", "institutions": "METR", "metric": "50% task-completion time horizon", "subject": "GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "Triframe (Inspect)", "agent": "", "value": 203, "unit": "minutes", "date": "2025-08-07", "frontier": false, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "metr-time-horizons", "benchmark_name": "METR Task-Completion Time Horizons", "institutions": "METR", "metric": "50% task-completion time horizon", "subject": "Claude Opus 4.5", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "METR ReAct agent (Inspect)", "agent": "", "value": 293, "unit": "minutes", "date": "2025-11-24", "frontier": false, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "metr-time-horizons", "benchmark_name": "METR Task-Completion Time Horizons", "institutions": "METR", "metric": "50% task-completion time horizon", "subject": "Claude Opus 4.6", "model": "Claude Opus 4.6", "model_org": "Anthropic", "harness": "METR ReAct agent (Inspect)", "agent": "", "value": 718.8, "unit": "minutes", "date": "2026-02-05", "frontier": false, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "metr-time-horizons", "benchmark_name": "METR Task-Completion Time Horizons", "institutions": "METR", "metric": "50% task-completion time horizon", "subject": "Claude Mythos Preview (early)", "model": "Claude Mythos Preview (early)", "model_org": "Anthropic", "harness": "METR ReAct agent (Inspect)", "agent": "", "value": 1044.8, "unit": "minutes", "date": "2026-04-07", "frontier": true, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "mle-bench", "benchmark_name": "MLE-bench", "institutions": "OpenAI", "metric": "Any medal (%)", "subject": "AIDE + o1-preview", "model": "o1-preview", "model_org": "OpenAI", "harness": "AIDE", "agent": "", "value": 17.12, "unit": "percent", "date": "2024-10-08", "frontier": false, "source_url": "https://github.com/openai/mle-bench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "mle-bench", "benchmark_name": "MLE-bench", "institutions": "OpenAI", "metric": "Any medal (%)", "subject": "R&D-Agent (o3 + GPT-4.1)", "model": "o3 + GPT-4.1", "model_org": "OpenAI", "harness": "R&D-Agent", "agent": "R&D-Agent (Microsoft)", "value": 30.22, "unit": "percent", "date": "2025-08-15", "frontier": false, "source_url": "https://github.com/openai/mle-bench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "mle-bench", "benchmark_name": "MLE-bench", "institutions": "OpenAI", "metric": "Any medal (%)", "subject": "Famou-Agent 2.0 (Gemini-2.5-Pro)", "model": "Gemini 2.5 Pro", "model_org": "Google", "harness": "", "agent": "Famou-Agent 2.0 (Baidu)", "value": 59.56, "unit": "percent", "date": "2025-12-27", "frontier": false, "source_url": "https://github.com/openai/mle-bench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "mle-bench", "benchmark_name": "MLE-bench", "institutions": "OpenAI", "metric": "Any medal (%)", "subject": "AIBuildAI (Claude Opus 4.6)", "model": "Claude Opus 4.6", "model_org": "Anthropic", "harness": "", "agent": "AIBuildAI", "value": 63.11, "unit": "percent", "date": "2026-03-06", "frontier": false, "source_url": "https://github.com/openai/mle-bench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "mle-bench", "benchmark_name": "MLE-bench", "institutions": "OpenAI", "metric": "Any medal (%)", "subject": "Famou-Agent 2.0 (Gemini-3-Pro-Preview)", "model": "Gemini 3 Pro Preview", "model_org": "Google", "harness": "", "agent": "Famou-Agent 2.0 (Baidu)", "value": 64.44, "unit": "percent", "date": "2026-02-23", "frontier": true, "source_url": "https://github.com/openai/mle-bench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "osworld-verified", "benchmark_name": "OSWorld-Verified", "institutions": "HKU; Salesforce AI Research; Carnegie Mellon University; University of Waterloo", "metric": "Success rate", "subject": "CoACT-1", "model": "", "model_org": "Salesforce", "harness": "", "agent": "CoACT-1", "value": 60.76, "unit": "percent", "date": "2025-08-04", "frontier": false, "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "osworld-verified", "benchmark_name": "OSWorld-Verified", "institutions": "HKU; Salesforce AI Research; Carnegie Mellon University; University of Waterloo", "metric": "Success rate", "subject": "agent s3 w/ GPT-5 bBoN (N=10)", "model": "GPT-5", "model_org": "OpenAI", "harness": "Agent S3", "agent": "Simular Agent S3", "value": 69.9, "unit": "percent", "date": "2025-10-04", "frontier": false, "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "osworld-verified", "benchmark_name": "OSWorld-Verified", "institutions": "HKU; Salesforce AI Research; Carnegie Mellon University; University of Waterloo", "metric": "Success rate", "subject": "Holo3-35B-A3B", "model": "Holo3-35B-A3B", "model_org": "H Company", "harness": "", "agent": "", "value": 82.56, "unit": "percent", "date": "2026-04-20", "frontier": false, "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "osworld-verified", "benchmark_name": "OSWorld-Verified", "institutions": "HKU; Salesforce AI Research; Carnegie Mellon University; University of Waterloo", "metric": "Success rate", "subject": "claude-fable-5[1m]", "model": "Claude Fable 5", "model_org": "Anthropic", "harness": "", "agent": "", "value": 85.96, "unit": "percent", "date": "2026-08-01", "frontier": false, "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "osworld-verified", "benchmark_name": "OSWorld-Verified", "institutions": "HKU; Salesforce AI Research; Carnegie Mellon University; University of Waterloo", "metric": "Success rate", "subject": "Intelligence-Indeed Agent", "model": "", "model_org": "Intelligence Indeed", "harness": "", "agent": "Intelligence-Indeed Agent", "value": 90.19, "unit": "percent", "date": "2026-07-25", "frontier": true, "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "paperbench", "benchmark_name": "PaperBench", "institutions": "OpenAI", "metric": "Average replication score", "subject": "BasicAgent claude-3.5-sonnet", "model": "Claude 3.5 Sonnet (New)", "model_org": "Anthropic", "harness": "BasicAgent", "agent": "", "value": 21, "unit": "percent", "date": "2025-04-02", "frontier": false, "source_url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "paperbench", "benchmark_name": "PaperBench", "institutions": "OpenAI", "metric": "Average replication score", "subject": "IterativeAgent o1-high (24h limit)", "model": "o1", "model_org": "OpenAI", "harness": "IterativeAgent", "agent": "", "value": 24.4, "unit": "percent", "date": "2025-04-02", "frontier": false, "source_url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "paperbench", "benchmark_name": "PaperBench", "institutions": "OpenAI", "metric": "Average replication score", "subject": "IterativeAgent o1-high (36h limit)", "model": "o1", "model_org": "OpenAI", "harness": "IterativeAgent", "agent": "", "value": 26, "unit": "percent", "date": "2025-04-02", "frontier": true, "source_url": "https://github.com/openai/frontier-evals/tree/main/project/paperbench", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "pr-arena", "benchmark_name": "PR Arena", "institutions": "aavetis", "metric": "Merge rate of ready PRs", "subject": "Codex", "model": "", "model_org": "OpenAI", "harness": "", "agent": "OpenAI Codex", "value": 85.8, "unit": "percent", "date": "2025-05-26", "frontier": false, "source_url": "https://raw.githubusercontent.com/aavetis/PRarena/main/data.csv", "source_kind": "primary", "reported_by": "aavetis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "pr-arena", "benchmark_name": "PR Arena", "institutions": "aavetis", "metric": "Merge rate of ready PRs", "subject": "Copilot", "model": "", "model_org": "GitHub", "harness": "", "agent": "GitHub Copilot coding agent", "value": 75.94, "unit": "percent", "date": "2025-05-26", "frontier": false, "source_url": "https://raw.githubusercontent.com/aavetis/PRarena/main/data.csv", "source_kind": "primary", "reported_by": "aavetis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "pr-arena", "benchmark_name": "PR Arena", "institutions": "aavetis", "metric": "Merge rate of ready PRs", "subject": "Copilot", "model": "", "model_org": "GitHub", "harness": "", "agent": "GitHub Copilot coding agent", "value": 95.11, "unit": "percent", "date": "2026-01-10", "frontier": false, "source_url": "https://raw.githubusercontent.com/aavetis/PRarena/main/data.csv", "source_kind": "primary", "reported_by": "aavetis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "pr-arena", "benchmark_name": "PR Arena", "institutions": "aavetis", "metric": "Merge rate of ready PRs", "subject": "Copilot", "model": "", "model_org": "GitHub", "harness": "", "agent": "GitHub Copilot coding agent", "value": 96.02, "unit": "percent", "date": "2026-09-23", "frontier": true, "source_url": "https://github.com/aavetis/PRarena", "source_kind": "primary", "reported_by": "aavetis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "pr-arena", "benchmark_name": "PR Arena", "institutions": "aavetis", "metric": "Merge rate of ready PRs", "subject": "Codex", "model": "", "model_org": "OpenAI", "harness": "", "agent": "OpenAI Codex", "value": 89.65, "unit": "percent", "date": "2026-09-23", "frontier": false, "source_url": "https://github.com/aavetis/PRarena", "source_kind": "primary", "reported_by": "aavetis", "retrieved_at": "2026-09-23" }, { "benchmark_id": "re-bench", "benchmark_name": "RE-Bench", "institutions": "METR", "metric": "Human-expert percentile matched (8-hour budget)", "subject": "o1 (AIDE scaffold)", "model": "o1", "model_org": "OpenAI", "harness": "AIDE", "agent": "", "value": 30, "unit": "percent", "date": "2025-01-31", "frontier": false, "source_url": "https://metr.org/blog/2025-01-31-update-sonnet-o1-evals/", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "re-bench", "benchmark_name": "RE-Bench", "institutions": "METR", "metric": "Human-expert percentile matched (8-hour budget)", "subject": "Claude 3.5 Sonnet (New) (Modular scaffold)", "model": "Claude 3.5 Sonnet (New)", "model_org": "Anthropic", "harness": "Modular", "agent": "", "value": 37, "unit": "percent", "date": "2025-01-31", "frontier": true, "source_url": "https://metr.org/blog/2025-01-31-update-sonnet-o1-evals/", "source_kind": "primary", "reported_by": "metr", "retrieved_at": "2026-09-23" }, { "benchmark_id": "remote-labor-index", "benchmark_name": "Remote Labor Index", "institutions": "CAIS; Scale AI", "metric": "Automation rate", "subject": "Manus", "model": "", "model_org": "Manus", "harness": "", "agent": "Manus", "value": 2.5, "unit": "percent", "date": "2025-10", "frontier": false, "source_url": "https://labs.scale.com/leaderboard/rli", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "remote-labor-index", "benchmark_name": "Remote Labor Index", "institutions": "CAIS; Scale AI", "metric": "Automation rate", "subject": "Claude Opus 4.6 (Claude Cowork scaffold)", "model": "Claude Opus 4.6", "model_org": "Anthropic", "harness": "Claude Cowork", "agent": "", "value": 4.17, "unit": "percent", "date": "2026", "frontier": false, "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation", "source_kind": "primary", "reported_by": "cais", "retrieved_at": "2026-09-23" }, { "benchmark_id": "remote-labor-index", "benchmark_name": "Remote Labor Index", "institutions": "CAIS; Scale AI", "metric": "Automation rate", "subject": "GPT-5.5", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "", "agent": "", "value": 6.3, "unit": "percent", "date": "2026-07-01", "frontier": false, "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation", "source_kind": "primary", "reported_by": "cais", "retrieved_at": "2026-09-23" }, { "benchmark_id": "remote-labor-index", "benchmark_name": "Remote Labor Index", "institutions": "CAIS; Scale AI", "metric": "Automation rate", "subject": "Claude Opus 4.8", "model": "Claude Opus 4.8", "model_org": "Anthropic", "harness": "", "agent": "", "value": 8.3, "unit": "percent", "date": "2026-07-01", "frontier": false, "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation", "source_kind": "primary", "reported_by": "cais", "retrieved_at": "2026-09-23" }, { "benchmark_id": "remote-labor-index", "benchmark_name": "Remote Labor Index", "institutions": "CAIS; Scale AI", "metric": "Automation rate", "subject": "Claude Fable 5", "model": "Claude Fable 5", "model_org": "Anthropic", "harness": "", "agent": "", "value": 15.8, "unit": "percent", "date": "2026-07-01", "frontier": true, "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation", "source_kind": "primary", "reported_by": "cais", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spider-2", "benchmark_name": "Spider 2.0", "institutions": "HKU; Salesforce AI Research", "metric": "Execution accuracy (Spider 2.0-Snow)", "subject": "Spider-Agent + o1-preview", "model": "o1-preview", "model_org": "OpenAI", "harness": "Spider-Agent", "agent": "", "value": 23.58, "unit": "percent", "date": "2024-11-30", "frontier": false, "source_url": "https://spider2-sql.github.io", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spider-2", "benchmark_name": "Spider 2.0", "institutions": "HKU; Salesforce AI Research", "metric": "Execution accuracy (Spider 2.0-Snow)", "subject": "ReFoRCE + o1-preview", "model": "o1-preview", "model_org": "OpenAI", "harness": "ReFoRCE", "agent": "ReFoRCE (Hao AI Lab x Snowflake)", "value": 31.26, "unit": "percent", "date": "2025-01-28", "frontier": false, "source_url": "https://spider2-sql.github.io", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spider-2", "benchmark_name": "Spider 2.0", "institutions": "HKU; Salesforce AI Research", "metric": "Execution accuracy (Spider 2.0-Snow)", "subject": "LingXi Agent + Claude-Sonnet-4.5", "model": "Claude Sonnet 4.5", "model_org": "Anthropic", "harness": "", "agent": "LingXi Agent (Ant Group)", "value": 79.89, "unit": "percent", "date": "2025-12-05", "frontier": false, "source_url": "https://spider2-sql.github.io", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spider-2", "benchmark_name": "Spider 2.0", "institutions": "HKU; Salesforce AI Research", "metric": "Execution accuracy (Spider 2.0-Snow)", "subject": "Prism Swarm with Deepthink + Claude-Sonnet-4.5", "model": "Claude Sonnet 4.5", "model_org": "Anthropic", "harness": "", "agent": "Prism Swarm (Paytm)", "value": 90.49, "unit": "percent", "date": "2026-01-27", "frontier": false, "source_url": "https://spider2-sql.github.io", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spider-2", "benchmark_name": "Spider 2.0", "institutions": "HKU; Salesforce AI Research", "metric": "Execution accuracy (Spider 2.0-Snow)", "subject": "Genloop's Sentinel Agent v2 Pro", "model": "", "model_org": "", "harness": "", "agent": "Sentinel Agent v2 Pro (Genloop)", "value": 96.7, "unit": "percent", "date": "2026-03-01", "frontier": true, "source_url": "https://spider2-sql.github.io", "source_kind": "primary", "reported_by": "hku", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spreadsheetbench", "benchmark_name": "SpreadsheetBench", "institutions": "Renmin University of China", "metric": "Overall pass@1 (V1, 912 questions)", "subject": "ChatGPT Agent w/ .xlsx", "model": "", "model_org": "OpenAI", "harness": "", "agent": "ChatGPT Agent", "value": 45.5, "unit": "percent", "date": "2025-07-17", "frontier": false, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "reported_by": "renmin-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spreadsheetbench", "benchmark_name": "SpreadsheetBench", "institutions": "Renmin University of China", "metric": "Overall pass@1 (V1, 912 questions)", "subject": "Shortcut.ai", "model": "", "model_org": "Shortcut.ai", "harness": "", "agent": "Shortcut.ai", "value": 59.25, "unit": "percent", "date": "2025-10-16", "frontier": false, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "reported_by": "renmin-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spreadsheetbench", "benchmark_name": "SpreadsheetBench", "institutions": "Renmin University of China", "metric": "Overall pass@1 (V1, 912 questions)", "subject": "Gemini in Google Sheets", "model": "", "model_org": "Google", "harness": "", "agent": "Gemini in Google Sheets", "value": 70.48, "unit": "percent", "date": "2026-03-10", "frontier": false, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "reported_by": "renmin-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spreadsheetbench", "benchmark_name": "SpreadsheetBench", "institutions": "Renmin University of China", "metric": "Overall pass@1 (V1, 912 questions)", "subject": "WPS AI (Seed 2.0)", "model": "", "model_org": "Kingsoft Office", "harness": "", "agent": "WPS AI", "value": 73.46, "unit": "percent", "date": "2026-06-16", "frontier": false, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "reported_by": "renmin-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "spreadsheetbench", "benchmark_name": "SpreadsheetBench", "institutions": "Renmin University of China", "metric": "Overall pass@1 (V1, 912 questions)", "subject": "Qingqiu Agent", "model": "", "model_org": "Kingsoft Office", "harness": "", "agent": "Qingqiu Agent", "value": 83.11, "unit": "percent", "date": "2026-06-23", "frontier": true, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "reported_by": "renmin-university", "retrieved_at": "2026-09-23" }, { "benchmark_id": "sregym", "benchmark_name": "SREGym", "institutions": "UIUC; University of Toronto", "metric": "End-to-end success rate", "subject": "Stratus (Kimi K2.5)", "model": "Kimi K2.5", "model_org": "Moonshot AI", "harness": "", "agent": "Stratus", "value": 27.4, "unit": "percent", "date": "2026-05", "frontier": false, "source_url": "https://arxiv.org/abs/2605.07161", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23" }, { "benchmark_id": "sregym", "benchmark_name": "SREGym", "institutions": "UIUC; University of Toronto", "metric": "End-to-end success rate", "subject": "Codex (GPT-5.4)", "model": "GPT-5.4", "model_org": "OpenAI", "harness": "", "agent": "Codex", "value": 53.3, "unit": "percent", "date": "2026-05", "frontier": false, "source_url": "https://arxiv.org/abs/2605.07161", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23" }, { "benchmark_id": "sregym", "benchmark_name": "SREGym", "institutions": "UIUC; University of Toronto", "metric": "End-to-end success rate", "subject": "Claude Code (Claude Sonnet 4.6)", "model": "Claude Sonnet 4.6", "model_org": "Anthropic", "harness": "", "agent": "Claude Code", "value": 60.7, "unit": "percent", "date": "2026-05", "frontier": false, "source_url": "https://arxiv.org/abs/2605.07161", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23" }, { "benchmark_id": "sregym", "benchmark_name": "SREGym", "institutions": "UIUC; University of Toronto", "metric": "End-to-end success rate", "subject": "GitHub Copilot (GPT-5.6 Sol, max)", "model": "GPT-5.6 Sol", "model_org": "OpenAI", "harness": "", "agent": "GitHub Copilot", "value": 72.2, "unit": "percent", "date": "2026", "frontier": true, "source_url": "https://sregym.com/leaderboard", "source_kind": "primary", "reported_by": "uiuc", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-pro", "benchmark_name": "SWE-Bench Pro", "institutions": "Scale AI", "metric": "Resolve rate (public set)", "subject": "OpenAI GPT-5 (medium)", "model": "GPT-5", "model_org": "OpenAI", "harness": "SWE-Agent", "agent": "", "value": 23.3, "unit": "percent", "date": "2025-09-21", "frontier": false, "source_url": "https://arxiv.org/html/2509.16941v2", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-pro", "benchmark_name": "SWE-Bench Pro", "institutions": "Scale AI", "metric": "Resolve rate (public set)", "subject": "Claude Sonnet 4.5", "model": "Claude Sonnet 4.5", "model_org": "Anthropic", "harness": "SWE-Agent", "agent": "", "value": 43.6, "unit": "percent", "date": "2025-11-14", "frontier": false, "source_url": "https://arxiv.org/html/2509.16941v2", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-pro", "benchmark_name": "SWE-Bench Pro", "institutions": "Scale AI", "metric": "Resolve rate (public set)", "subject": "claude-opus-4-6 (thinking)", "model": "Claude Opus 4.6", "model_org": "Anthropic", "harness": "mini-swe-agent", "agent": "", "value": 51.9, "unit": "percent", "date": "2026-04-08", "frontier": false, "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-pro", "benchmark_name": "SWE-Bench Pro", "institutions": "Scale AI", "metric": "Resolve rate (public set)", "subject": "gpt-5.4 (xHigh)", "model": "GPT-5.4", "model_org": "OpenAI", "harness": "mini-swe-agent", "agent": "", "value": 59.1, "unit": "percent", "date": "2026-04-08", "frontier": false, "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-pro", "benchmark_name": "SWE-Bench Pro", "institutions": "Scale AI", "metric": "Resolve rate (public set)", "subject": "Muse Spark 1.1", "model": "Muse Spark 1.1", "model_org": "Meta", "harness": "mini-swe-agent", "agent": "", "value": 61.5, "unit": "percent", "date": "2026-07-09", "frontier": true, "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "source_kind": "primary", "reported_by": "scale-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-verified", "benchmark_name": "SWE-bench Verified", "institutions": "Princeton University; OpenAI", "metric": "Resolved", "subject": "GPT-4o (best open-source scaffold)", "model": "GPT-4o", "model_org": "OpenAI", "harness": "", "agent": "", "value": 33.2, "unit": "percent", "date": "2024-08-13", "frontier": false, "source_url": "https://openai.com/index/introducing-swe-bench-verified/", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-verified", "benchmark_name": "SWE-bench Verified", "institutions": "Princeton University; OpenAI", "metric": "Resolved", "subject": "OpenHands + CodeAct v2.1 (claude-3-5-sonnet-20241022)", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "harness": "OpenHands", "agent": "", "value": 53, "unit": "percent", "date": "2024-10-29", "frontier": false, "source_url": "https://www.swebench.com/", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-verified", "benchmark_name": "SWE-bench Verified", "institutions": "Princeton University; OpenAI", "metric": "Resolved", "subject": "TRAE + Claude Sonnet 4 + Opus 4 + Sonnet 3.7 + Gemini 2.5 Pro", "model": "Multiple", "model_org": "", "harness": "TRAE", "agent": "", "value": 75.2, "unit": "percent", "date": "2025-06-12", "frontier": false, "source_url": "https://www.swebench.com/", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-verified", "benchmark_name": "SWE-bench Verified", "institutions": "Princeton University; OpenAI", "metric": "Resolved", "subject": "Claude 4.5 Opus (high), mini-SWE-agent", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "mini-SWE-agent", "agent": "", "value": 76.8, "unit": "percent", "date": "2026-02-17", "frontier": false, "source_url": "https://www.swebench.com/", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-bench-verified", "benchmark_name": "SWE-bench Verified", "institutions": "Princeton University; OpenAI", "metric": "Resolved", "subject": "Sonar Foundation Agent + Claude 4.5 Opus", "model": "Claude Opus 4.5", "model_org": "Anthropic", "harness": "Sonar Foundation Agent", "agent": "Sonar Foundation Agent", "value": 79.2, "unit": "percent", "date": "2025-12-05", "frontier": true, "source_url": "https://www.swebench.com/", "source_kind": "primary", "reported_by": "princeton", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-lancer", "benchmark_name": "SWE-Lancer", "institutions": "OpenAI", "metric": "IC SWE Diamond pass@1", "subject": "Claude 3.5 Sonnet", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "harness": "", "agent": "", "value": 26.2, "unit": "percent", "date": "2025-02-17", "frontier": false, "source_url": "https://arxiv.org/html/2502.12115v4", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-lancer", "benchmark_name": "SWE-Lancer", "institutions": "OpenAI", "metric": "IC SWE Diamond pass@1", "subject": "gpt-5 (no browsing)", "model": "GPT-5", "model_org": "OpenAI", "harness": "", "agent": "", "value": 55, "unit": "percent", "date": "2025-11-18", "frontier": false, "source_url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-lancer", "benchmark_name": "SWE-Lancer", "institutions": "OpenAI", "metric": "IC SWE Diamond pass@1", "subject": "gpt-5-codex", "model": "GPT-5-Codex", "model_org": "OpenAI", "harness": "", "agent": "", "value": 67, "unit": "percent", "date": "2025-11-18", "frontier": false, "source_url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-lancer", "benchmark_name": "SWE-Lancer", "institutions": "OpenAI", "metric": "IC SWE Diamond pass@1", "subject": "gpt-5.1-codex-max", "model": "GPT-5.1-Codex-Max", "model_org": "OpenAI", "harness": "", "agent": "", "value": 80, "unit": "percent", "date": "2025-11-18", "frontier": true, "source_url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer", "source_kind": "primary", "reported_by": "openai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-rebench", "benchmark_name": "SWE-rebench", "institutions": "Nebius", "metric": "Resolved rate", "subject": "gpt-4.1-2025-04-14", "model": "GPT-4.1", "model_org": "OpenAI", "harness": "SWE-rebench ReAct scaffold", "agent": "", "value": 31.1, "unit": "percent", "date": "2025-05-26", "frontier": false, "source_url": "https://arxiv.org/html/2505.20411v2", "source_kind": "primary", "reported_by": "nebius", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-rebench", "benchmark_name": "SWE-rebench", "institutions": "Nebius", "metric": "Resolved rate", "subject": "Anthropic Opus 5 [high]", "model": "Claude Opus 5", "model_org": "Anthropic", "harness": "SWE-rebench ReAct scaffold", "agent": "", "value": 63.4, "unit": "percent", "date": "2026-07-01", "frontier": false, "source_url": "https://swe-rebench.com/", "source_kind": "primary", "reported_by": "nebius", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-rebench", "benchmark_name": "SWE-rebench", "institutions": "Nebius", "metric": "Resolved rate", "subject": "Grok 4.5 [high]", "model": "Grok 4.5", "model_org": "xAI", "harness": "SWE-rebench ReAct scaffold", "agent": "", "value": 63.8, "unit": "percent", "date": "2026-07-01", "frontier": false, "source_url": "https://swe-rebench.com/", "source_kind": "primary", "reported_by": "nebius", "retrieved_at": "2026-09-23" }, { "benchmark_id": "swe-rebench", "benchmark_name": "SWE-rebench", "institutions": "Nebius", "metric": "Resolved rate", "subject": "Anthropic Fable 5 [high]", "model": "Claude Fable 5", "model_org": "Anthropic", "harness": "SWE-rebench ReAct scaffold", "agent": "", "value": 64.5, "unit": "percent", "date": "2026-07-01", "frontier": true, "source_url": "https://swe-rebench.com/", "source_kind": "primary", "reported_by": "nebius", "retrieved_at": "2026-09-23" }, { "benchmark_id": "tau2-bench", "benchmark_name": "τ²-bench", "institutions": "Sierra", "metric": "Telecom pass^1", "subject": "GPT-4.1", "model": "GPT-4.1", "model_org": "OpenAI", "harness": "tau2-bench default agent", "agent": "", "value": 34, "unit": "percent", "date": "2025-06-09", "frontier": false, "source_url": "https://arxiv.org/abs/2506.07982", "source_kind": "primary", "reported_by": "sierra", "retrieved_at": "2026-09-23" }, { "benchmark_id": "tau2-bench", "benchmark_name": "τ²-bench", "institutions": "Sierra", "metric": "Telecom pass^1", "subject": "o4-mini", "model": "o4-mini", "model_org": "OpenAI", "harness": "tau2-bench default agent", "agent": "", "value": 50.2, "unit": "percent", "date": "2025-06-09", "frontier": false, "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/web/leaderboard/public/submissions/o4-mini_openai_2024-06-20/submission.json", "source_kind": "primary", "reported_by": "sierra", "retrieved_at": "2026-09-23" }, { "benchmark_id": "tau2-bench", "benchmark_name": "τ²-bench", "institutions": "Sierra", "metric": "Telecom pass^1", "subject": "GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "tau2-bench default agent", "agent": "", "value": 95.8, "unit": "percent", "date": "2025-08-09", "frontier": false, "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/web/leaderboard/public/submissions/gpt-5_sierra_2025-08-09/submission.json", "source_kind": "primary", "reported_by": "sierra", "retrieved_at": "2026-09-23" }, { "benchmark_id": "tau2-bench", "benchmark_name": "τ²-bench", "institutions": "Sierra", "metric": "Telecom pass^1", "subject": "Qwen3-Max-Thinking", "model": "Qwen3-Max-Thinking", "model_org": "Alibaba (Qwen)", "harness": "tau2-bench default agent", "agent": "", "value": 98.2, "unit": "percent", "date": "2026-01-23", "frontier": false, "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/web/leaderboard/public/submissions/qwen3-max_qwen_2026-01-23/submission.json", "source_kind": "lab-reported", "reported_by": "Qwen team (leaderboard submission)", "retrieved_at": "2026-09-23" }, { "benchmark_id": "tau2-bench", "benchmark_name": "τ²-bench", "institutions": "Sierra", "metric": "Telecom pass^1", "subject": "Qwen3.5-397B-A17B (thinking)", "model": "Qwen3.5-397B-A17B", "model_org": "Alibaba (Qwen)", "harness": "tau2-bench default agent", "agent": "", "value": 97.8, "unit": "percent", "date": "2026-02-27", "frontier": true, "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/web/leaderboard/public/submissions/qwen3.5-397b-a17b-think_sierra_2026-03-02/submission.json", "source_kind": "primary", "reported_by": "sierra", "retrieved_at": "2026-09-23" }, { "benchmark_id": "terminal-bench-2", "benchmark_name": "Terminal-Bench 2.0", "institutions": "Laude Institute; Stanford University", "metric": "Resolution rate", "subject": "Codex CLI + GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "Codex CLI", "agent": "", "value": 49.6, "unit": "percent", "date": "2025-08-07", "frontier": false, "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "source_kind": "primary", "reported_by": "laude-institute", "retrieved_at": "2026-09-23" }, { "benchmark_id": "terminal-bench-2", "benchmark_name": "Terminal-Bench 2.0", "institutions": "Laude Institute; Stanford University", "metric": "Resolution rate", "subject": "Ante + Gemini 3 Pro", "model": "Gemini 3 Pro", "model_org": "Google", "harness": "Ante", "agent": "", "value": 69.4, "unit": "percent", "date": "2025-11-18", "frontier": false, "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "source_kind": "primary", "reported_by": "laude-institute", "retrieved_at": "2026-09-23" }, { "benchmark_id": "terminal-bench-2", "benchmark_name": "Terminal-Bench 2.0", "institutions": "Laude Institute; Stanford University", "metric": "Resolution rate", "subject": "Codex CLI + GPT-5.5", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "Codex CLI", "agent": "", "value": 82.2, "unit": "percent", "date": "2026-04-23", "frontier": false, "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "source_kind": "primary", "reported_by": "laude-institute", "retrieved_at": "2026-09-23" }, { "benchmark_id": "terminal-bench-2", "benchmark_name": "Terminal-Bench 2.0", "institutions": "Laude Institute; Stanford University", "metric": "Resolution rate", "subject": "LemonHarness (Gemini 3.1 Pro Preview + GPT-5.3-Codex)", "model": "Multiple", "model_org": "", "harness": "LemonHarness", "agent": "", "value": 84.5, "unit": "percent", "date": "2026-05-14", "frontier": false, "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "source_kind": "primary", "reported_by": "laude-institute", "retrieved_at": "2026-09-23" }, { "benchmark_id": "terminal-bench-2", "benchmark_name": "Terminal-Bench 2.0", "institutions": "Laude Institute; Stanford University", "metric": "Resolution rate", "subject": "NexAU-AHE + GPT-5.5", "model": "GPT-5.5", "model_org": "OpenAI", "harness": "NexAU-AHE", "agent": "", "value": 84.7, "unit": "percent", "date": "2026-04-23", "frontier": true, "source_url": "https://www.tbench.ai/leaderboard/terminal-bench/2.0", "source_kind": "primary", "reported_by": "laude-institute", "retrieved_at": "2026-09-23" }, { "benchmark_id": "theagentcompany", "benchmark_name": "TheAgentCompany", "institutions": "Carnegie Mellon University", "metric": "Tasks resolved", "subject": "OpenHands + Claude 3.5 Sonnet", "model": "Claude 3.5 Sonnet", "model_org": "Anthropic", "harness": "OpenHands", "agent": "", "value": 24, "unit": "percent", "date": "2024-12-17", "frontier": false, "source_url": "https://raw.githubusercontent.com/TheAgentCompany/theagentcompany.github.io/main/src/data/leaderboard.json", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "theagentcompany", "benchmark_name": "TheAgentCompany", "institutions": "Carnegie Mellon University", "metric": "Tasks resolved", "subject": "OpenHands + Gemini 2.5 Pro", "model": "Gemini 2.5 Pro", "model_org": "Google", "harness": "OpenHands", "agent": "", "value": 30.29, "unit": "percent", "date": "2025-05-10", "frontier": false, "source_url": "https://raw.githubusercontent.com/TheAgentCompany/theagentcompany.github.io/main/src/data/leaderboard.json", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "theagentcompany", "benchmark_name": "TheAgentCompany", "institutions": "Carnegie Mellon University", "metric": "Tasks resolved", "subject": "OpenHands-Versa + Claude Sonnet 4", "model": "Claude Sonnet 4", "model_org": "Anthropic", "harness": "OpenHands-Versa", "agent": "", "value": 33.14, "unit": "percent", "date": "2025-06-14", "frontier": false, "source_url": "https://raw.githubusercontent.com/TheAgentCompany/theagentcompany.github.io/main/src/data/leaderboard.json", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "theagentcompany", "benchmark_name": "TheAgentCompany", "institutions": "Carnegie Mellon University", "metric": "Tasks resolved", "subject": "TTE-MatrixAgent + DeepSeek-V3.2", "model": "DeepSeek-V3.2", "model_org": "DeepSeek", "harness": "TTE-MatrixAgent", "agent": "", "value": 42.86, "unit": "percent", "date": "2025-11-10", "frontier": true, "source_url": "https://raw.githubusercontent.com/TheAgentCompany/theagentcompany.github.io/main/src/data/leaderboard.json", "source_kind": "primary", "reported_by": "cmu", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vals-finance-agent", "benchmark_name": "Finance Agent Benchmark", "institutions": "Vals AI", "metric": "Accuracy", "subject": "o3", "model": "o3", "model_org": "OpenAI", "harness": "Vals finance-agent v1.0", "agent": "", "value": 46.8, "unit": "percent", "date": "2025-05-20", "frontier": false, "source_url": "https://arxiv.org/abs/2508.00828", "source_kind": "primary", "reported_by": "vals-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vals-finance-agent", "benchmark_name": "Finance Agent Benchmark", "institutions": "Vals AI", "metric": "Accuracy", "subject": "GPT-5.2", "model": "GPT-5.2", "model_org": "OpenAI", "harness": "Vals finance-agent v1.1", "agent": "", "value": 58.54, "unit": "percent", "date": "2026-06-04", "frontier": false, "source_url": "https://www.vals.ai/benchmarks/finance_agent", "source_kind": "primary", "reported_by": "vals-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vals-finance-agent", "benchmark_name": "Finance Agent Benchmark", "institutions": "Vals AI", "metric": "Accuracy", "subject": "Claude Sonnet 4.6", "model": "Claude Sonnet 4.6", "model_org": "Anthropic", "harness": "Vals finance-agent v1.1", "agent": "", "value": 63.33, "unit": "percent", "date": "2026-06-04", "frontier": false, "source_url": "https://www.vals.ai/benchmarks/finance_agent", "source_kind": "primary", "reported_by": "vals-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vals-finance-agent", "benchmark_name": "Finance Agent Benchmark", "institutions": "Vals AI", "metric": "Accuracy", "subject": "Claude Opus 4.7", "model": "Claude Opus 4.7", "model_org": "Anthropic", "harness": "Vals finance-agent v1.1", "agent": "", "value": 64.37, "unit": "percent", "date": "2026-06-04", "frontier": true, "source_url": "https://www.vals.ai/benchmarks/finance_agent", "source_kind": "primary", "reported_by": "vals-ai", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vending-bench-2", "benchmark_name": "Vending-Bench 2", "institutions": "Andon Labs", "metric": "Final bank balance", "subject": "Gemini 3 Pro", "model": "Gemini 3 Pro", "model_org": "Google", "harness": "", "agent": "", "value": 5478.16, "unit": "dollars", "date": "2025-11-18", "frontier": false, "source_url": "http://web.archive.org/web/20251118175816/https://andonlabs.com/evals/vending-bench-2", "source_kind": "primary", "reported_by": "andon-labs", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vending-bench-2", "benchmark_name": "Vending-Bench 2", "institutions": "Andon Labs", "metric": "Final bank balance", "subject": "Claude Opus 4.6", "model": "Claude Opus 4.6", "model_org": "Anthropic", "harness": "", "agent": "", "value": 8017.59, "unit": "dollars", "date": "2026-02-04", "frontier": false, "source_url": "https://andonlabs.com/evals/vending-bench-2", "source_kind": "primary", "reported_by": "andon-labs", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vending-bench-2", "benchmark_name": "Vending-Bench 2", "institutions": "Andon Labs", "metric": "Final bank balance", "subject": "Claude Opus 5", "model": "Claude Opus 5", "model_org": "Anthropic", "harness": "", "agent": "", "value": 11181.87, "unit": "dollars", "date": "2026-07-28", "frontier": false, "source_url": "https://andonlabs.com/evals/vending-bench-2", "source_kind": "primary", "reported_by": "andon-labs", "retrieved_at": "2026-09-23" }, { "benchmark_id": "vending-bench-2", "benchmark_name": "Vending-Bench 2", "institutions": "Andon Labs", "metric": "Final bank balance", "subject": "GPT-6 Astra", "model": "GPT-6 Astra", "model_org": "OpenAI", "harness": "", "agent": "", "value": 15514.7, "unit": "dollars", "date": "2026-09-07", "frontier": true, "source_url": "https://andonlabs.com/evals/vending-bench-2", "source_kind": "primary", "reported_by": "andon-labs", "retrieved_at": "2026-09-23" }, { "benchmark_id": "workarena", "benchmark_name": "WorkArena", "institutions": "ServiceNow", "metric": "WorkArena-L1 success rate", "subject": "GenericAgent-GPT-4o", "model": "GPT-4o", "model_org": "OpenAI", "harness": "BrowserGym GenericAgent", "agent": "", "value": 45.5, "unit": "percent", "date": "2024-10-23", "frontier": false, "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-4o/workarena-l1.json", "source_kind": "primary", "reported_by": "servicenow", "retrieved_at": "2026-09-23" }, { "benchmark_id": "workarena", "benchmark_name": "WorkArena", "institutions": "ServiceNow", "metric": "WorkArena-L1 success rate", "subject": "GenericAgent-Claude-4-Sonnet", "model": "Claude Sonnet 4", "model_org": "Anthropic", "harness": "BrowserGym GenericAgent", "agent": "", "value": 63.3, "unit": "percent", "date": "2025-08-07", "frontier": false, "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-Claude-4-Sonnet/workarena-l1.json", "source_kind": "primary", "reported_by": "servicenow", "retrieved_at": "2026-09-23" }, { "benchmark_id": "workarena", "benchmark_name": "WorkArena", "institutions": "ServiceNow", "metric": "WorkArena-L1 success rate", "subject": "GenericAgent-GPT-5", "model": "GPT-5", "model_org": "OpenAI", "harness": "BrowserGym GenericAgent", "agent": "", "value": 79.1, "unit": "percent", "date": "2025-08-07", "frontier": false, "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/GenericAgent-GPT-5/workarena-l1.json", "source_kind": "primary", "reported_by": "servicenow", "retrieved_at": "2026-09-23" }, { "benchmark_id": "workarena", "benchmark_name": "WorkArena", "institutions": "ServiceNow", "metric": "WorkArena-L1 success rate", "subject": "IpaziaHPA-Gemini-3-flash-preview", "model": "Gemini 3 Flash Preview", "model_org": "Google", "harness": "IpaziaHPA", "agent": "IpaziaHPA (Ipazia S.p.a.)", "value": 90.3, "unit": "percent", "date": "2026-03-07", "frontier": true, "source_url": "https://huggingface.co/spaces/ServiceNow/browsergym-leaderboard/raw/main/results/IpaziaHPA-Gemini-3-flash-preview/workarena-l1.json", "source_kind": "third-party", "reported_by": "Ipazia S.p.a. (submitted to the ServiceNow BrowserGym leaderboard)", "retrieved_at": "2026-09-23" } ], "institutions": [ { "id": "aavetis", "name": "aavetis", "full_name": "aavetis (independent maintainer of PR Arena)", "type": "independent", "url": "https://github.com/aavetis", "x_handle": "albfresco", "logo_permission": false, "description": "Individual GitHub user who runs PR Arena (prarena.ai), a public tracker of pull requests that coding agents open on GitHub. The GitHub profile lists Microsoft as employer; PR Arena is a personal project. The X handle comes from the twitter_username field of the GitHub profile." }, { "id": "ai-digest", "name": "AI Digest", "full_name": "AI Digest (a project of Sage)", "type": "nonprofit", "url": "https://theaidigest.org", "x_handle": "aidigest_", "logo_permission": false }, { "id": "aider", "name": "Aider", "full_name": "Aider (open-source project)", "type": "community", "url": "https://aider.chat", "x_handle": null, "logo_permission": false }, { "id": "andon-labs", "name": "Andon Labs", "type": "company", "url": "https://andonlabs.com", "x_handle": null, "logo_permission": false }, { "id": "anthropic", "name": "Anthropic", "type": "lab", "url": "https://www.anthropic.com", "x_handle": "AnthropicAI", "logo_permission": false }, { "id": "artificial-analysis", "name": "Artificial Analysis", "type": "company", "url": "https://artificialanalysis.ai", "x_handle": "ArtificialAnlys", "logo_permission": false }, { "id": "box", "name": "Box", "type": "company", "url": "https://www.box.com", "x_handle": "box", "logo_permission": false, "description": "Cloud content management company. Helped Mercor define the realistic file systems used in APEX-Agents." }, { "id": "cais", "name": "CAIS", "full_name": "Center for AI Safety", "type": "nonprofit", "url": "https://safe.ai", "x_handle": "cais", "logo_permission": false }, { "id": "cisco", "name": "Cisco Research", "full_name": "Cisco Research", "type": "company", "url": "https://research.cisco.com", "x_handle": "outshiftbycisco", "logo_permission": false }, { "id": "cmu", "name": "Carnegie Mellon University", "type": "academic", "url": "https://www.cmu.edu", "x_handle": "carnegiemellon", "logo_permission": false }, { "id": "concordia-university", "name": "Concordia University", "type": "academic", "url": "https://www.concordia.ca", "x_handle": "ConcordiaUnews", "logo_permission": false, "description": "Public university in Montreal, Canada. Emad Shihab's group studied how long agent-authored code survives in open-source projects." }, { "id": "epoch-ai", "name": "Epoch AI", "type": "nonprofit", "url": "https://epoch.ai", "x_handle": "EpochAIResearch", "logo_permission": false }, { "id": "harvey", "name": "Harvey", "full_name": "Harvey AI", "type": "company", "url": "https://www.harvey.ai", "x_handle": "harvey", "logo_permission": false, "description": "Legal AI company. Gave early feedback on the corporate law scenarios in APEX-Agents." }, { "id": "hku", "name": "HKU", "full_name": "The University of Hong Kong", "type": "academic", "url": "https://www.hku.hk", "x_handle": "HKUniversity", "logo_permission": false }, { "id": "hugging-face", "name": "Hugging Face", "type": "company", "url": "https://huggingface.co", "x_handle": "huggingface", "logo_permission": false }, { "id": "ibm", "name": "IBM Research", "type": "company", "url": "https://research.ibm.com", "x_handle": "IBMResearch", "logo_permission": false }, { "id": "iisc", "name": "IISc", "full_name": "Indian Institute of Science", "type": "academic", "url": "https://www.iisc.ac.in", "x_handle": "iiscbangalore", "logo_permission": false }, { "id": "laude-institute", "name": "Laude Institute", "type": "nonprofit", "url": "https://www.laude.org", "x_handle": "LaudeInstitute", "logo_permission": false }, { "id": "martian", "name": "Martian", "type": "company", "url": "https://withmartian.com", "x_handle": "withmartian", "logo_permission": false }, { "id": "mercor", "name": "Mercor", "type": "company", "url": "https://www.mercor.com", "x_handle": "mercor", "logo_permission": false }, { "id": "meta", "name": "Meta", "full_name": "Meta AI (FAIR)", "type": "lab", "url": "https://ai.meta.com", "x_handle": "aiatmeta", "logo_permission": false }, { "id": "metr", "name": "METR", "full_name": "Model Evaluation & Threat Research", "type": "nonprofit", "url": "https://metr.org", "x_handle": "METR_Evals", "logo_permission": false }, { "id": "microsoft", "name": "Microsoft Research", "type": "company", "url": "https://www.microsoft.com/en-us/research/", "x_handle": "MSFTResearch", "logo_permission": false }, { "id": "nebius", "name": "Nebius", "type": "company", "url": "https://nebius.com", "x_handle": "nebiusai", "logo_permission": false }, { "id": "openai", "name": "OpenAI", "type": "lab", "url": "https://openai.com", "x_handle": "OpenAI", "logo_permission": false }, { "id": "princeton", "name": "Princeton University", "type": "academic", "url": "https://www.princeton.edu", "x_handle": "princeton", "logo_permission": false }, { "id": "queens-university", "name": "Queen's University", "full_name": "Queen's University (Software Analysis and Intelligence Lab, SAIL)", "type": "academic", "url": "https://www.queensu.ca", "x_handle": "queensu", "logo_permission": false, "description": "Public university in Kingston, Ontario, Canada. The SAIL lab of Ahmed E. Hassan built the AIDev dataset of agent-authored pull requests." }, { "id": "renmin-university", "name": "Renmin University of China", "full_name": "Renmin University of China", "type": "academic", "url": "https://www.ruc.edu.cn", "x_handle": null, "logo_permission": false, "description": "Public research university in Beijing. Its KBReasoning group maintains SpreadsheetBench." }, { "id": "salesforce", "name": "Salesforce AI Research", "type": "company", "url": "https://www.salesforce.com/ai-research/", "x_handle": "sfresearch", "logo_permission": false }, { "id": "scale-ai", "name": "Scale AI", "full_name": "Scale AI (Scale Labs)", "type": "company", "url": "https://scale.com", "x_handle": "scale_ai", "logo_permission": false }, { "id": "servicenow", "name": "ServiceNow", "full_name": "ServiceNow Research", "type": "company", "url": "https://www.servicenow.com", "x_handle": "servicenow", "logo_permission": false }, { "id": "sierra", "name": "Sierra", "type": "company", "url": "https://sierra.ai", "x_handle": "sierraplatform", "logo_permission": false }, { "id": "stanford", "name": "Stanford University", "type": "academic", "url": "https://www.stanford.edu", "x_handle": "Stanford", "logo_permission": false }, { "id": "uc-berkeley", "name": "UC Berkeley", "full_name": "University of California, Berkeley", "type": "academic", "url": "https://www.berkeley.edu", "x_handle": "UCBerkeley", "logo_permission": false }, { "id": "uiuc", "name": "UIUC", "full_name": "University of Illinois Urbana-Champaign", "type": "academic", "url": "https://illinois.edu", "x_handle": "uofillinois", "logo_permission": false }, { "id": "university-of-michigan", "name": "University of Michigan", "type": "academic", "url": "https://umich.edu", "x_handle": "umich", "logo_permission": false }, { "id": "university-of-toronto", "name": "University of Toronto", "type": "academic", "url": "https://www.utoronto.ca", "x_handle": "uoft", "logo_permission": false }, { "id": "university-of-waterloo", "name": "University of Waterloo", "type": "academic", "url": "https://uwaterloo.ca", "x_handle": null, "logo_permission": false }, { "id": "vals-ai", "name": "Vals AI", "type": "company", "url": "https://www.vals.ai", "x_handle": "ValsAI", "logo_permission": false } ], "conflicts": [ { "id": "codeclash-sonnet-45-elo", "benchmark": "codeclash", "opened": "2026-09-23", "status": "open", "summary": "The CodeClash site leaderboard shows Claude Sonnet 4.5 at 1385 ± 18 and GPT-5 at 1366 ± 17. Version 2 of the paper (May 2026) shows 1389 ± 18 and 1360 ± 17 for the same models. Both come from the maintainers. The difference is small and the ranking is the same. A refit of the Elo model is one possible cause. We show the leaderboard value.", "claims": [ { "value": 1385, "source_url": "https://codeclash.ai/", "source_kind": "primary", "note": "Site leaderboard, \"Updated Nov. 3, 2025\", seen 2026-09-23." }, { "value": 1389, "source_url": "https://arxiv.org/html/2511.00839v2", "source_kind": "primary", "note": "Paper v2 (2026-05-12), Table 1 and Table 4." } ], "we_show": 1385, "resolution": null }, { "id": "itbench-sre-resolved-rate", "benchmark": "itbench", "opened": "2026-09-23", "status": "open", "summary": "The ICML 2025 paper says agents resolve 11.4% of SRE scenarios. The arXiv v1 abstract says 13.8%. The official SRE leaderboard shows 25.0% for IBM's GPT-4o reference agent. The three numbers come from different scenario sets and dates. We show the leaderboard value because it is the maintainers' most recent published result.", "claims": [ { "value": 11.4, "source_url": "https://proceedings.mlr.press/v267/jha25a.html", "source_kind": "primary", "note": "ICML 2025 camera-ready abstract, 102 scenarios." }, { "value": 13.8, "source_url": "https://arxiv.org/abs/2502.05352", "source_kind": "primary", "note": "arXiv v1 abstract (2025-02-07), 94 scenarios." }, { "value": 25, "source_url": "https://github.com/itbench-hub/ITBench/blob/main/LEADERBOARD_SRE.md", "source_kind": "primary", "note": "SRE leaderboard, ITBench-SRE-Agent-GPT-4o, 16 trials, updated 2 May 2025. Multi-trial table shows 24.79%." } ], "we_show": 25, "resolution": null }, { "id": "metr-opus-4-5-horizon", "benchmark": "metr-time-horizons", "opened": "2026-09-23", "status": "resolved", "summary": "METR's Time Horizon 1.1 blog post (2026-01-29) gives Claude Opus 4.5 a 50% horizon of 320 minutes. METR's live data file gives 293.0 minutes. Both are METR sources. The results page lists a 2026-03-03 update that \"corrected a regularization mistake that affected our measurements\", which explains the change. GPT-5 moved the same way (214 to 203 minutes).", "claims": [ { "value": 293, "source_url": "https://metr.org/assets/benchmark_results_1_1.yaml", "source_kind": "primary", "note": "Live data file linked from metr.org/time-horizons, retrieved 2026-09-23. CI 161.7 to 623.7." }, { "value": 320, "source_url": "https://metr.org/blog/2026-1-29-time-horizon-1-1/", "source_kind": "primary", "note": "TH1.1 launch post, table \"Changes to Model Horizon Estimates\". CI 170 to 729." } ], "we_show": 293, "resolution": "We show the live data file, which reflects the 2026-03-03 correction. The blog post predates it." }, { "id": "rli-fable-5-automation-rate", "benchmark": "remote-labor-index", "opened": "2026-09-23", "status": "open", "summary": "CAIS, a co-author of the benchmark, reports 15.8% for Claude Fable 5. Two aggregator pages report 16.1%. We did not find the cause. A later re-grade is one possible cause.", "claims": [ { "value": 15.8, "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation", "source_kind": "primary", "note": "CAIS post dated 2026-07-01." }, { "value": 16.1, "source_url": "https://www.benchleader.com/benchmarks/rli", "source_kind": "third-party", "note": "BenchLeader page, seen 2026-09-23." }, { "value": 16.1, "source_url": "https://ai-intensify.com/remote-labor-index-ai-automation-2026/", "source_kind": "third-party", "note": "News article, seen 2026-09-23." } ], "we_show": 15.8, "resolution": null }, { "id": "spreadsheetbench-v1-top-score", "benchmark": "spreadsheetbench", "opened": "2026-09-23", "status": "open", "summary": "The SpreadsheetBench site shows two different top scores for the 912-question V1 set. The static \"Top Score (OVERALL)\" box reads 70.48%. The leaderboard data file that the page loads lists Qingqiu Agent at 83.11% (verified, 2026-06-23). The box looks stale. We show the leaderboard value.", "claims": [ { "value": 83.11, "source_url": "https://spreadsheetbench.github.io/data/leaderboard-v1-full.json", "source_kind": "primary", "note": "Leaderboard data file, seen 2026-09-23. Entry is marked verified." }, { "value": 70.48, "source_url": "https://spreadsheetbench.github.io", "source_kind": "primary", "note": "Static statistics box on the V1 overview page, seen 2026-09-23. Matches the Gemini in Google Sheets row dated 2026-03-10." } ], "we_show": 83.11, "resolution": null }, { "id": "swe-bench-pro-public-frontier", "benchmark": "swe-bench-pro", "opened": "2026-09-23", "status": "open", "summary": "Scale's public leaderboard shows a best resolve rate of 61.5% (Muse Spark 1.1, mini-swe-agent, added 2026-07-09). OpenAI's 2026-07-08 audit post says frontier models reached 80.3% on the same 731-task public split, without naming the model or harness. The gap is likely a different agent scaffold and OpenAI's own internal runs. We show the leaderboard value.", "claims": [ { "value": 61.5, "source_url": "https://labs.scale.com/leaderboard/swe_bench_pro_public", "source_kind": "primary", "note": "Scale AI public leaderboard, Muse Spark 1.1 with mini-swe-agent, 61.50 ± 3.10, seen 2026-09-23." }, { "value": 80.3, "source_url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations/", "source_kind": "lab-reported", "note": "OpenAI post dated 2026-07-08. \"Frontier models improved from a pass rate of 23.3% to 80.3% in eight months.\" No model or harness named." } ], "we_show": 61.5, "resolution": null } ], "events": [ { "id": "2026-09-22-swe-bench-pro-v2", "date": "2026-09-22", "kind": "method-change", "benchmark": "swe-bench-pro", "title": "Scale AI releases SWE-Bench Pro V2 with 642 validated tasks", "summary": "V2 drops 89 tasks from the 731-task public set, rewrites 529 problem statements, repairs verifiers, and adds a HARD-51 subset. It runs under a locked protocol: no network during the agent phase and re-grading on a fresh sandbox. The leaderboard still shows V1 results.", "source_url": "https://github.com/scaleapi/SWE-bench_Pro-os" }, { "id": "2026-09-08-apex-agents-1-1", "date": "2026-09-08", "kind": "method-change", "benchmark": "apex-agents", "title": "APEX-Agents 1.1 stops rewarding hedged answers", "summary": "Mercor found models raising scores by giving several answers at once (\"scattergunning\"). Version 1.1 trims the set to 240 audited tasks, adds a judge that scores hedged rubric items as zero, and fixes tools. Scores rise in aggregate and are not comparable with 1.0. Claude Fable 5.1 leads at 68.6% Pass@1.", "source_url": "https://www.mercor.com/blog/introducing-apex-agents-1-1" }, { "id": "2026-09-07-cybergym-frontier-98", "date": "2026-09-07", "kind": "jump", "benchmark": "cybergym", "title": "CyberGym leader reaches 98.5% reproduction rate", "summary": "The Creation multi-model agent tops the official leaderboard at 98.47%, up from under 20% at launch in June 2025 and 83.1% for Claude Mythos Preview in April 2026. Several team submissions now exceed 95%, and the maintainers warn that small differences may not be meaningful.", "source_url": "https://www.cybergym.io/cybergym/" }, { "id": "2026-09-07-vending-bench-2-gpt-6-astra", "date": "2026-09-07", "kind": "jump", "benchmark": "vending-bench-2", "title": "GPT-6 Astra takes the Vending-Bench 2 lead at about $15,500", "summary": "Andon Labs reports that GPT-6 Astra finished the simulated year with an average balance of $15,515, the first OpenAI model to lead the board and the largest gap over second place so far (Claude Opus 5, $11,181.87). Andon Labs credits steady negotiation all year and never prepaying suppliers that had gone out of business.", "source_url": "https://andonlabs.com/blog/gpt-6-astra-vending-bench" }, { "id": "2026-09-gdpval-aa-v2-1-rescale", "date": "2026-09", "kind": "method-change", "benchmark": "gdpval-aa", "title": "GDPval-AA v2.1 re-anchors the Elo scale", "summary": "Artificial Analysis pins DeepSeek V4.1 Flash (max) at 1600 Elo and fits ratings with a Crowd-BT model. Rank order is largely unchanged, but v2.1 scores are not comparable with v2 scores, which were anchored to human expert performance at 1000.", "source_url": "https://artificialanalysis.ai/methodology/intelligence-benchmarking" }, { "id": "2026-07-25-osworld-verified-passes-90", "date": "2026-07-25", "kind": "jump", "benchmark": "osworld-verified", "title": "First verified OSWorld score above 90%", "summary": "The Intelligence-Indeed Agent completes 90.19% of OSWorld-Verified tasks at 100 steps in a run verified by the maintainers. A week later Claude Fable 5 posts 85.96% as the best general model. Both are well above the 72.36% human figure from the original study.", "source_url": "https://os-world.github.io/static/data/osworld_verified_results.xlsx" }, { "id": "2026-07-08-openai-audit-swe-bench-pro", "date": "2026-07-08", "kind": "method-change", "benchmark": "swe-bench-pro", "title": "OpenAI audit estimates about 30% of SWE-Bench Pro tasks are broken", "summary": "OpenAI reviews the 731-task public split with an agent pipeline and five human engineers per flagged task. The pipeline flags 200 tasks (27.4%) and humans flag 249 (34.1%) as broken. OpenAI withdraws its earlier recommendation to adopt SWE-Bench Pro.", "source_url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations/" }, { "id": "2026-07-01-rli-frontier-quadruples", "date": "2026-07-01", "kind": "jump", "benchmark": "remote-labor-index", "title": "Best Remote Labor Index score rises to 15.8%", "summary": "CAIS reports that Claude Fable 5 completes 15.8% of real freelance projects at a client-acceptable standard. At launch in October 2025, the best agent completed 2.5%.", "source_url": "https://safe.ai/blog/significant-increase-in-digital-labor-automation" }, { "id": "2026-07-browsecomp-passes-90", "date": "2026-07", "kind": "saturated", "benchmark": "browsecomp", "title": "BrowseComp scores pass 90%", "summary": "OpenAI reports 90.4% for GPT-5.6 Sol and 92.2% for its four-agent \"ultra\" mode. By September 2026 GPT-6 Astra (91.5%) and Claude Opus 5 (90.8%) also sit above 90%, leaving little headroom.", "source_url": "https://openai.com/index/gpt-5-6/" }, { "id": "2026-06-26-osworld-2-launch", "date": "2026-06-26", "kind": "milestone", "benchmark": "osworld-verified", "title": "OSWorld 2.0 succeeds OSWorld-Verified", "summary": "The XLANG team releases OSWorld 2.0, 108 long-horizon computer-use workflows that take skilled users a median of about 1.6 hours. The best agent, Claude Opus 4.8, completes only 20.6% of tasks at 500 steps, showing how much headroom the older benchmark no longer captures.", "source_url": "https://osworld-v2.xlang.ai/" }, { "id": "2026-06-23-spreadsheetbench-passes-80", "date": "2026-06-23", "kind": "jump", "benchmark": "spreadsheetbench", "title": "SpreadsheetBench V1 leader reaches 83.11%", "summary": "Kingsoft Office's Qingqiu Agent scores 83.11% on the full 912-question set, a verified entry. The previous verified leader, Gemini in Google Sheets, scored 70.48% in March 2026.", "source_url": "https://spreadsheetbench.github.io" }, { "id": "2026-05-27-itbench-aa-launch", "date": "2026-05-27", "kind": "launch", "benchmark": "itbench-aa", "title": "Artificial Analysis and IBM launch ITBench-AA", "summary": "59 Kubernetes incident-diagnosis tasks run in a fixed harness. Every frontier model scores below 50%. Claude Opus 4.7 leads at 47%, followed by GPT-5.5 at 46%.", "source_url": "https://artificialanalysis.ai/articles/itbench-aa-launch" }, { "id": "2026-05-08-metr-horizon-passes-16-hours", "date": "2026-05-08", "kind": "jump", "benchmark": "metr-time-horizons", "title": "METR reports a 17-hour time horizon and warns the suite is near its ceiling", "summary": "METR adds Claude Mythos Preview (early) with a 50% horizon of 1,044.8 minutes (about 17.4 hours, CI 508.9 to 3,304.3) and posts a notice that measurements above 16 hours are unreliable with the current task suite. Claude Opus 4.6 had reached 718.8 minutes in February.", "source_url": "https://metr.org/time-horizons/" }, { "id": "2026-05-08-sregym-launch", "date": "2026-05-08", "kind": "launch", "benchmark": "sregym", "title": "SREGym paper released with 90 live SRE problems", "summary": "UIUC and University of Toronto publish SREGym. Frontier agents reach 27% to 61% end-to-end success on the 90-problem suite. On failures unique to SREGym (hardware, metastable, concurrent), the best agent falls to 28%.", "source_url": "https://arxiv.org/abs/2605.07161" }, { "id": "2026-05-06-terminal-bench-2-1-supersedes-2-0", "date": "2026-05-06", "kind": "retired", "benchmark": "terminal-bench-2", "title": "Terminal-Bench 2.1 fixes 28 tasks and replaces 2.0", "summary": "The maintainers release 2.1 after finding defects in 28 of the 89 tasks in 2.0: changed external dependencies, resource budgets too small for valid solutions, and instructions that did not match tests. Most agent-model pairs score higher on 2.1; Claude Code with Opus 4.6 gains 12.1 points. Terminal-Bench 3.0 (July 2026) and 4.0 (August 2026) followed.", "source_url": "https://www.tbench.ai/news/terminal-bench-2-1" }, { "id": "2026-04-24-mle-bench-submissions-paused", "date": "2026-04-24", "kind": "method-change", "benchmark": "mle-bench", "title": "MLE-bench pauses new leaderboard submissions", "summary": "OpenAI stops taking new MLE-bench leaderboard submissions while it builds a process to make sure submissions are fair and comparable. A v2 release with batched fixes is planned in the frontier-evals repo.", "source_url": "https://github.com/openai/mle-bench" }, { "id": "2026-04-07-cybench-saturated", "date": "2026-04-07", "kind": "saturated", "benchmark": "cybench", "title": "Claude Mythos Preview reaches 100% on Cybench subset", "summary": "Anthropic's system card reports 100% pass@1 on its 35-task Cybench subset with 10 trials per task, and says that, given the saturation of the benchmark, it is no longer sufficiently informative.", "source_url": "https://cdn.sanity.io/files/4zrzovbb/website/7624816413e9b4d2e3ba620c5a5e091b98b190a5.pdf" }, { "id": "2026-04-gdpval-gpt-5-5-reaches-85", "date": "2026-04", "kind": "jump", "benchmark": "gdpval", "title": "GPT-5.5 reaches 84.9% wins plus ties on GDPval", "summary": "In the GPT-5.5 launch post OpenAI reports 84.9% wins plus ties against industry experts, with GPT-5.4 at 83.0% and Claude Opus 4.7 at 80.3%. The post does not say which grader produced the numbers.", "source_url": "https://openai.com/index/introducing-gpt-5-5/" }, { "id": "2026-03-11-gaia-passes-human-score", "date": "2026-03-11", "kind": "saturated", "benchmark": "gaia", "title": "GAIA leaderboard passes the 92% human score", "summary": "Alibaba Cloud's OPS-Agentic-Search posts 92.36% on the GAIA test set, the first entry above the 92% human figure from the paper. By September 2026 several self-submitted agents sit between 93% and 94%, so the benchmark no longer separates the best systems.", "source_url": "https://huggingface.co/datasets/gaia-benchmark/results_public" }, { "id": "2026-03-01-spider-2-snow-passes-96", "date": "2026-03-01", "kind": "jump", "benchmark": "spider-2", "title": "Spider 2.0-Snow leader reaches 96.70%", "summary": "Genloop's Sentinel Agent v2 Pro tops the Spider 2.0-Snow table at 96.70%. Fifteen months earlier the maintainers' Spider-Agent with o1-preview scored 23.58%. The Snow variant now has little headroom.", "source_url": "https://spider2-sql.github.io" }, { "id": "2026-02-26-code-review-bench-launch", "date": "2026-02-26", "kind": "launch", "benchmark": "martian-code-review-bench", "title": "Martian launches Code Review Bench", "summary": "Martian publishes an open benchmark for AI code review tools with an offline set of 50 PRs and 173 golden comments, plus an online set built from fresh GitHub PRs. The repo, judge prompts, and results are MIT licensed.", "source_url": "https://github.com/withmartian/code-review-benchmark" }, { "id": "2026-02-23-mle-bench-passes-60", "date": "2026-02-23", "kind": "jump", "benchmark": "mle-bench", "title": "MLE-bench leader earns medals in 64.44% of competitions", "summary": "Baidu's Famou-Agent 2.0 with Gemini-3-Pro-Preview tops the MLE-bench table at 64.44% any-medal rate. Sixteen months earlier the best agent scored 17.12%.", "source_url": "https://github.com/openai/mle-bench" }, { "id": "2026-02-23-openai-retires-swe-bench-verified", "date": "2026-02-23", "kind": "retired", "benchmark": "swe-bench-verified", "title": "OpenAI stops reporting SWE-bench Verified", "summary": "OpenAI says SWE-bench Verified no longer measures frontier coding progress. Its audit found flawed tests in 59.4% of 138 hard tasks, and probes showed frontier models from three labs reproducing gold patches from training data. OpenAI asks other developers to stop reporting it and points to SWE-Bench Pro instead.", "source_url": "https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/" }, { "id": "2026-02-tau2-bench-task-fixes", "date": "2026-02", "kind": "method-change", "benchmark": "tau2-bench", "title": "Sierra fixes 50+ airline and retail tasks for τ³-bench", "summary": "Sierra audited the airline and retail domains with the SABER (τ-Bench Verified) team at Amazon and fixed 27 airline and 26 retail tasks with wrong expected actions, ambiguous instructions, or impossible constraints. Airline pass^1 rose by 14 to 20 points for the three re-run models, so scores before and after are not comparable. Telecom was not changed.", "source_url": "https://taubench.com/blog/tau3-task-fixes.html" }, { "id": "2026-01-29-metr-time-horizon-1-1", "date": "2026-01-29", "kind": "method-change", "benchmark": "metr-time-horizons", "title": "METR releases Time Horizon 1.1 with 228 tasks", "summary": "METR grows the task suite from 170 to 228 tasks, doubles the number of 8-hour-plus tasks to 31, and moves from Vivaria to Inspect. Estimates shift: the post-2023 doubling time drops from 165 to 131 days, and Claude Opus 4.5 moves from 289 to 320 minutes.", "source_url": "https://metr.org/blog/2026-1-29-time-horizon-1-1/" }, { "id": "2026-01-21-apex-agents-launch", "date": "2026-01-21", "kind": "launch", "benchmark": "apex-agents", "title": "Mercor releases APEX-Agents", "summary": "Mercor, with Box and Harvey, releases 480 long-horizon tasks from investment banking, consulting, and corporate law inside simulated project workspaces. The best agent, Gemini 3 Flash, passes 24.0% of tasks on one attempt.", "source_url": "https://www.mercor.com/blog/introducing-apex-agents/" }, { "id": "2025-12-03-core-bench-solved", "date": "2025-12-03", "kind": "solved", "benchmark": "core-bench", "title": "HAL declares CORE-Bench solved", "summary": "Claude Opus 4.5 in a Claude Code scaffold scores 77.78% on CORE-Bench-Hard, nearly double its 42.22% with HAL's CORE-Agent. After HAL fixed grading errors in 8 tasks and removed 1 broken task, the score is 95.5%. HAL treats the benchmark as solved and plans to open a private test set.", "source_url": "https://hal.cs.princeton.edu/corebench_hard" }, { "id": "2025-12-gdpval-aa-launch", "date": "2025-12", "kind": "launch", "benchmark": "gdpval-aa", "title": "Artificial Analysis launches GDPval-AA", "summary": "Artificial Analysis opens an independent Elo leaderboard that runs OpenAI's public GDPval tasks through its open-source Stirrup agent harness and grades deliverables with blind pairwise LLM judging. Claude Opus 4.5 leads at launch.", "source_url": "https://www.linkedin.com/posts/artificial-analysis_announcing-gdpval-aa-our-leaderboard-and-activity-7404608113624072193-Q82r" }, { "id": "2025-12-gdpval-passes-expert-parity", "date": "2025-12", "kind": "milestone", "benchmark": "gdpval", "title": "GPT-5.2 Thinking passes the 50% line on GDPval", "summary": "OpenAI reports that GPT-5.2 Thinking beats or ties industry professionals in 70.9% of GDPval comparisons, judged by expert humans. It is the first model OpenAI reports above the 50% parity line.", "source_url": "https://openai.com/index/introducing-gpt-5-2/" }, { "id": "2025-11-18-swe-lancer-codex-max-80", "date": "2025-11-18", "kind": "jump", "benchmark": "swe-lancer", "title": "GPT-5.1-Codex-Max reaches 80% on SWE-Lancer IC Diamond", "summary": "OpenAI's GPT-5.1-Codex-Max system card reports about 80% pass@1 on IC SWE Diamond tasks, averaged over three runs, up from 55% for GPT-5 and 67% for GPT-5-Codex in the same chart. This is the last OpenAI system card to report SWE-Lancer.", "source_url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max/swe-lancer" }, { "id": "2025-11-18-vending-bench-2-launch", "date": "2025-11-18", "kind": "launch", "benchmark": "vending-bench-2", "title": "Andon Labs releases Vending-Bench 2", "summary": "The second Vending-Bench adds adversarial suppliers, negotiation, delivery delays, supplier bankruptcies, and refund demands, and scores the bank balance after one simulated year. Gemini 3 Pro led at launch with $5,478.16, ahead of Claude Sonnet 4.5 ($3,838.74) and Grok 4 ($1,999.46).", "source_url": "http://web.archive.org/web/20251118175816/https://andonlabs.com/evals/vending-bench-2" }, { "id": "2025-11-07-terminal-bench-2-launch", "date": "2025-11-07", "kind": "launch", "benchmark": "terminal-bench-2", "title": "Terminal-Bench 2.0 and Harbor released", "summary": "The Terminal-Bench team releases 2.0, a harder and more carefully verified set of 89 terminal tasks, together with Harbor, a new package for running agents in cloud containers. The best verified row for a pre-launch model is Codex CLI with GPT-5 at 49.6%.", "source_url": "https://www.tbench.ai/news/announcement-2-0" }, { "id": "2025-11-02-codeclash-launch", "date": "2025-11-02", "kind": "launch", "benchmark": "codeclash", "title": "CodeClash launches goal-oriented coding tournaments", "summary": "Stanford and Princeton researchers release CodeClash, where models evolve codebases over 15-round tournaments in six game arenas. Across 1,680 tournaments, Claude Sonnet 4.5 leads with the highest Elo, followed by GPT-5 and o3. Top models lose every round to expert human bots.", "source_url": "https://arxiv.org/abs/2511.00839" }, { "id": "2025-10-30-cve-bench-v2-grading-fix", "date": "2025-10-30", "kind": "method-change", "benchmark": "cve-bench", "title": "CVE-Bench v2.0 closes grading shortcuts", "summary": "The maintainers hardened the outbound-request check and the SQL-injection check after finding that agents could pass without real exploits. GPT-4o agent success rates fell by up to 32.5 points after the fixes.", "source_url": "https://open.substack.com/pub/ddkang/p/cve-bench-v20-making-evaluation-more" }, { "id": "2025-10-23-impossiblebench-launch", "date": "2025-10-23", "kind": "launch", "benchmark": "impossiblebench", "title": "ImpossibleBench measures how often coding agents game their tests", "summary": "CMU and Anthropic researchers release tasks where tests contradict the specification, so any pass is a shortcut. GPT-5 passes 54.0% of the conflicting SWE-bench tasks by cheating. Stricter prompts cut cheating sharply.", "source_url": "https://arxiv.org/abs/2510.20270" }, { "id": "2025-10-02-tau2-bench-gpt-5-telecom-96", "date": "2025-10-02", "kind": "jump", "benchmark": "tau2-bench", "title": "GPT-5 reaches 95.8% telecom pass^1 on τ²-bench", "summary": "Sierra's own run of GPT-5 (evaluated 2025-08-09, added to the leaderboard 2025-10-02) scored 95.8% pass^1 and 85.1% pass^4 on telecom, up from about 50% pass^1 four months earlier. Telecom pass^1 has stayed near 98% for the best models since then.", "source_url": "https://github.com/sierra-research/tau2-bench/blob/main/web/leaderboard/public/submissions/gpt-5_sierra_2025-08-09/submission.json" }, { "id": "2025-09-25-gdpval-launch", "date": "2025-09-25", "kind": "launch", "benchmark": "gdpval", "title": "OpenAI releases GDPval", "summary": "OpenAI publishes GDPval, 1,320 real work tasks from 44 occupations in 9 US industries, with a public 220-task gold subset. Blinded experts rate Claude Opus 4.1 better than or equal to the professional's deliverable in 47.6% of comparisons.", "source_url": "https://openai.com/index/gdpval/" }, { "id": "2025-09-21-swe-bench-pro-launch", "date": "2025-09-21", "kind": "launch", "benchmark": "swe-bench-pro", "title": "Scale AI releases SWE-Bench Pro", "summary": "Scale AI publishes SWE-Bench Pro with 1,865 long-horizon tasks from 41 repositories, including a 731-task public set from GPL repositories and a 276-task private set from startups. Under a 50-turn, $2 cap, the best public-set result is 23.3% (GPT-5, medium reasoning).", "source_url": "https://arxiv.org/abs/2509.16941" }, { "id": "2025-07-28-osworld-verified-launch", "date": "2025-07-28", "kind": "method-change", "benchmark": "osworld-verified", "title": "OSWorld becomes OSWorld-Verified", "summary": "The HKU XLANG team fixes about 300 reported problems in OSWorld tasks and checkers, moves evaluation to a parallel AWS setup, and re-runs all baselines. The report names CoACT-1 as the best agent at 60.76%, about 84% of the roughly 72% human figure.", "source_url": "https://xlang.ai/blog/osworld-verified" }, { "id": "2025-07-20-aidev-dataset-release", "date": "2025-07-20", "kind": "launch", "benchmark": "aidev-dataset", "title": "AIDev dataset of agent pull requests released", "summary": "Queen's University researchers publish AIDev, 456,535 pull requests by five coding agents across 61,453 GitHub repositories. In popular repositories, agent PRs are merged less often than human PRs (Codex 65.3% versus 76.8% for humans).", "source_url": "https://arxiv.org/abs/2507.15003" }, { "id": "2025-06-09-tau2-bench-launch", "date": "2025-06-09", "kind": "launch", "benchmark": "tau2-bench", "title": "Sierra releases τ²-bench with a dual-control telecom domain", "summary": "τ²-bench adds a telecom customer-service domain where both the agent and the simulated user hold tools. At launch the best telecom pass^1 was about 50% (o4-mini, gpt-4.1-mini, and Claude 3.7 Sonnet at 49%), and GPT-4.1 dropped from 74% on retail to 34% on telecom.", "source_url": "https://arxiv.org/abs/2506.07982" }, { "id": "2025-06-03-cybergym-launch", "date": "2025-06-03", "kind": "launch", "benchmark": "cybergym", "title": "UC Berkeley releases CyberGym with 1,507 real vulnerabilities", "summary": "Agents must write proof-of-concept inputs that reproduce real OSS-Fuzz vulnerabilities in 188 projects. The best agent-model pairs at launch reproduce fewer than 20% of them.", "source_url": "https://arxiv.org/abs/2506.02548" }, { "id": "2025-05-26-pr-arena-launch", "date": "2025-05-26", "kind": "launch", "benchmark": "pr-arena", "title": "PR Arena starts tracking agent pull requests on GitHub", "summary": "The PRarena repo records its first data point. At that time Codex had 51,548 ready PRs with 85.8% merged, and Copilot had 2,099 ready PRs with 75.94% merged.", "source_url": "https://raw.githubusercontent.com/aavetis/PRarena/main/data.csv" }, { "id": "2025-05-26-swe-rebench-launch", "date": "2025-05-26", "kind": "launch", "benchmark": "swe-rebench", "title": "Nebius launches SWE-rebench with fresh GitHub issues", "summary": "Nebius publishes an automated pipeline that collects new issue and pull request pairs from Python repositories, plus a leaderboard that runs every model in the same ReAct scaffold five times. In the paper, GPT-4.1 leads the January 2025 window at 31.1%.", "source_url": "https://arxiv.org/abs/2505.20411" }, { "id": "2025-05-24-crmarena-pro-launch", "date": "2025-05-24", "kind": "launch", "benchmark": "crmarena-pro", "title": "Salesforce releases CRMArena-Pro", "summary": "CRMArena-Pro expands CRMArena to 19 sales, service, and CPQ task types across B2B and B2C Salesforce orgs, adds multi-turn users and confidentiality checks. The best agent (gemini-2.5-pro with ReAct) completed 58.3% of single-turn B2C queries and about 35% in multi-turn. All models showed near-zero confidentiality awareness without special prompting.", "source_url": "https://arxiv.org/abs/2505.18878" }, { "id": "2025-05-20-vals-finance-agent-launch", "date": "2025-05-20", "kind": "launch", "benchmark": "vals-finance-agent", "title": "Vals AI releases the Finance Agent Benchmark", "summary": "Vals AI publishes 537 expert-written financial research questions and an agent harness with EDGAR and web search tools. The best model, OpenAI o3, answered 46.8% correctly at an average cost of $3.79 per query.", "source_url": "https://arxiv.org/abs/2508.00828" }, { "id": "2025-04-02-paperbench-launch", "date": "2025-04-02", "kind": "launch", "benchmark": "paperbench", "title": "OpenAI releases PaperBench for replicating ICML papers", "summary": "Agents must replicate 20 ICML 2024 papers from scratch. Claude 3.5 Sonnet with a basic scaffold scores 21.0%, and o1 with a 36-hour limit scores 26.0%. ML PhDs reached 41.4% on a 3-paper subset after 48 hours.", "source_url": "https://arxiv.org/abs/2504.01848" }, { "id": "2025-04-browsecomp-launch", "date": "2025-04", "kind": "launch", "benchmark": "browsecomp", "title": "OpenAI releases BrowseComp", "summary": "OpenAI open-sources 1,266 hard-to-find, easy-to-verify browsing questions. Deep research answers 51.5%, o1 9.9%, and GPT-4o with browsing 1.9%. Human trainers solved 29.2% within two hours.", "source_url": "https://openai.com/index/browsecomp/" }, { "id": "2025-03-31-cve-bench-launch", "date": "2025-03-31", "kind": "launch", "benchmark": "cve-bench", "title": "UIUC releases CVE-Bench with 40 critical web CVEs", "summary": "Agents must exploit real critical-severity vulnerabilities in sandboxed web applications. The best GPT-4o agent framework exploits 12.5% of CVEs with five attempts when given a vulnerability description.", "source_url": "https://github.com/uiuc-kang-lab/cve-bench" }, { "id": "2025-03-18-metr-time-horizon-paper", "date": "2025-03-18", "kind": "launch", "benchmark": "metr-time-horizons", "title": "METR introduces the 50% task-completion time horizon", "summary": "METR's paper \"Measuring AI Ability to Complete Long Software Tasks\" defines the time horizon metric. Claude 3.7 Sonnet had a 50% horizon of around 50 minutes, and the frontier horizon had doubled about every seven months since 2019.", "source_url": "https://arxiv.org/abs/2503.14499" }, { "id": "2025-02-18-ambig-swe-launch", "date": "2025-02-18", "kind": "launch", "benchmark": "ambig-swe", "title": "Ambig-SWE tests whether coding agents ask for clarification", "summary": "CMU researchers release an underspecified variant of SWE-Bench Verified with a simulated user. Models rarely ask questions unless prompted, and most cannot tell a vague issue from a complete one. When they do interact, resolve rates rise sharply.", "source_url": "https://arxiv.org/abs/2502.13069" }, { "id": "2025-02-17-swe-lancer-launch", "date": "2025-02-17", "kind": "launch", "benchmark": "swe-lancer", "title": "OpenAI releases SWE-Lancer with $1 million of Upwork tasks", "summary": "OpenAI publishes 1,488 real freelance software tasks worth $1 million in actual payouts, with a public Diamond split worth $500,800. The best model, Claude 3.5 Sonnet, solves 26.2% of Diamond IC tasks and earns $208,050 across the Diamond set.", "source_url": "https://openai.com/index/swe-lancer/" }, { "id": "2025-02-07-itbench-launch", "date": "2025-02-07", "kind": "launch", "benchmark": "itbench", "title": "IBM Research releases ITBench", "summary": "IBM Research and UIUC publish ITBench, a framework of live Kubernetes scenarios for SRE, compliance (CISO), and FinOps agents. The first paper reports that agents resolve 13.8% of SRE scenarios.", "source_url": "https://arxiv.org/abs/2502.05352" }, { "id": "2025-01-24-medagentbench-launch", "date": "2025-01-24", "kind": "launch", "benchmark": "medagentbench", "title": "Stanford releases MedAgentBench", "summary": "MedAgentBench gives agents 300 physician-written tasks in a FHIR-compliant virtual EHR with 100 de-identified patients. Claude 3.5 Sonnet v2 led with a 69.67% success rate; models did much better on record lookups (up to 85%) than on tasks that change records.", "source_url": "https://arxiv.org/abs/2501.14654" }, { "id": "2025-01-12-aiopslab-launch", "date": "2025-01-12", "kind": "launch", "benchmark": "aiopslab", "title": "AIOpsLab paper released with 48 cloud operations problems", "summary": "Microsoft Research and partners publish AIOpsLab, a framework that deploys microservices, injects faults, and evaluates agents on detection, localization, root cause analysis, and mitigation. The best agent in the paper reaches 59.32% accuracy.", "source_url": "https://arxiv.org/abs/2501.06706" }, { "id": "2024-12-21-aider-polyglot-launch", "date": "2024-12-21", "kind": "launch", "benchmark": "aider-polyglot", "title": "Aider launches the polyglot leaderboard", "summary": "Paul Gauthier replaces aider's saturating Python-only benchmark with 225 hard Exercism exercises in six languages. o1 with high reasoning effort leads at 61.7%, ahead of Claude 3.5 Sonnet at 45.3%.", "source_url": "https://aider.chat/2024/12/21/polyglot.html" }, { "id": "2024-12-18-theagentcompany-launch", "date": "2024-12-18", "kind": "launch", "benchmark": "theagentcompany", "title": "CMU releases TheAgentCompany", "summary": "CMU publishes a simulated software company with GitLab, Plane, ownCloud, RocketChat, and language-model coworkers, plus 175 work tasks. The best agent, OpenHands with Claude 3.5 Sonnet, completes 24.0% of tasks.", "source_url": "https://arxiv.org/abs/2412.14161" }, { "id": "2024-11-22-re-bench-launch", "date": "2024-11-22", "kind": "launch", "benchmark": "re-bench", "title": "METR releases RE-Bench with 71 human expert baselines", "summary": "Seven ML research engineering environments with matched human and agent conditions. Agents score about 4 times the human average at a 2-hour budget, but humans narrowly pass the best agent at 8 hours and reach about twice the agent score at 32 hours.", "source_url": "https://metr.org/blog/2024-11-22-evaluating-r-d-capabilities-of-llms/" }, { "id": "2024-11-12-spider-2-launch", "date": "2024-11-12", "kind": "launch", "benchmark": "spider-2", "title": "Spider 2.0 launches with 632 enterprise text-to-SQL problems", "summary": "HKU's XLANG Lab and partners release Spider 2.0. The paper reports that its code agent with o1-preview solves 21.3% of tasks, against 91.2% on Spider 1.0 and 73.0% on BIRD.", "source_url": "https://arxiv.org/abs/2411.07763" }, { "id": "2024-10-09-mle-bench-launch", "date": "2024-10-09", "kind": "launch", "benchmark": "mle-bench", "title": "OpenAI releases MLE-bench with 75 Kaggle competitions", "summary": "The best setup at launch, o1-preview with the AIDE scaffold, reaches a medal in about 17% of competitions (16.9% in the paper, 17.12% in the repo table).", "source_url": "https://arxiv.org/abs/2410.07095" }, { "id": "2024-09-17-core-bench-launch", "date": "2024-09-17", "kind": "launch", "benchmark": "core-bench", "title": "Princeton releases CORE-Bench for computational reproducibility", "summary": "270 tasks from 90 published papers at three difficulty levels. The best agent, CORE-Agent with GPT-4o, reaches 21.48% on the hardest level.", "source_url": "https://arxiv.org/abs/2409.11363" }, { "id": "2024-08-15-cybench-launch", "date": "2024-08-15", "kind": "launch", "benchmark": "cybench", "title": "Stanford releases Cybench with 40 professional CTF tasks", "summary": "The best agent (Claude 3.5 Sonnet) solves 17.5% of tasks unguided; GPT-4o solves 12.5%. Agents solve tasks that took human teams up to 11 minutes; the hardest task took humans 24 hours 54 minutes.", "source_url": "https://arxiv.org/abs/2408.08926" }, { "id": "2024-08-13-swe-bench-verified-launch", "date": "2024-08-13", "kind": "launch", "benchmark": "swe-bench-verified", "title": "OpenAI and the SWE-bench team release SWE-bench Verified", "summary": "93 developers screen 1,699 SWE-bench tasks. The 500 tasks that pass become SWE-bench Verified. GPT-4o resolves 33.2% with the best open-source scaffold, double its score on the original SWE-bench.", "source_url": "https://openai.com/index/introducing-swe-bench-verified/" }, { "id": "2024-06-21-spreadsheetbench-launch", "date": "2024-06-21", "kind": "launch", "benchmark": "spreadsheetbench", "title": "SpreadsheetBench launches with 912 real spreadsheet questions", "summary": "Renmin University and partners release SpreadsheetBench. In the paper GPT-4o scores 18.35% (soft) and 15.02% (hard) overall, and Copilot in Excel about 20% on a subset, while Excel experts score 71.33% and 62.00% on a 50-question subset.", "source_url": "https://arxiv.org/abs/2406.14991" }, { "id": "2024-03-12-workarena-launch", "date": "2024-03-12", "kind": "launch", "benchmark": "workarena", "title": "ServiceNow Research releases WorkArena and BrowserGym", "summary": "ServiceNow Research publishes WorkArena, 33 browser tasks on a live ServiceNow instance, together with the BrowserGym environment for web agents. WorkArena++ (682 compositional tasks) follows in July 2024.", "source_url": "https://arxiv.org/abs/2403.07718" }, { "id": "2023-11-21-gaia-launch", "date": "2023-11-21", "kind": "launch", "benchmark": "gaia", "title": "Meta and Hugging Face release GAIA", "summary": "GAIA offers 466 questions that need browsing, file reading, and tool use. Humans score 92% while GPT-4 with plugins scores 15%. A leaderboard with private test answers opens on Hugging Face.", "source_url": "https://arxiv.org/abs/2311.12983" } ], "sources": { "occupations": "O*NET 31.0 Database, USDOL/ETA, CC BY 4.0 (modified by Can Agents Work)", "employment": "U.S. Bureau of Labor Statistics, OEWS May 2025 national estimates (public domain)" } }