{"benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test", "description": "SWE-bench Test board on swebench.com lists community agent submissions on the Test evaluation split. Catalog ingests agentic rows from leaderboards.json (`swe_bench_site`).\n\nCompare only within this board \u2014 task coverage and scoring differ from Verified and Lite.", "category": "eval", "split": null, "language": "python", "task_count": null, "homepage_url": "https://www.swebench.com/", "paper_url": "https://arxiv.org/abs/2310.06770", "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/", "is_public_leaderboard": true, "harness_count": 5, "model_count": 7, "score_count": 10, "best_success_rate": 52.62}, "protocol": {"eval_protocol": "SWE-bench Test public submission", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 10}, "rankings": {"benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test", "description": "SWE-bench Test board on swebench.com lists community agent submissions on the Test evaluation split. Catalog ingests agentic rows from leaderboards.json (`swe_bench_site`).\n\nCompare only within this board \u2014 task coverage and scoring differ from Verified and Lite.", "category": "eval", "split": null, "language": "python", "task_count": null, "homepage_url": "https://www.swebench.com/", "paper_url": "https://arxiv.org/abs/2310.06770", "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "sonar-foundation-agent", "name": "Sonar Foundation Agent", "vendor": "Sonar", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.perplexity.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20251219_sonar-foundation-agent_claude-opus-4-5.png"}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "success_rate": 52.62, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://www.sonarsource.com", "observed_at": "2025-12-19T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-7-20250219", "name": "Claude 3 7 20250219", "vendor": "Anthropic"}, "success_rate": 33.83, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://github.com/swe-agent/swe-agent", "observed_at": "2025-02-27T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 29.38, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-11-03T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "scaffold_code": null, "repo_url": "https://github.com/nus-apr/auto-code-rover", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 24.89, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://www.autocoderover.net/", "observed_at": "2024-11-21T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "scaffold_code": null, "repo_url": "https://github.com/nus-apr/auto-code-rover", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 18.83, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://autocoderover.dev/", "observed_at": "2024-06-28T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 18.13, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/", "observed_at": "2024-06-20T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "appmap-navie", "name": "Appmap Navie", "vendor": "AppMap", "scaffold_code": null, "repo_url": "https://github.com/applandeo/appmap", "homepage_url": "https://appmap.io", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240615_appmap-navie_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 14.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-06-15T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 12.47, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 11.99, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-07-28T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-opus", "name": "Claude 3 Opus", "vendor": "Anthropic"}, "success_rate": 10.51, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/", "observed_at": "2024-04-02T00:00:00+00:00"}]}}