{"benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified", "description": "SWE-bench Verified is the standard held-out evaluation split for software engineering agents: 500 real GitHub issues from popular Python repositories, human-filtered for quality and reproducibility.\n\nThe public leaderboard at swebench.com accepts community harness submissions (`source = swe_bench_site`). Rows are harness \u00d7 model \u00d7 % resolved on the full 500 tasks. This is distinct from the bash-only board (mini-SWE-agent only) and from Artificial Analysis component scores.\n\nHarnessRL uses Verified as the primary eval benchmark in published configs. Scores tagged `is_harnessrl_measured` use fixed harness versions under the HarnessRL eval protocol.", "category": "eval", "split": "verified", "language": "python", "task_count": 500, "homepage_url": "https://www.swebench.com/", "paper_url": "https://arxiv.org/abs/2310.06770", "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/verified", "is_public_leaderboard": true, "harness_count": 40, "model_count": 88, "score_count": 116, "best_success_rate": 79.2}, "protocol": {"eval_protocol": "SWE-bench Verified public submission", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 116}, "rankings": {"benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified", "description": "SWE-bench Verified is the standard held-out evaluation split for software engineering agents: 500 real GitHub issues from popular Python repositories, human-filtered for quality and reproducibility.\n\nThe public leaderboard at swebench.com accepts community harness submissions (`source = swe_bench_site`). Rows are harness \u00d7 model \u00d7 % resolved on the full 500 tasks. This is distinct from the bash-only board (mini-SWE-agent only) and from Artificial Analysis component scores.\n\nHarnessRL uses Verified as the primary eval benchmark in published configs. Scores tagged `is_harnessrl_measured` use fixed harness versions under the HarnessRL eval protocol.", "category": "eval", "split": "verified", "language": "python", "task_count": 500, "homepage_url": "https://www.swebench.com/", "paper_url": "https://arxiv.org/abs/2310.06770", "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/verified", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "live-swe-agent", "name": "Live SWE Agent", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/live-swe-agent", "homepage_url": "https://github.com/OpenAutoCoder/live-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20251120_livesweagent_gemini-3-pro-preview.png"}, "model": {"slug": "claude-4-5-opus-medium", "name": "Claude 4.5 Opus (medium)", "vendor": "Anthropic"}, "success_rate": 79.2, "resolved_count": 396, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/live-swe-agent", "observed_at": "2025-12-15T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "sonar-foundation-agent", "name": "Sonar Foundation Agent", "vendor": "Sonar", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.perplexity.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20251219_sonar-foundation-agent_claude-opus-4-5.png"}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "success_rate": 79.2, "resolved_count": 396, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.sonarsource.com", "observed_at": "2025-12-05T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "trae-agent", "name": "Trae Agent", "vendor": "ByteDance", "scaffold_code": null, "repo_url": "https://github.com/bytedance/trae-agent", "homepage_url": "https://github.com/bytedance/trae-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250612_trae.png"}, "model": {"slug": "doubao-seed-code", "name": "Doubao Seed Code", "vendor": "ByteDance"}, "success_rate": 78.8, "resolved_count": 394, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/bytedance/trae-agent", "observed_at": "2025-09-28T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "live-swe-agent", "name": "Live SWE Agent", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/live-swe-agent", "homepage_url": "https://github.com/OpenAutoCoder/live-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20251120_livesweagent_gemini-3-pro-preview.png"}, "model": {"slug": "gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", "vendor": "Google"}, "success_rate": 77.4, "resolved_count": 387, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/live-swe-agent", "observed_at": "2025-11-20T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "epam-ai-run-developer-agent", "name": "EPAM AI Run", "vendor": "EPAM Systems, Inc.", "scaffold_code": null, "repo_url": "https://github.com/epam/AI-RUN", "homepage_url": "https://github.com/epam/AI-RUN", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240820_epam-ai-run-gpt-4o.jpg"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 76.8, "resolved_count": 384, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.epam.com/services/artificial-intelligence", "observed_at": "2025-08-04T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-opus-high", "name": "Claude 4.5 Opus (high)", "vendor": "Anthropic"}, "success_rate": 76.8, "resolved_count": 384, "total_count": 500, "cost_usd": 0.753907997, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-3-flash-high", "name": "Gemini 3 Flash (high)", "vendor": "Google"}, "success_rate": 75.8, "resolved_count": 379, "total_count": 500, "cost_usd": 0.35596413079999994, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "minimax-m2-5-high", "name": "MiniMax M2.5 (high)", "vendor": "MiniMax"}, "success_rate": 75.8, "resolved_count": 379, "total_count": 500, "cost_usd": 0.073289987844, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-6-opus", "name": "Claude 4.6 Opus", "vendor": "Anthropic"}, "success_rate": 75.6, "resolved_count": 378, "total_count": 500, "cost_usd": 0.5515226445, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "trae-agent", "name": "Trae Agent", "vendor": "ByteDance", "scaffold_code": null, "repo_url": "https://github.com/bytedance/trae-agent", "homepage_url": "https://github.com/bytedance/trae-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250612_trae.png"}, "model": {"slug": "ensemble-claude-4-sonnet-claude-4-opus-claude-3-7-sonnet-gemini-2-5-pro", "name": "Ensemble (Claude 4 Sonnet, Claude 4 Opus, Claude 3.7 Sonnet, Gemini 2.5 Pro)", "vendor": "Anthropic"}, "success_rate": 75.2, "resolved_count": 376, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/bytedance/trae-agent", "observed_at": "2025-06-12T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "sonar-foundation-agent", "name": "Sonar Foundation Agent", "vendor": "Sonar", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.perplexity.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20251219_sonar-foundation-agent_claude-opus-4-5.png"}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 74.8, "resolved_count": 374, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.sonarsource.com", "observed_at": "2025-11-03T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "lingxi-v1-5", "name": "Lingxi V1.5 Claude 4 Sonnet 20250514", "vendor": "Nimasteryang", "scaffold_code": null, "repo_url": "https://github.com/nimasteryang/Lingxi", "homepage_url": "https://github.com/nimasteryang/Lingxi", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250720_Lingxi-v1.5_claude-4-sonnet-20250514.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 74.6, "resolved_count": 373, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/nimasteryang/Lingxi", "observed_at": "2025-07-20T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "joycode", "name": "Joycode", "vendor": "Jd Opensource", "scaffold_code": null, "repo_url": "https://github.com/jd-opensource/joycode-agent", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250915_JoyCode.png"}, "model": {"slug": "ensemble-claude-4-sonnet-gpt-4-1", "name": "Ensemble (Claude 4 Sonnet, GPT-4.1)", "vendor": "Anthropic"}, "success_rate": 74.6, "resolved_count": 373, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/jd-opensource/joycode-agent", "observed_at": "2025-09-15T00:00:00+00:00"}, {"rank": 14, "harness": {"slug": "refact-ai-agent", "name": "Refact Ai Agent", "vendor": "Refact.ai", "scaffold_code": null, "repo_url": "https://github.com/smallcloudai/refact", "homepage_url": "https://refact.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250611_Refact_Agent_claude-4-sonnet.png"}, "model": {"slug": "ensemble-claude-4-sonnet-o4-mini", "name": "Ensemble (Claude 4 Sonnet, o4 mini)", "vendor": "Anthropic"}, "success_rate": 74.4, "resolved_count": 372, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://refact.ai", "observed_at": "2025-06-03T00:00:00+00:00"}, {"rank": 15, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-opus-medium", "name": "Claude 4.5 Opus (medium)", "vendor": "Anthropic"}, "success_rate": 74.4, "resolved_count": 372, "total_count": 500, "cost_usd": 0.721244997, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-11-24T00:00:00+00:00"}, {"rank": 16, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", "vendor": "Google"}, "success_rate": 74.2, "resolved_count": 371, "total_count": 500, "cost_usd": 0.45995751895000003, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-11-18T00:00:00+00:00"}, {"rank": 17, "harness": {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "scaffold_code": null, "repo_url": "https://github.com/anthropics", "homepage_url": "https://www.anthropic.com", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241022_tools_claude-3-5-haiku.jpg"}, "model": {"slug": "claude-4-opus", "name": "Claude 4 Opus", "vendor": "Anthropic"}, "success_rate": 73.2, "resolved_count": 366, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/", "observed_at": "2025-05-22T00:00:00+00:00"}, {"rank": 18, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-2-codex", "name": "GPT-5.2 codex", "vendor": "openai"}, "success_rate": 72.8, "resolved_count": 364, "total_count": 500, "cost_usd": 0.4494238987, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-19T00:00:00+00:00"}, {"rank": 19, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "glm-5-high", "name": "GLM 5 (high)", "vendor": "Zhipu"}, "success_rate": 72.8, "resolved_count": 364, "total_count": 500, "cost_usd": 0.53438913622, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 20, "harness": {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "scaffold_code": null, "repo_url": "https://github.com/anthropics", "homepage_url": "https://www.anthropic.com", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241022_tools_claude-3-5-haiku.jpg"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 72.4, "resolved_count": 362, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/", "observed_at": "2025-05-22T00:00:00+00:00"}, {"rank": 21, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-2-high", "name": "GPT-5.2 (high)", "vendor": "OpenAI"}, "success_rate": 71.8, "resolved_count": 359, "total_count": 500, "cost_usd": 0.5202590036, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-11T00:00:00+00:00"}, {"rank": 22, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gpt-5", "name": "GPT-5", "vendor": "openai"}, "success_rate": 71.8, "resolved_count": 359, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://all-hands.dev/", "observed_at": "2025-08-07T00:00:00+00:00"}, {"rank": 23, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-sonnet-high", "name": "Claude 4.5 Sonnet (high)", "vendor": "Anthropic"}, "success_rate": 71.4, "resolved_count": 357, "total_count": 500, "cost_usd": 0.6578964984000001, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 24, "harness": {"slug": "prometheus-v1-2", "name": "Prometheus V1 2", "vendor": "EuniAI", "scaffold_code": null, "repo_url": "https://github.com/GAIR-NLP/Prometheus", "homepage_url": "https://github.com/GAIR-NLP/Prometheus", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250929_Prometheus_v1.2_gpt5.jpg"}, "model": {"slug": "gpt-5", "name": "GPT-5", "vendor": "openai"}, "success_rate": 71.2, "resolved_count": 356, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://euni.ai/", "observed_at": "2025-09-29T00:00:00+00:00"}, {"rank": 25, "harness": {"slug": "lingxi-v1-5-x", "name": "Lingxi V1.5 X KIMI K2", "vendor": "Lingxi Agent", "scaffold_code": null, "repo_url": "https://github.com/lingxi-agent/Lingxi/tree/master", "homepage_url": "https://github.com/lingxi-agent/Lingxi/tree/master", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20251014_Lingxi_kimi_k2.png"}, "model": {"slug": "lingxi-kimi", "name": "Lingxi KIMI", "vendor": "Moonshot"}, "success_rate": 71.2, "resolved_count": 356, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/lingxi-agent/Lingxi/tree/master", "observed_at": "2025-10-14T00:00:00+00:00"}, {"rank": 26, "harness": {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "scaffold_code": null, "repo_url": "https://github.com/aorwall/moatless-tools", "homepage_url": "https://github.com/aorwall/moatless-tools", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240617_moatless_gpt4o.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 70.8, "resolved_count": 354, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/aorwall/moatless-tools", "observed_at": "2025-06-11T00:00:00+00:00"}, {"rank": 27, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "kimi-k2-5-high", "name": "KIMI K2.5 (high)", "vendor": "Moonshot"}, "success_rate": 70.8, "resolved_count": 354, "total_count": 500, "cost_usd": 0.14655374074400002, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 28, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 70.6, "resolved_count": 353, "total_count": 500, "cost_usd": 0.5583347409, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-09-29T00:00:00+00:00"}, {"rank": 29, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 70.4, "resolved_count": 352, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://all-hands.dev/", "observed_at": "2025-05-24T00:00:00+00:00"}, {"rank": 30, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "deepseek-v3-2-high", "name": "DeepSeek V3.2 (high)", "vendor": "DeepSeek"}, "success_rate": 70.0, "resolved_count": 350, "total_count": 500, "cost_usd": 0.4478464965, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 31, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "qwen3-coder-480b-a35b-instruct", "name": "Qwen3 Coder 480B A35B Instruct", "vendor": "Qwen"}, "success_rate": 69.6, "resolved_count": 348, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://qwenlm.github.io/blog/qwen3-coder/", "observed_at": "2025-08-05T00:00:00+00:00"}, {"rank": 32, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-3-pro", "name": "Gemini 3 Pro", "vendor": "Google"}, "success_rate": 69.6, "resolved_count": 348, "total_count": 500, "cost_usd": 0.9600198564, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-26T00:00:00+00:00"}, {"rank": 33, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-2", "name": "GPT-5.2", "vendor": "openai"}, "success_rate": 69.0, "resolved_count": 345, "total_count": 500, "cost_usd": 0.2696674974, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-11T00:00:00+00:00"}, {"rank": 34, "harness": {"slug": "nemotron-cortexa", "name": "Nemotron Cortexa", "vendor": "NVIDIA", "scaffold_code": null, "repo_url": null, "homepage_url": "https://developer.nvidia.com/nemotron", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250516_cortexa_o3.jpg"}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "success_rate": 68.2, "resolved_count": 341, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://research.nvidia.com/labs/adlr/cortexa/", "observed_at": "2025-05-16T00:00:00+00:00"}, {"rank": 35, "harness": {"slug": "glm-4-6", "name": "GLM 4 6", "vendor": "Z.ai", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.zhipuai.cn", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250930_zai_glm4-6.png"}, "model": {"slug": "glm4-6", "name": "Glm4 6", "vendor": "Zhipu"}, "success_rate": 68.2, "resolved_count": 341, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://chat.z.ai/", "observed_at": "2025-09-30T00:00:00+00:00"}, {"rank": 36, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-opus", "name": "Claude 4 Opus", "vendor": "Anthropic"}, "success_rate": 67.6, "resolved_count": 338, "total_count": 500, "cost_usd": 1.131270447, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-02T00:00:00+00:00"}, {"rank": 37, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 66.6, "resolved_count": 333, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2025-05-22T00:00:00+00:00"}, {"rank": 38, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-haiku-high", "name": "Claude 4.5 Haiku (high)", "vendor": "Anthropic"}, "success_rate": 66.6, "resolved_count": 333, "total_count": 500, "cost_usd": 0.33092443610000005, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 39, "harness": {"slug": "aime-coder-v1", "name": "Aime Coder V1", "vendor": "Swe Bench", "scaffold_code": null, "repo_url": "https://github.com/SWE-bench/experiments/blob/main/evaluation/verified/20250514_aime_coder/README.md", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250514_aime_coder.png"}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "success_rate": 66.4, "resolved_count": 332, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-bench/experiments/blob/main/evaluation/verified/20250514_aime_coder/README.md", "observed_at": "2025-05-14T00:00:00+00:00"}, {"rank": 40, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-1-codex-medium", "name": "GPT-5.1 codex (medium)", "vendor": "OpenAI"}, "success_rate": 66.0, "resolved_count": 330, "total_count": 500, "cost_usd": 0.5888823500000001, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-11-24T00:00:00+00:00"}, {"rank": 41, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-1-medium", "name": "GPT-5.1 (medium)", "vendor": "OpenAI"}, "success_rate": 66.0, "resolved_count": 330, "total_count": 500, "cost_usd": 0.3061554725, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-11-20T00:00:00+00:00"}, {"rank": 42, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "kimi-k2", "name": "KIMI K2", "vendor": "moonshotai"}, "success_rate": 65.4, "resolved_count": 327, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://all-hands.dev/", "observed_at": "2025-07-16T00:00:00+00:00"}, {"rank": 43, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "success_rate": 65.0, "resolved_count": 325, "total_count": 500, "cost_usd": 0.2803830175, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-07T00:00:00+00:00"}, {"rank": 44, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 64.93, "resolved_count": 325, "total_count": 500, "cost_usd": 0.37145316780000004, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 45, "harness": {"slug": "w&b-programmer-o1-crosscheck5", "name": "W&b Programmer O1 Crosscheck5", "vendor": "Weights & Biases", "scaffold_code": null, "repo_url": "https://github.com/wandb/wandb", "homepage_url": "https://wandb.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250117_wandb_programmer_o1_crosscheck5.png"}, "model": {"slug": "o1", "name": "o1", "vendor": "openai"}, "success_rate": 64.6, "resolved_count": 323, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://wandb.ai", "observed_at": "2025-01-17T00:00:00+00:00"}, {"rank": 46, "harness": {"slug": "patchpilot", "name": "Patchpilot", "vendor": "Ucsb Mlsec", "scaffold_code": null, "repo_url": "https://github.com/ucsb-mlsec/Co-PatcheR", "homepage_url": "https://github.com/InternLM/PatchPilot", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250528_patchpilot_Co-PatcheR.gif"}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "success_rate": 64.6, "resolved_count": 323, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/verified", "observed_at": "2025-05-03T00:00:00+00:00"}, {"rank": 47, "harness": {"slug": "glm-4-5", "name": "GLM 4 5", "vendor": "Z.ai", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.zhipuai.cn", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250728_zai_glm4-5.png"}, "model": {"slug": "glm4-5", "name": "Glm4 5", "vendor": "Zhipu"}, "success_rate": 64.2, "resolved_count": 321, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://chat.z.ai/", "observed_at": "2025-07-28T00:00:00+00:00"}, {"rank": 48, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "kimi-k2-thinking", "name": "KIMI K2 Thinking", "vendor": "moonshotai"}, "success_rate": 63.4, "resolved_count": 317, "total_count": 500, "cost_usd": 0.4383072354, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-10T00:00:00+00:00"}, {"rank": 49, "harness": {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "scaffold_code": null, "repo_url": "https://github.com/anthropics", "homepage_url": "https://www.anthropic.com", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241022_tools_claude-3-5-haiku.jpg"}, "model": {"slug": "claude-3-7-sonnet-20250224", "name": "Claude 3.7 Sonnet (2025-02-24)", "vendor": "Anthropic"}, "success_rate": 63.2, "resolved_count": 316, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/", "observed_at": "2025-02-24T00:00:00+00:00"}, {"rank": 50, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "success_rate": 62.4, "resolved_count": 312, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2025-02-25T00:00:00+00:00"}, {"rank": 51, "harness": {"slug": "codestory-midwit-agent", "name": "Codestory Midwit Agent", "vendor": "CodeStory", "scaffold_code": null, "repo_url": null, "homepage_url": "https://codestory.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240702_codestory_aide_mixed.png"}, "model": {"slug": "swe-search", "name": "SWE Search", "vendor": "Princeton NLP"}, "success_rate": 62.2, "resolved_count": 311, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://aide.dev/", "observed_at": "2024-12-21T00:00:00+00:00"}, {"rank": 52, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "minimax-m2", "name": "MiniMax M2", "vendor": "minimax"}, "success_rate": 61.0, "resolved_count": 305, "total_count": 500, "cost_usd": 0.42825617729, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-11-24T00:00:00+00:00"}, {"rank": 53, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "4x-scaled-2024-02-03", "name": "4x-scaled-2024-02-03", "vendor": "Research baseline"}, "success_rate": 60.8, "resolved_count": 304, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://all-hands.dev/", "observed_at": "2025-02-03T00:00:00+00:00"}, {"rank": 54, "harness": {"slug": "learn-by-interact", "name": "Learn By Interact", "vendor": "Google", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 60.2, "resolved_count": 301, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://arxiv.org/pdf/2501.10893", "observed_at": "2025-01-10T00:00:00+00:00"}, {"rank": 55, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "deepseek-v3-2-reasoner", "name": "DeepSeek V3.2 Reasoner", "vendor": "DeepSeek"}, "success_rate": 60.0, "resolved_count": 300, "total_count": 500, "cost_usd": 0.0280763028, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-01T00:00:00+00:00"}, {"rank": 56, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-mini-medium", "name": "GPT-5 mini (medium)", "vendor": "OpenAI"}, "success_rate": 59.8, "resolved_count": 299, "total_count": 500, "cost_usd": 0.035477067300000005, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-07T00:00:00+00:00"}, {"rank": 57, "harness": {"slug": "deepswe-preview", "name": "Deepswe Preview", "vendor": "Agentica", "scaffold_code": null, "repo_url": null, "homepage_url": "https://datacurve.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250629_deepswerl_r2eagent_tts.jpg"}, "model": {"slug": "tts-bo16", "name": "tts-bo16", "vendor": "TTS"}, "success_rate": 58.8, "resolved_count": 294, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "agentica-project.com/", "observed_at": "2025-06-29T00:00:00+00:00"}, {"rank": 58, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "success_rate": 58.4, "resolved_count": 292, "total_count": 500, "cost_usd": 0.333652748, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 59, "harness": {"slug": "swe-rizzo", "name": "SWE Rizzo", "vendor": "SWE-Rizzo", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250405_swe-rizzo_claude37.png"}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "success_rate": 56.6, "resolved_count": 283, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://gen42.ai", "observed_at": "2025-04-05T00:00:00+00:00"}, {"rank": 60, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "devstral-small-2512", "name": "Devstral Small (2512)", "vendor": "Mistral"}, "success_rate": 56.4, "resolved_count": 282, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-09T00:00:00+00:00"}, {"rank": 61, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-mini", "name": "GPT-5 mini", "vendor": "openai"}, "success_rate": 56.2, "resolved_count": 281, "total_count": 500, "cost_usd": 0.0472012191, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2026-02-17T00:00:00+00:00"}, {"rank": 62, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "qwen3-coder-480b-a35b-instruct", "name": "Qwen3 Coder 480B A35B Instruct", "vendor": "Qwen"}, "success_rate": 55.4, "resolved_count": 277, "total_count": 500, "cost_usd": 0.247927108, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-02T00:00:00+00:00"}, {"rank": 63, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "glm-4-6-t=1", "name": "GLM 4.6 (T=1)", "vendor": "Zhipu"}, "success_rate": 55.4, "resolved_count": 277, "total_count": 500, "cost_usd": 0.09661007519999999, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-01T00:00:00+00:00"}, {"rank": 64, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "glm-4-5-2025-08-22", "name": "GLM 4.5 (2025 08 22)", "vendor": "Zhipu"}, "success_rate": 54.2, "resolved_count": 271, "total_count": 500, "cost_usd": 0.2971903488, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-22T00:00:00+00:00"}, {"rank": 65, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "devstral-2512", "name": "Devstral (2512)", "vendor": "mistralai"}, "success_rate": 53.8, "resolved_count": 269, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-12-09T00:00:00+00:00"}, {"rank": 66, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-2-5-pro-20250506", "name": "Gemini 2.5 Pro (2025-05-06)", "vendor": "Google"}, "success_rate": 53.6, "resolved_count": 268, "total_count": 500, "cost_usd": 0.28837209562500005, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 67, "harness": {"slug": "codesweep-swe-agent", "name": "CodeSweep SWE Agent KIMI K2 Instruct", "vendor": "CodeSweep Inc.", "scaffold_code": null, "repo_url": null, "homepage_url": "https://codesweep.ai/", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250804_codesweep_sweagent_kimi_k2_instruct.svg"}, "model": {"slug": "codesweep-sweagent-kimi-k2", "name": "Codesweep Sweagent KIMI K2", "vendor": "Moonshot"}, "success_rate": 53.4, "resolved_count": 267, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://codesweep.ai/", "observed_at": "2025-08-04T00:00:00+00:00"}, {"rank": 68, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 53.0, "resolved_count": 265, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-10-29T00:00:00+00:00"}, {"rank": 69, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-3-7-sonnet-20250219", "name": "Claude 3.7 Sonnet (2025-02-19)", "vendor": "Anthropic"}, "success_rate": 52.8, "resolved_count": 264, "total_count": 500, "cost_usd": 0.3546154482, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-20T00:00:00+00:00"}, {"rank": 70, "harness": {"slug": "entropo", "name": "Entropo", "vendor": "Sherdencooper", "scaffold_code": null, "repo_url": "https://github.com/sherdencooper/R2E-Gym", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250901_entroPO_R2E_QwenCoder30BA3B.png"}, "model": {"slug": "qwen3-coder-30b-a3b-instruct", "name": "Qwen3 Coder 30B A3B Instruct", "vendor": "qwen"}, "success_rate": 52.2, "resolved_count": 261, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/sherdencooper/R2E-Gym", "observed_at": "2025-09-01T00:00:00+00:00"}, {"rank": 71, "harness": {"slug": "google-jules", "name": "Google Jules", "vendor": "Google", "scaffold_code": null, "repo_url": null, "homepage_url": "https://jules.google", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241212_google_jules_gemini_2.0_flash_experimental.png"}, "model": {"slug": "gemini-2-0-flash-20241212", "name": "Gemini 2.0 Flash (2024-12-12)", "vendor": "Google"}, "success_rate": 52.2, "resolved_count": 261, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://developers.googleblog.com/en/the-next-chapter-of-the-gemini-era-for-developers/", "observed_at": "2024-12-12T00:00:00+00:00"}, {"rank": 72, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "qwen3-coder-30b-a3b-instruct", "name": "Qwen3 Coder 30B A3B Instruct", "vendor": "qwen"}, "success_rate": 51.6, "resolved_count": 258, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://qwenlm.github.io/blog/qwen3-coder/", "observed_at": "2025-08-05T00:00:00+00:00"}, {"rank": 73, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 50.8, "resolved_count": 254, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-12-02T00:00:00+00:00"}, {"rank": 74, "harness": {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "scaffold_code": null, "repo_url": "https://github.com/anthropics", "homepage_url": "https://www.anthropic.com", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241022_tools_claude-3-5-haiku.jpg"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 49.0, "resolved_count": 245, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/", "observed_at": "2024-10-22T00:00:00+00:00"}, {"rank": 75, "harness": {"slug": "appmap-navie-v2", "name": "Appmap Navie V2", "vendor": "AppMap", "scaffold_code": null, "repo_url": "https://github.com/applandeo/appmap", "homepage_url": "https://appmap.io", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241113_navie-2-gpt4o-sonnet.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 47.2, "resolved_count": 236, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-11-06T00:00:00+00:00"}, {"rank": 76, "harness": {"slug": "skywork-swe-32b", "name": "Skywork SWE 32b", "vendor": "Skywork AI", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.skywork.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250616_Skywork-SWE-32B+TTS_Bo8.jpg"}, "model": {"slug": "tts-bo8", "name": "tts-bo8", "vendor": "TTS"}, "success_rate": 47.0, "resolved_count": 235, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://quixotic-sting-239.notion.site/eb17f379610040ceb54da5d5d24065bd", "observed_at": "2025-06-16T00:00:00+00:00"}, {"rank": 77, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "devstral-small-2505", "name": "Devstral Small 2505", "vendor": "Mistral"}, "success_rate": 46.8, "resolved_count": 234, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://all-hands.dev/", "observed_at": "2025-05-20T00:00:00+00:00"}, {"rank": 78, "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "scaffold_code": null, "repo_url": "https://github.com/nus-apr/auto-code-rover", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 46.2, "resolved_count": 231, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.autocoderover.net/", "observed_at": "2024-11-08T00:00:00+00:00"}, {"rank": 79, "harness": {"slug": "patchpilot", "name": "Patchpilot", "vendor": "Ucsb Mlsec", "scaffold_code": null, "repo_url": "https://github.com/ucsb-mlsec/Co-PatcheR", "homepage_url": "https://github.com/InternLM/PatchPilot", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250528_patchpilot_Co-PatcheR.gif"}, "model": {"slug": "co-patcher", "name": "Co PatcheR", "vendor": "Co-Patcher"}, "success_rate": 46.0, "resolved_count": 230, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/ucsb-mlsec/Co-PatcheR", "observed_at": "2025-05-28T00:00:00+00:00"}, {"rank": 80, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "success_rate": 45.0, "resolved_count": 225, "total_count": 500, "cost_usd": 0.2099733328, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 81, "harness": {"slug": "codeshellagent", "name": "Codeshellagent", "vendor": "Wisdomshell", "scaffold_code": null, "repo_url": "https://github.com/WisdomShell/codeshell", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250118_codeshellagent_gemini_2.0_flash_experimental.png"}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "success_rate": 44.2, "resolved_count": 221, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/WisdomShell/codeshell", "observed_at": "2025-01-18T00:00:00+00:00"}, {"rank": 82, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "kimi-k2-instruct", "name": "KIMI K2 Instruct", "vendor": "Moonshot"}, "success_rate": 43.8, "resolved_count": 219, "total_count": 500, "cost_usd": 0.531593366, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-07T00:00:00+00:00"}, {"rank": 83, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "o3-mini", "name": "o3 mini", "vendor": "openai"}, "success_rate": 42.4, "resolved_count": 212, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/sorendunn/Agentless-Lite", "observed_at": "2025-02-14T00:00:00+00:00"}, {"rank": 84, "harness": {"slug": "swe-exp", "name": "SWE Exp", "vendor": "Yerbapage", "scaffold_code": null, "repo_url": "https://github.com/YerbaPage/SWE-Exp", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250806_SWE-Exp_DeepSeek-V3.png"}, "model": {"slug": "deepseek-v3", "name": "DeepSeek V3", "vendor": "DeepSeek"}, "success_rate": 42.0, "resolved_count": 210, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/YerbaPage/SWE-Exp", "observed_at": "2025-08-06T00:00:00+00:00"}, {"rank": 85, "harness": {"slug": "swe-rl-llama3-swe-rl-70b", "name": "SWE Rl Llama3 SWE Rl 70b", "vendor": "Facebookresearch", "scaffold_code": null, "repo_url": "https://github.com/facebookresearch/swe-rl", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250226_swerl_llama3_70b.png"}, "model": {"slug": "agentless-mini-20250226", "name": "Agentless Mini) (20250226", "vendor": "Agentless"}, "success_rate": 41.2, "resolved_count": 206, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/facebookresearch/swe-rl", "observed_at": "2025-02-26T00:00:00+00:00"}, {"rank": 86, "harness": {"slug": "nebius-ai", "name": "Nebius AI QWEN 2.5 72B Generator", "vendor": "Nebius", "scaffold_code": null, "repo_url": null, "homepage_url": "https://nebius.com/blog/posts/training-and-search-for-software-engineering-agents", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241113_nebius-search-open-weight-models-11-24.png"}, "model": {"slug": "ensemble-nebius-ai-qwen-2-5-72b-generator-llama-3-1-70b-critic", "name": "Ensemble (Nebius AI QWEN 2.5 72B Generator, Llama 3.1 70B Critic)", "vendor": "Qwen"}, "success_rate": 40.6, "resolved_count": 203, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://nebius.com/blog/posts/training-and-search-for-software-engineering-agents", "observed_at": "2024-11-13T00:00:00+00:00"}, {"rank": 87, "harness": {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "scaffold_code": null, "repo_url": "https://github.com/anthropics", "homepage_url": "https://www.anthropic.com", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241022_tools_claude-3-5-haiku.jpg"}, "model": {"slug": "claude-3-5-haiku", "name": "Claude 3.5 Haiku", "vendor": "Anthropic"}, "success_rate": 40.6, "resolved_count": 203, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/", "observed_at": "2024-10-22T00:00:00+00:00"}, {"rank": 88, "harness": {"slug": "composio-swekit", "name": "Composio SWEKit", "vendor": "Composiohq", "scaffold_code": null, "repo_url": "https://github.com/ComposioHQ/composio/tree/master/python/swe/agent", "homepage_url": "https://composio.dev", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241016_composio_swekit.png"}, "model": {"slug": "claude-3-5-sonnet-20241016", "name": "Claude 3.5 Sonnet (2024-10-16)", "vendor": "Anthropic"}, "success_rate": 40.6, "resolved_count": 203, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/ComposioHQ/composio/tree/master/python/swe/agent", "observed_at": "2024-10-16T00:00:00+00:00"}, {"rank": 89, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "swe-agent-lm-32b", "name": "SWE-agent-LM-32B", "vendor": "Princeton NLP"}, "success_rate": 40.2, "resolved_count": 201, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://swesmith.com/", "observed_at": "2025-05-11T00:00:00+00:00"}, {"rank": 90, "harness": {"slug": "epam-ai-run-developer-agent", "name": "EPAM AI Run", "vendor": "EPAM Systems, Inc.", "scaffold_code": null, "repo_url": "https://github.com/epam/AI-RUN", "homepage_url": "https://github.com/epam/AI-RUN", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240820_epam-ai-run-gpt-4o.jpg"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 39.6, "resolved_count": 198, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.epam.com/services/artificial-intelligence", "observed_at": "2024-10-29T00:00:00+00:00"}, {"rank": 91, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-4-1-20250414", "name": "GPT-4.1 (2025-04-14)", "vendor": "OpenAI"}, "success_rate": 39.58, "resolved_count": 198, "total_count": 500, "cost_usd": 0.146313124, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 92, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 38.8, "resolved_count": 194, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-10-28T00:00:00+00:00"}, {"rank": 93, "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "scaffold_code": null, "repo_url": "https://github.com/nus-apr/auto-code-rover", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 38.4, "resolved_count": 192, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://autocoderover.dev/", "observed_at": "2024-06-28T00:00:00+00:00"}, {"rank": 94, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "devstral-small-2507", "name": "DevStral Small 2507", "vendor": "Mistral"}, "success_rate": 38.0, "resolved_count": 190, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://swe-agent.com/", "observed_at": "2025-07-25T00:00:00+00:00"}, {"rank": 95, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-nano-medium", "name": "GPT-5 nano (medium)", "vendor": "OpenAI"}, "success_rate": 34.8, "resolved_count": 174, "total_count": 500, "cost_usd": 0.03807518078, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-07T00:00:00+00:00"}, {"rank": 96, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 33.6, "resolved_count": 168, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-06-20T00:00:00+00:00"}, {"rank": 97, "harness": {"slug": "swe-fixer", "name": "SWE Fixer", "vendor": "Internlm", "scaffold_code": null, "repo_url": "https://github.com/InternLM/SWE-Fixer", "homepage_url": "https://github.com/InternLM/SWE-Fixer", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128.png"}, "model": {"slug": "ensemble-swe-fixer-qwen2-5-7b-retriever-qwen2-5-72b-editor", "name": "Ensemble (SWE Fixer (Qwen2.5 7b Retriever, Qwen2.5 72b Editor)", "vendor": "Qwen"}, "success_rate": 32.8, "resolved_count": 164, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/InternLM/SWE-Fixer", "observed_at": "2025-03-06T00:00:00+00:00"}, {"rank": 98, "harness": {"slug": "masai", "name": "Masai", "vendor": "Masai", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4o-20240612", "name": "GPT-4o (2024-06-12)", "vendor": "OpenAI"}, "success_rate": 32.6, "resolved_count": 163, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://masai-dev-agent.github.io/", "observed_at": "2024-06-12T00:00:00+00:00"}, {"rank": 99, "harness": {"slug": "swe-fixer", "name": "SWE Fixer", "vendor": "Internlm", "scaffold_code": null, "repo_url": "https://github.com/InternLM/SWE-Fixer", "homepage_url": "https://github.com/InternLM/SWE-Fixer", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128.png"}, "model": {"slug": "ensemble-swe-fixer-qwen2-5-7b-retriever-qwen2-5-72b-editor-20241128", "name": "Ensemble (SWE Fixer (Qwen2.5 7b Retriever, Qwen2.5 72b Editor) 20241128)", "vendor": "Qwen"}, "success_rate": 30.2, "resolved_count": 151, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/InternLM/SWE-Fixer", "observed_at": "2024-11-28T00:00:00+00:00"}, {"rank": 100, "harness": {"slug": "lingma-agent", "name": "Lingma Agent", "vendor": "Alibaba", "scaffold_code": null, "repo_url": "https://github.com/modelscope/modelscope", "homepage_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240918_lingma-agent_lingma-swe-gpt-7b.jpg"}, "model": {"slug": "lingma-swe-gpt-72b-v0925", "name": "Lingma SWE GPT 72b (v0925)", "vendor": "OpenAI"}, "success_rate": 28.8, "resolved_count": 144, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT (https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT-v20240925)", "observed_at": "2024-10-02T00:00:00+00:00"}, {"rank": 101, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-2-5-flash-20250417", "name": "Gemini 2.5 Flash (2025-04-17)", "vendor": "Google"}, "success_rate": 28.73, "resolved_count": 144, "total_count": 500, "cost_usd": 0.13388571835, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 102, "harness": {"slug": "appmap-navie", "name": "Appmap Navie", "vendor": "AppMap", "scaffold_code": null, "repo_url": "https://github.com/applandeo/appmap", "homepage_url": "https://appmap.io", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240615_appmap-navie_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 26.2, "resolved_count": 131, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-06-15T00:00:00+00:00"}, {"rank": 103, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-oss-120b", "name": "GPT Oss 120b", "vendor": "openai"}, "success_rate": 26.0, "resolved_count": 130, "total_count": 500, "cost_usd": 0.05711905579999999, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-07T00:00:00+00:00"}, {"rank": 104, "harness": {"slug": "lingma-agent", "name": "Lingma Agent", "vendor": "Alibaba", "scaffold_code": null, "repo_url": "https://github.com/modelscope/modelscope", "homepage_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240918_lingma-agent_lingma-swe-gpt-7b.jpg"}, "model": {"slug": "lingma-swe-gpt-72b-v0918", "name": "Lingma SWE GPT 72b (v0918)", "vendor": "OpenAI"}, "success_rate": 25.0, "resolved_count": 125, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT", "observed_at": "2024-09-18T00:00:00+00:00"}, {"rank": 105, "harness": {"slug": "epam-ai-run-developer-agent", "name": "EPAM AI Run", "vendor": "EPAM Systems, Inc.", "scaffold_code": null, "repo_url": "https://github.com/epam/AI-RUN", "homepage_url": "https://github.com/epam/AI-RUN", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240820_epam-ai-run-gpt-4o.jpg"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 24.0, "resolved_count": 120, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.epam.com/services/artificial-intelligence", "observed_at": "2024-08-20T00:00:00+00:00"}, {"rank": 106, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-4-1-mini-20250414", "name": "GPT-4.1 mini (2025-04-14)", "vendor": "OpenAI"}, "success_rate": 23.94, "resolved_count": 120, "total_count": 500, "cost_usd": 0.43515415279999997, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-20T00:00:00+00:00"}, {"rank": 107, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 23.2, "resolved_count": 116, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-07-28T00:00:00+00:00"}, {"rank": 108, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 22.4, "resolved_count": 112, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00"}, {"rank": 109, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-4o-20241120", "name": "GPT-4o (2024-11-20)", "vendor": "OpenAI"}, "success_rate": 21.62, "resolved_count": 108, "total_count": 500, "cost_usd": 1.531148125, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-20T00:00:00+00:00"}, {"rank": 110, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "llama-4-maverick-instruct", "name": "Llama 4 Maverick Instruct", "vendor": "Meta"}, "success_rate": 21.04, "resolved_count": 105, "total_count": 500, "cost_usd": 0.31370609759999996, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-20T00:00:00+00:00"}, {"rank": 111, "harness": {"slug": "lingma-agent", "name": "Lingma Agent", "vendor": "Alibaba", "scaffold_code": null, "repo_url": "https://github.com/modelscope/modelscope", "homepage_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240918_lingma-agent_lingma-swe-gpt-7b.jpg"}, "model": {"slug": "lingma-swe-gpt-7b-v0925", "name": "Lingma SWE GPT 7b (v0925)", "vendor": "OpenAI"}, "success_rate": 18.2, "resolved_count": 91, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT (https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT-v20240925)", "observed_at": "2024-10-02T00:00:00+00:00"}, {"rank": 112, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-opus", "name": "Claude 3 Opus", "vendor": "Anthropic"}, "success_rate": 15.8, "resolved_count": 79, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00"}, {"rank": 113, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "success_rate": 13.52, "resolved_count": 68, "total_count": 500, "cost_usd": 0.0, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-26T00:00:00+00:00"}, {"rank": 114, "harness": {"slug": "lingma-agent", "name": "Lingma Agent", "vendor": "Alibaba", "scaffold_code": null, "repo_url": "https://github.com/modelscope/modelscope", "homepage_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240918_lingma-agent_lingma-swe-gpt-7b.jpg"}, "model": {"slug": "lingma-swe-gpt-7b-v0918", "name": "Lingma SWE GPT 7b (v0918)", "vendor": "OpenAI"}, "success_rate": 10.2, "resolved_count": 51, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT", "observed_at": "2024-09-18T00:00:00+00:00"}, {"rank": 115, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "llama-4-scout-instruct", "name": "Llama 4 Scout Instruct", "vendor": "Meta"}, "success_rate": 9.06, "resolved_count": 45, "total_count": 500, "cost_usd": 0.115631523, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-07-20T00:00:00+00:00"}, {"rank": 116, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder 32B Instruct", "vendor": "qwen"}, "success_rate": 9.0, "resolved_count": 45, "total_count": 500, "cost_usd": 0.06811614065999999, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://mini-swe-agent.com/latest/", "observed_at": "2025-08-03T00:00:00+00:00"}]}}