{"benchmark": {"slug": "swe-bench-multimodal-site", "name": "SWE-bench Multimodal (site)", "description": "The Multimodal board on swebench.com tracks harness\u00d7model scores on frontend and multimodal software engineering tasks (~517 instances). Ingested from the same leaderboards.json cache as Verified (`swe_bench_site`).\n\nNot the same as OpenHands Index `swe-bench-multimodal` component scores (`openhands_index` source, OpenHands SDK fixed harness).", "category": "eval", "split": null, "language": "python", "task_count": 517, "homepage_url": "https://www.swebench.com/multimodal", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/multimodal", "is_public_leaderboard": true, "harness_count": 7, "model_count": 8, "score_count": 15, "best_success_rate": 35.98}, "protocol": {"eval_protocol": "SWE-bench Multimodal public submission", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 15}, "rankings": {"benchmark": {"slug": "swe-bench-multimodal-site", "name": "SWE-bench Multimodal (site)", "description": "The Multimodal board on swebench.com tracks harness\u00d7model scores on frontend and multimodal software engineering tasks (~517 instances). Ingested from the same leaderboards.json cache as Verified (`swe_bench_site`).\n\nNot the same as OpenHands Index `swe-bench-multimodal` component scores (`openhands_index` source, OpenHands SDK fixed harness).", "category": "eval", "split": null, "language": "python", "task_count": 517, "homepage_url": "https://www.swebench.com/multimodal", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/multimodal", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "guirepair", "name": "GUIRepair", "vendor": "GUIRepair", "scaffold_code": null, "repo_url": null, "homepage_url": "https://sites.google.com/view/guirepair", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250531_GUIRepair_gpt4o.png"}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "success_rate": 35.98, "resolved_count": 186, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://sites.google.com/view/guirepair", "observed_at": "2025-07-01T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "refact-ai-agent", "name": "Refact Ai Agent", "vendor": "Refact.ai", "scaffold_code": null, "repo_url": "https://github.com/smallcloudai/refact", "homepage_url": "https://refact.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250611_Refact_Agent_claude-4-sonnet.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 35.59, "resolved_count": 184, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://refact.ai", "observed_at": "2025-06-11T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "openhands-versa-claude4", "name": "Openhands Versa Claude4", "vendor": "Anthropic"}, "success_rate": 34.43, "resolved_count": 178, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/adityasoni9998/OpenHands-Versa", "observed_at": "2025-05-28T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "guirepair", "name": "GUIRepair", "vendor": "GUIRepair", "scaffold_code": null, "repo_url": null, "homepage_url": "https://sites.google.com/view/guirepair", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250531_GUIRepair_gpt4o.png"}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "success_rate": 33.85, "resolved_count": 175, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://sites.google.com/view/guirepair", "observed_at": "2025-05-31T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "success_rate": 31.33, "resolved_count": 162, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/adityasoni9998/OpenHands-Versa", "observed_at": "2025-05-09T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "guirepair", "name": "GUIRepair", "vendor": "GUIRepair", "scaffold_code": null, "repo_url": null, "homepage_url": "https://sites.google.com/view/guirepair", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250531_GUIRepair_gpt4o.png"}, "model": {"slug": "gpt-4-1-20250414", "name": "GPT-4.1 (2025-04-14)", "vendor": "OpenAI"}, "success_rate": 31.14, "resolved_count": 161, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://sites.google.com/view/guirepair", "observed_at": "2025-05-31T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "guirepair", "name": "GUIRepair", "vendor": "GUIRepair", "scaffold_code": null, "repo_url": null, "homepage_url": "https://sites.google.com/view/guirepair", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250531_GUIRepair_gpt4o.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 30.37, "resolved_count": 157, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://sites.google.com/view/guirepair", "observed_at": "2025-05-31T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "swe-agent-multimodal", "name": "SWE-agent (multimodal)", "vendor": "Princeton NLP", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_SWE-agent_M_claude3.5.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 12.19, "resolved_count": 63, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 12.19, "resolved_count": 63, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "swe-agent-javascript", "name": "SWE-agent (JavaScript)", "vendor": "Princeton NLP", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_SWE-agent_JS_gpt4o.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 11.99, "resolved_count": 62, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 11.99, "resolved_count": 62, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "swe-agent-multimodal", "name": "SWE-agent (multimodal)", "vendor": "Princeton NLP", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_SWE-agent_M_claude3.5.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 11.41, "resolved_count": 59, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "swe-agent-javascript", "name": "SWE-agent (JavaScript)", "vendor": "Princeton NLP", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_SWE-agent_JS_gpt4o.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 9.28, "resolved_count": 48, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 14, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 6.19, "resolved_count": 32, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-10-06T00:00:00+00:00"}, {"rank": 15, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 3.09, "resolved_count": 16, "total_count": 517, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multimodal public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-10-06T00:00:00+00:00"}]}}