{"benchmark": {"slug": "swe-bench-multimodal", "name": "SWE-bench Multimodal (OpenHands Index)", "description": "SWE-bench Multimodal in OpenHands Index covers UI-forward software engineering with visual context. Harness is fixed to OpenHands SDK (`openhands_index` source).\n\nDistinct from the smaller Multimodal board on swebench.com leaderboards.json (not ingested here).", "category": "eval", "split": null, "language": null, "task_count": null, "homepage_url": "https://index.openhands.dev/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://index.openhands.dev/", "is_public_leaderboard": true, "harness_count": 1, "model_count": 33, "score_count": 33, "best_success_rate": 70.6}, "protocol": {"eval_protocol": "OpenHands SDK v1.8.3", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 12}, "rankings": {"benchmark": {"slug": "swe-bench-multimodal", "name": "SWE-bench Multimodal (OpenHands Index)", "description": "SWE-bench Multimodal in OpenHands Index covers UI-forward software engineering with visual context. Harness is fixed to OpenHands SDK (`openhands_index` source).\n\nDistinct from the smaller Multimodal board on swebench.com leaderboards.json (not ingested here).", "category": "eval", "split": null, "language": null, "task_count": null, "homepage_url": "https://index.openhands.dev/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://index.openhands.dev/", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "fable-5", "name": "Fable 5", "vendor": "Anthropic"}, "success_rate": 70.6, "resolved_count": null, "total_count": null, "cost_usd": 4.39, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.18.1", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/53fd0c2a-c7bf-4efa-b74d-14258d81ed7a", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 2, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-8-opus", "name": "Claude 4.8 Opus", "vendor": "Anthropic"}, "success_rate": 50.0, "resolved_count": null, "total_count": null, "cost_usd": 1.81, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.18.1", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/472e594d-c161-4836-8a93-474c2e1e4168", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 3, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-7-opus", "name": "Claude 4.7 Opus", "vendor": "Anthropic"}, "success_rate": 48.5, "resolved_count": null, "total_count": null, "cost_usd": 2.83, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/cfe39593-1e41-44ba-bf2f-a29e64a51432", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 4, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gemini-3-1-pro", "name": "Gemini 3.1 Pro", "vendor": "Google"}, "success_rate": 44.1, "resolved_count": null, "total_count": null, "cost_usd": 1.24, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/cca1e142-0f1b-45ef-95fc-fde8e4c9d3aa", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 5, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-6-opus", "name": "Claude 4.6 Opus", "vendor": "Anthropic"}, "success_rate": 41.8, "resolved_count": null, "total_count": null, "cost_usd": 2.37, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/0fe00101-1f0a-4698-8a43-70fd665f4936", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 6, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "success_rate": 41.2, "resolved_count": null, "total_count": null, "cost_usd": 2.54, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 7, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "glm-5-1", "name": "GLM 5 1", "vendor": "Zhipu"}, "success_rate": 41.2, "resolved_count": null, "total_count": null, "cost_usd": 6.92, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/26dfa179-9ab7-4b15-988d-b7ed2bf91328", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 8, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "kimi-k2-6", "name": "KIMI K2.6", "vendor": "Moonshot"}, "success_rate": 41.2, "resolved_count": null, "total_count": null, "cost_usd": 0.64, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/bf318301-2c72-4fe2-a705-d104161d05c0", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 9, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "vendor": "google"}, "success_rate": 39.7, "resolved_count": null, "total_count": null, "cost_usd": 1.99, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.24.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/20d43172-1fef-4d83-af1b-01784887e7ca", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 10, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gpt-5-5", "name": "GPT-5.5", "vendor": "openai"}, "success_rate": 38.2, "resolved_count": null, "total_count": null, "cost_usd": 2.81, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.18.1", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/55c33492-2ff8-4f59-ba0a-566bcdbb2cd4", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 11, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gpt-5-4", "name": "GPT-5.4", "vendor": "openai"}, "success_rate": 36.8, "resolved_count": null, "total_count": null, "cost_usd": 1.45, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.13.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/b8e55418-a747-461b-92b8-2981612e5650", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 12, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "minimax-m3", "name": "MiniMax M3", "vendor": "minimax"}, "success_rate": 36.8, "resolved_count": null, "total_count": null, "cost_usd": 0.35, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.18.1", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/5cbf2dac-ffe8-47d0-9af4-db172c42e7b7", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 13, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 36.8, "resolved_count": null, "total_count": null, "cost_usd": 1.89, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 14, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gemini-3-pro", "name": "Gemini 3 Pro", "vendor": "Google"}, "success_rate": 36.8, "resolved_count": null, "total_count": null, "cost_usd": 1.46, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 15, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "vendor": "deepseek"}, "success_rate": 36.8, "resolved_count": null, "total_count": null, "cost_usd": 0.3746, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.0.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/926fc133-2a36-4c9c-b40b-353aa98057a9", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 16, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gpt-5-2-codex", "name": "GPT-5.2 codex", "vendor": "openai"}, "success_rate": 35.9, "resolved_count": null, "total_count": null, "cost_usd": 2.97, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 17, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "glm-5", "name": "GLM 5", "vendor": "z-ai"}, "success_rate": 35.3, "resolved_count": null, "total_count": null, "cost_usd": 0.58, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/756a9904-f7f3-4c23-a663-8a7dd853d6dd", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 18, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "kimi-k2-5", "name": "KIMI K2.5", "vendor": "moonshotai"}, "success_rate": 32.8, "resolved_count": null, "total_count": null, "cost_usd": 1.6213, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 19, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "kimi-k2-thinking", "name": "KIMI K2 Thinking", "vendor": "moonshotai"}, "success_rate": 32.4, "resolved_count": null, "total_count": null, "cost_usd": 2.31, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 20, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gpt-5-2", "name": "GPT-5.2", "vendor": "openai"}, "success_rate": 30.9, "resolved_count": null, "total_count": null, "cost_usd": 2.77, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 21, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "qwen3-6-plus", "name": "Qwen3.6 Plus", "vendor": "qwen"}, "success_rate": 30.9, "resolved_count": null, "total_count": null, "cost_usd": 2.3158, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/7c984e26-6009-4208-819f-a9ba57d5476f", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 22, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-4-6-sonnet", "name": "Claude 4.6 Sonnet", "vendor": "Anthropic"}, "success_rate": 30.9, "resolved_count": null, "total_count": null, "cost_usd": 2.24, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/ed24d3b6-988c-4ef1-b709-b0f223173547", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 23, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "qwen3-coder-next", "name": "Qwen3 Coder Next", "vendor": "qwen"}, "success_rate": 30.9, "resolved_count": null, "total_count": null, "cost_usd": 1.52, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/8964f6e6-fcbd-4f26-a1ce-9fc657b8da43", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 24, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "deepseek-v3-2-reasoner", "name": "DeepSeek V3.2 Reasoner", "vendor": "DeepSeek"}, "success_rate": 27.9, "resolved_count": null, "total_count": null, "cost_usd": 0.19, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 25, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "minimax-2-7", "name": "Minimax 2.7", "vendor": "MiniMax"}, "success_rate": 27.9, "resolved_count": null, "total_count": null, "cost_usd": 0.388, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.14.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/161dd621-b70d-4089-afcf-ea2e3d13e874", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 26, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "qwen3-5-flash", "name": "Qwen3.5 Flash", "vendor": "Qwen"}, "success_rate": 27.9, "resolved_count": null, "total_count": null, "cost_usd": 2.6218, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.11.5", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/84c8dc14-d082-430d-b19c-e142de615385", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 27, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "minimax-m2-5", "name": "MiniMax M2.5", "vendor": "minimax"}, "success_rate": 25.0, "resolved_count": null, "total_count": null, "cost_usd": 0.5941, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v0.38.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/194e3e71-5874-45a5-96b8-220155ea1e24", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 28, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "trinity-large-thinking", "name": "Trinity Large Thinking", "vendor": "arcee-ai"}, "success_rate": 25.0, "resolved_count": null, "total_count": null, "cost_usd": 0.74, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.16.1", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/f327bc7a-c85b-4a89-b0c9-2da1532bcec4", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 29, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "qwen3-coder-480b", "name": "Qwen3 Coder 480B", "vendor": "Qwen"}, "success_rate": 23.5, "resolved_count": null, "total_count": null, "cost_usd": 2.09, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 30, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "gemini-3-flash", "name": "Gemini 3 Flash", "vendor": "Google"}, "success_rate": 22.1, "resolved_count": null, "total_count": null, "cost_usd": 0.8, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 31, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "glm-4-7", "name": "GLM 4.7", "vendor": "z-ai"}, "success_rate": 22.1, "resolved_count": null, "total_count": null, "cost_usd": 0.66, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 32, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "nemotron-3-super", "name": "Nemotron 3 Super", "vendor": "NVIDIA"}, "success_rate": 20.6, "resolved_count": null, "total_count": null, "cost_usd": 0.7679, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.14.0", "source": "openhands_index", "source_url": "https://laminar.sh/shared/evals/44cc701e-6ecf-4c70-9cca-4246cbd402f4", "observed_at": "2026-07-31T11:38:58.577710"}, {"rank": 33, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "minimax-m2-1", "name": "MiniMax M2.1", "vendor": "minimax"}, "success_rate": 16.2, "resolved_count": null, "total_count": null, "cost_usd": 1.4043, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "OpenHands SDK v1.8.3", "source": "openhands_index", "source_url": "https://index.openhands.dev/", "observed_at": "2026-07-31T11:38:58.577710"}]}}