{"benchmark": {"slug": "swe-bench-multilingual", "name": "SWE-bench Multilingual", "description": "SWE-bench Multilingual evaluates issue fixing across multiple languages. The swebench.com board uses mini-SWE-agent for every model (like bash-only on Verified) so rows isolate foundation-model capability.\n\nCatalog: harness `mini-swe-agent`, benchmark `swe-bench-multilingual`, `source = swe_bench_site`.", "category": "eval", "split": null, "language": "multilingual", "task_count": 300, "homepage_url": "https://www.swebench.com/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/", "is_public_leaderboard": true, "harness_count": 1, "model_count": 13, "score_count": 13, "best_success_rate": 72.7}, "protocol": {"eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 13}, "rankings": {"benchmark": {"slug": "swe-bench-multilingual", "name": "SWE-bench Multilingual", "description": "SWE-bench Multilingual evaluates issue fixing across multiple languages. The swebench.com board uses mini-SWE-agent for every model (like bash-only on Verified) so rows isolate foundation-model capability.\n\nCatalog: harness `mini-swe-agent`, benchmark `swe-bench-multilingual`, `source = swe_bench_site`.", "category": "eval", "split": null, "language": "multilingual", "task_count": 300, "homepage_url": "https://www.swebench.com/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-3-flash", "name": "Gemini 3 Flash", "vendor": "Google"}, "success_rate": 72.7, "resolved_count": 218, "total_count": 300, "cost_usd": 0.3518355868333333, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://ai.google.dev/gemini-api/docs/models", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-6-opus", "name": "Claude 4.6 Opus", "vendor": "Anthropic"}, "success_rate": 72.0, "resolved_count": 216, "total_count": 300, "cost_usd": 0.6629735766666667, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/news/claude-opus-4-5", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "success_rate": 70.7, "resolved_count": 212, "total_count": 300, "cost_usd": 0.8335529758333333, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/news/claude-opus-4-5", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "glm-5", "name": "GLM 5", "vendor": "z-ai"}, "success_rate": 69.7, "resolved_count": 209, "total_count": 300, "cost_usd": 0.6424743668666667, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://z.ai", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gemini-3-pro", "name": "Gemini 3 Pro", "vendor": "Google"}, "success_rate": 68.7, "resolved_count": 206, "total_count": 300, "cost_usd": 1.0210855846666667, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://ai.google.dev/gemini-api/docs/models", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "minimax-m2-5", "name": "MiniMax M2.5", "vendor": "minimax"}, "success_rate": 68.3, "resolved_count": 205, "total_count": 300, "cost_usd": 0.10121049956228956, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://www.minimax.io", "observed_at": "2026-02-16T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "kimi-k2-5", "name": "KIMI K2.5", "vendor": "moonshotai"}, "success_rate": 67.3, "resolved_count": 202, "total_count": 300, "cost_usd": 0.6928368501333334, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://moonshotai.github.io/Kimi-K2/", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 67.0, "resolved_count": 201, "total_count": 300, "cost_usd": 0.669532236, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/news/claude-opus-4-5", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-2-high", "name": "GPT-5.2 (high)", "vendor": "OpenAI"}, "success_rate": 66.7, "resolved_count": 200, "total_count": 300, "cost_usd": 0.5364109905, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://platform.openai.com", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-2-codex", "name": "GPT-5.2 codex", "vendor": "openai"}, "success_rate": 66.3, "resolved_count": 199, "total_count": 300, "cost_usd": 0.6621715578333333, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://platform.openai.com/docs/models/gpt-5", "observed_at": "2026-02-20T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "claude-4-5-haiku", "name": "Claude 4.5 Haiku", "vendor": "Anthropic"}, "success_rate": 64.7, "resolved_count": 194, "total_count": 300, "cost_usd": 0.3794902823333333, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/news/claude-opus-4-5", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "deepseek-v3-2", "name": "DeepSeek V3.2", "vendor": "deepseek"}, "success_rate": 59.0, "resolved_count": 177, "total_count": 300, "cost_usd": 0.3838255249, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://api-docs.deepseek.com", "observed_at": "2026-02-13T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "mini-swe-agent", "name": "mini-SWE-agent", "vendor": "anthropic", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/mini-swe-agent", "homepage_url": "https://github.com/SWE-agent/mini-swe-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/mini-icon.svg"}, "model": {"slug": "gpt-5-mini", "name": "GPT-5 mini", "vendor": "openai"}, "success_rate": 39.7, "resolved_count": 119, "total_count": 300, "cost_usd": 0.051632880333333325, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Multilingual (mini-SWE-agent)", "source": "swe_bench_site", "source_url": "https://platform.openai.com", "observed_at": "2026-02-13T00:00:00+00:00"}]}}