{"benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot", "description": "Aider Polyglot benchmarks multi-language editing via Aider on 225 Exercism exercises. Scores ingested from public YAML (`aider_leaderboard` source, harness `aider`, metric `pass_rate_2`).\n\nMeasures Aider's edit loop \u2014 not SWE-bench issue resolution. Top public runs approach ~88% pass_rate_2 on recent frontier models.", "category": "eval", "split": null, "language": null, "task_count": 225, "homepage_url": "https://aider.chat/docs/leaderboards/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://aider.chat/docs/leaderboards/", "is_public_leaderboard": true, "harness_count": 1, "model_count": 59, "score_count": 59, "best_success_rate": 88.0}, "protocol": {"eval_protocol": "Aider polyglot leaderboard", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 59}, "rankings": {"benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot", "description": "Aider Polyglot benchmarks multi-language editing via Aider on 225 Exercism exercises. Scores ingested from public YAML (`aider_leaderboard` source, harness `aider`, metric `pass_rate_2`).\n\nMeasures Aider's edit loop \u2014 not SWE-bench issue resolution. Top public runs approach ~88% pass_rate_2 on recent frontier models.", "category": "eval", "split": null, "language": null, "task_count": 225, "homepage_url": "https://aider.chat/docs/leaderboards/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://aider.chat/docs/leaderboards/", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-5-high", "name": "GPT-5 (high)", "vendor": "OpenAI"}, "success_rate": 88.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-23T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "success_rate": 86.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-25T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o3-pro-high", "name": "o3 pro (high)", "vendor": "OpenAI"}, "success_rate": 84.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-28T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-2-5-pro-preview", "name": "Gemini 2.5 Pro Preview", "vendor": "google"}, "success_rate": 83.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-06T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o3-high", "name": "o3 (high)", "vendor": "OpenAI"}, "success_rate": 81.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-25T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-5-low", "name": "GPT-5 (low)", "vendor": "OpenAI"}, "success_rate": 81.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-25T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "grok-4-high", "name": "GROK 4 (high)", "vendor": "xAI"}, "success_rate": 79.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-07-11T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "ensemble-o3-high-gpt-4-1", "name": "Ensemble (o3 (high), GPT-4.1)", "vendor": "OpenAI"}, "success_rate": 78.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-27T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "success_rate": 76.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-25T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-v3-2-exp-reasoner", "name": "DeepSeek V3.2 Exp (Reasoner)", "vendor": "DeepSeek"}, "success_rate": 74.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-10-03T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o4-mini-high", "name": "o4 mini (high)", "vendor": "openai"}, "success_rate": 72.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-16T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-4-opus-thinking", "name": "Claude 4 Opus (thinking)", "vendor": "Anthropic"}, "success_rate": 72.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-25T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-r1-0528", "name": "DeepSeek R1 0528", "vendor": "deepseek"}, "success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-06T00:00:00+00:00"}, {"rank": 14, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-v3-2-exp-chat", "name": "DeepSeek V3.2 Exp (Chat)", "vendor": "DeepSeek"}, "success_rate": 70.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-10-03T00:00:00+00:00"}, {"rank": 15, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-7-sonnet-thinking-20250219", "name": "Claude 3.7 Sonnet (thinking) (2025-02-19)", "vendor": "Anthropic"}, "success_rate": 64.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-24T00:00:00+00:00"}, {"rank": 16, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "ensemble-deepseek-r1-claude-3-5-sonnet-2024-10-22", "name": "Ensemble (DeepSeek R1, Claude 3.5 Sonnet (2024-10-22))", "vendor": "Anthropic"}, "success_rate": 64.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-23T00:00:00+00:00"}, {"rank": 17, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1-high", "name": "o1 (high)", "vendor": "OpenAI"}, "success_rate": 61.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00"}, {"rank": 18, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-4-sonnet-thinking", "name": "Claude 4 Sonnet (thinking)", "vendor": "Anthropic"}, "success_rate": 61.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-24T00:00:00+00:00"}, {"rank": 19, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o3-mini-high", "name": "o3 mini (high)", "vendor": "openai"}, "success_rate": 60.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-31T00:00:00+00:00"}, {"rank": 20, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen3-235b-a22b-diff-no-think-alibaba-api", "name": "Qwen3 235B A22B Diff, No Think, Alibaba API", "vendor": "Qwen"}, "success_rate": 59.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-09T00:00:00+00:00"}, {"rank": 21, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "kimi-k2", "name": "KIMI K2", "vendor": "moonshotai"}, "success_rate": 59.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-07-17T00:00:00+00:00"}, {"rank": 22, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-r1", "name": "Deepseek R1", "vendor": "deepseek"}, "success_rate": 56.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-20T00:00:00+00:00"}, {"rank": 23, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-v3-0324", "name": "DeepSeek V3 0324", "vendor": "DeepSeek"}, "success_rate": 55.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-24T00:00:00+00:00"}, {"rank": 24, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-2-5-flash-preview", "name": "Gemini 2.5 Flash Preview", "vendor": "Google"}, "success_rate": 55.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-25T00:00:00+00:00"}, {"rank": 25, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "quasar-alpha", "name": "Quasar Alpha", "vendor": "OpenRouter"}, "success_rate": 54.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-04T00:00:00+00:00"}, {"rank": 26, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o3-mini", "name": "o3 mini", "vendor": "openai"}, "success_rate": 53.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-31T00:00:00+00:00"}, {"rank": 27, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "grok-3-beta", "name": "GROK 3 Beta", "vendor": "xAI"}, "success_rate": 53.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00"}, {"rank": 28, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "optimus-alpha", "name": "Optimus Alpha", "vendor": "OpenAI"}, "success_rate": 52.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00"}, {"rank": 29, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "success_rate": 52.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-14T00:00:00+00:00"}, {"rank": 30, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 51.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-17T00:00:00+00:00"}, {"rank": 31, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "grok-3-mini-beta-high", "name": "GROK 3 Mini Beta (high)", "vendor": "xAI"}, "success_rate": 49.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00"}, {"rank": 32, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-chat-v3-prev", "name": "DeepSeek Chat V3 (prev)", "vendor": "DeepSeek"}, "success_rate": 48.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-25T00:00:00+00:00"}, {"rank": 33, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "chatgpt-4o-20250329", "name": "ChatGPT-4o (2025-03-29)", "vendor": "OpenAI"}, "success_rate": 45.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-29T00:00:00+00:00"}, {"rank": 34, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-5", "name": "GPT-4.5", "vendor": "OpenAI"}, "success_rate": 44.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-27T00:00:00+00:00"}, {"rank": 35, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-oss-120b-high", "name": "GPT Oss 120b High", "vendor": "OpenAI"}, "success_rate": 41.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-06T00:00:00+00:00"}, {"rank": 36, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen3-32b", "name": "Qwen3 32B", "vendor": "qwen"}, "success_rate": 40.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-08T00:00:00+00:00"}, {"rank": 37, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-exp-1206", "name": "Gemini Exp 1206", "vendor": "Google"}, "success_rate": 38.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-22T00:00:00+00:00"}, {"rank": 38, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-2-0-pro", "name": "Gemini 2.0 Pro", "vendor": "Google"}, "success_rate": 35.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-25T00:00:00+00:00"}, {"rank": 39, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "grok-3-mini-beta-low", "name": "GROK 3 Mini Beta (low)", "vendor": "xAI"}, "success_rate": 34.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00"}, {"rank": 40, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1-mini", "name": "o1 mini", "vendor": "OpenAI"}, "success_rate": 32.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-22T00:00:00+00:00"}, {"rank": 41, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-1-mini", "name": "GPT-4.1 mini", "vendor": "openai"}, "success_rate": 32.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-14T00:00:00+00:00"}, {"rank": 42, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-haiku-20241022", "name": "Claude 3.5 Haiku (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 28.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00"}, {"rank": 43, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "chatgpt-4o-20250215", "name": "ChatGPT-4o (2025-02-15)", "vendor": "OpenAI"}, "success_rate": 27.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-15T00:00:00+00:00"}, {"rank": 44, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "ensemble-qwq-32b-qwen-2-5-coder-instruct", "name": "Ensemble (QwQ 32B, QWEN 2.5 Coder Instruct)", "vendor": "Qwen"}, "success_rate": 26.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-07T00:00:00+00:00"}, {"rank": 45, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 23.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-30T00:00:00+00:00"}, {"rank": 46, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen-max-2025-01-25", "name": "QWEN Max 2025 01 25", "vendor": "Qwen"}, "success_rate": 21.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-28T00:00:00+00:00"}, {"rank": 47, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwq-32b", "name": "QwQ 32B", "vendor": "Qwen"}, "success_rate": 20.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-06T00:00:00+00:00"}, {"rank": 48, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-20241120", "name": "GPT-4o (2024-11-20)", "vendor": "OpenAI"}, "success_rate": 18.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-30T00:00:00+00:00"}, {"rank": 49, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "success_rate": 18.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-21T00:00:00+00:00"}, {"rank": 50, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-chat-v2-5", "name": "DeepSeek Chat V2.5", "vendor": "DeepSeek"}, "success_rate": 17.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00"}, {"rank": 51, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder 32B Instruct", "vendor": "qwen"}, "success_rate": 16.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-26T00:00:00+00:00"}, {"rank": 52, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-4-maverick", "name": "Llama 4 Maverick", "vendor": "meta-llama"}, "success_rate": 15.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-06T00:00:00+00:00"}, {"rank": 53, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "yi-lightning", "name": "Yi Lightning", "vendor": "01.AI"}, "success_rate": 12.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-23T00:00:00+00:00"}, {"rank": 54, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "command-a-03-2025-quality", "name": "Command A 03 2025 Quality", "vendor": "Cohere"}, "success_rate": 12.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-14T00:00:00+00:00"}, {"rank": 55, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codestral-25-01", "name": "Codestral 25.01", "vendor": "Mistral"}, "success_rate": 11.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-13T00:00:00+00:00"}, {"rank": 56, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "openhands-lm-32b-v0-1", "name": "openhands-lm-32b-v0.1", "vendor": "All Hands AI"}, "success_rate": 10.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-19T00:00:00+00:00"}, {"rank": 57, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-1-nano", "name": "GPT-4.1 nano", "vendor": "openai"}, "success_rate": 8.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-14T00:00:00+00:00"}, {"rank": 58, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemma-3-27b-it", "name": "Gemma 3 27b It", "vendor": "google"}, "success_rate": 4.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-15T00:00:00+00:00"}, {"rank": 59, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-mini-20240718", "name": "GPT-4o mini (2024-07-18)", "vendor": "OpenAI"}, "success_rate": 3.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00"}]}}