{"benchmark": {"slug": "aider-refactor", "name": "Aider Refactor", "description": "Aider Refactor measures refactoring tasks in Aider's terminal pair-programming loop (`aider_leaderboard` source, harness `aider`).", "category": "eval", "split": null, "language": null, "task_count": null, "homepage_url": "https://aider.chat/docs/leaderboards/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://aider.chat/docs/leaderboards/", "is_public_leaderboard": true, "harness_count": 1, "model_count": 13, "score_count": 13, "best_success_rate": 92.1}, "protocol": {"eval_protocol": "Aider refactor leaderboard", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 13}, "rankings": {"benchmark": {"slug": "aider-refactor", "name": "Aider Refactor", "description": "Aider Refactor measures refactoring tasks in Aider's terminal pair-programming loop (`aider_leaderboard` source, harness `aider`).", "category": "eval", "split": null, "language": null, "task_count": null, "homepage_url": "https://aider.chat/docs/leaderboards/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://aider.chat/docs/leaderboards/", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 92.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1-preview", "name": "o1 preview", "vendor": "OpenAI"}, "success_rate": 75.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-opus-20240229", "name": "Claude 3 Opus (2024-02-29)", "vendor": "Anthropic"}, "success_rate": 72.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-04T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-sonnet-20240620", "name": "Claude 3.5 Sonnet (2024-06-20)", "vendor": "Anthropic"}, "success_rate": 64.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-01T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o", "name": "GPT-4o", "vendor": "openai"}, "success_rate": 62.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-13T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-1106", "name": "GPT-4.1106", "vendor": "OpenAI"}, "success_rate": 50.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-08T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-1-5-pro", "name": "Gemini 1.5 Pro", "vendor": "Google"}, "success_rate": 49.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-04T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 49.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-08-06T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1-mini", "name": "o1 mini", "vendor": "OpenAI"}, "success_rate": 44.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-0125", "name": "GPT-4.0125", "vendor": "OpenAI"}, "success_rate": 33.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-08T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-coder-v2-0724-deprecated", "name": "DeepSeek Coder V2 0724 (deprecated)", "vendor": "DeepSeek"}, "success_rate": 32.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-chat-v2-5", "name": "DeepSeek Chat V2.5", "vendor": "DeepSeek"}, "success_rate": 31.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-05T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 21.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-10T00:00:00+00:00"}]}}