{"harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "description": "Aider is a mature open-source coding agent focused on pair programming in the terminal: unified diffs, repository map context, and git integration. Public YAML leaderboards on aider.chat publish harness\u00d7model pass rates for Polyglot (225 Exercism tasks), Edit, and Refactor tracks \u2014 ingested as `source = aider_leaderboard`.\n\nOn SWE-bench Verified, Aider also appears as a community harness submission (`swe_bench_site`) with vendor-chosen models. Artificial Analysis publishes component scores where Aider is the agent product.\n\nWhen reading scores, note whether the row is SWE-bench Verified (issue resolution %) or an Aider-native benchmark (edit accuracy, polyglot pass rate) \u2014 filter the scores table by benchmark slug.", "card_summary": "Terminal pair-programming agent \u2014 Polyglot, Edit, Refactor YAML leaderboards plus SWE-bench submissions.", "homepage_url": "https://aider.chat/", "repo_url": "https://github.com/Aider-AI/aider", "docs_url": null, "scaffold_code": null, "harbor_agent_name": null, "api_protocol": "openai", "status": "validated", "supports_rl": true, "score_count": 157, "best_success_rate": 92.1, "github_stars": 24800, "popularity_tier": null, "hrl_score": 35.0, "hrl_rank": 14, "catalog_token_volume": 0, "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png", "aa_coding_index": null, "aa_mean_cost_usd": null, "aa_mean_total_tokens": null}, "scores": [{"success_rate": 92.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 88.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-23T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-5-high", "name": "GPT-5 (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 86.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 84.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-28T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o3-pro-high", "name": "o3 pro (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 84.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 84.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-18T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1", "name": "o1", "vendor": "openai"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 83.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-2-5-pro-preview", "name": "Gemini 2.5 Pro Preview", "vendor": "google"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 81.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o3-high", "name": "o3 (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 81.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-5-low", "name": "GPT-5 (low)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 80.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-exp-1206-whole", "name": "Gemini Exp 1206 (whole)", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 79.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1-preview", "name": "o1 preview", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 79.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-07-11T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "grok-4-high", "name": "GROK 4 (high)", "vendor": "xAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 78.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-27T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "ensemble-o3-high-gpt-4-1", "name": "Ensemble (o3 (high), GPT-4.1)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 77.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-sonnet-20240620", "name": "Claude 3.5 Sonnet (2024-06-20)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 76.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 75.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1-preview", "name": "o1 preview", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 75.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-haiku-20241022", "name": "Claude 3.5 Haiku (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 74.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-10-03T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-v3-2-exp-reasoner", "name": "DeepSeek V3.2 Exp (Reasoner)", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 72.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-13T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 72.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-coder-v2-0724", "name": "DeepSeek Coder V2 0724", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 72.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-28T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-32b", "name": "Qwen2.5 Coder 32b", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 72.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-opus-20240229", "name": "Claude 3 Opus (2024-02-29)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 72.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-05T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-v2-5", "name": "DeepSeek V2.5", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 72.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "chatgpt-4o", "name": "ChatGPT-4o", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 72.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-v2-5-1210", "name": "DeepSeek V2.5 1210", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 72.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-16T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o4-mini-high", "name": "o4 mini (high)", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 72.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-4-opus-thinking", "name": "Claude 4 Opus (thinking)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-06-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-r1-0528", "name": "DeepSeek R1 0528", "vendor": "deepseek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-08-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-09T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder 32B Instruct", "vendor": "qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-20T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-20241120", "name": "GPT-4o (2024-11-20)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 70.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-10-03T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-v3-2-exp-chat", "name": "DeepSeek V3.2 Exp (Chat)", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 69.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-19T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-chat-v2-0628", "name": "DeepSeek Chat V2 0628", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 69.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-11T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 69.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-29T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-14b-instruct", "name": "Qwen2.5 Coder 14B Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 69.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-exp-1206-diff", "name": "Gemini Exp 1206 (diff)", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 68.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-01T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-opus-20240229", "name": "Claude 3 Opus (2024-02-29)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 67.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2023-12-16T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-0613", "name": "GPT-4.0613", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 66.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-05T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "dracarys2-72b-instruct", "name": "Dracarys2 72B Instruct", "vendor": "Abacus"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 66.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-01-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-0125", "name": "GPT-4.0125", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 66.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-0314", "name": "GPT-4.0314", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 66.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-3-1-405b-instruct-whole", "name": "Llama 3.1 405b Instruct (whole)", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-1-5-pro", "name": "Gemini 1.5 Pro", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-1106", "name": "GPT-4.1106", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-20T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen-2-5-72b-instruct-bf16", "name": "QWEN 2.5 72b Instruct (bf16)", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-20T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "mistral-large-2411", "name": "Mistral Large (2411)", "vendor": "Mistral"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-20T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen-qwen-2-5-coder-32b-instruct", "name": "Qwen/qwen 2.5 Coder 32b Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-11T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "yi-lightning", "name": "Yi Lightning", "vendor": "01.AI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 64.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-7-sonnet-thinking-20250219", "name": "Claude 3.7 Sonnet (thinking) (2025-02-19)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 64.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-23T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "ensemble-deepseek-r1-claude-3-5-sonnet-2024-10-22", "name": "Ensemble (DeepSeek R1, Claude 3.5 Sonnet (2024-10-22))", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 64.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-01T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-sonnet-20240620", "name": "Claude 3.5 Sonnet (2024-06-20)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 63.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-23T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-3-1-405b-instruct-diff", "name": "Llama 3.1 405b Instruct (diff)", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 63.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-08-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "nousresearch-hermes-3-llama-3-1-405b", "name": "Nousresearch/hermes 3 Llama 3.1 405b", "vendor": "nousresearch"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 63.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-07T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-1-coder-7b-instruct-gguf-q8-0-32k", "name": "Qwen2.5.1 Coder 7B Instruct GGUF Q8 0 32k", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 62.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-13T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o", "name": "GPT-4o", "vendor": "openai"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 61.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1-high", "name": "o1 (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 61.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-27T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-14b", "name": "Qwen2.5 Coder 14b", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 61.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-4-sonnet-thinking", "name": "Claude 4 Sonnet (thinking)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 60.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-15T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-exp-1114", "name": "Gemini Exp 1114", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 60.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-31T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o3-mini-high", "name": "o3 mini (high)", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 60.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "mistral-large-2-2407", "name": "Mistral Large 2 (2407)", "vendor": "Mistral"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 59.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-09T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen3-235b-a22b-diff-no-think-alibaba-api", "name": "Qwen3 235B A22B Diff, No Think, Alibaba API", "vendor": "Qwen"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 59.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-3-3-70b-instruct", "name": "Llama 3.3 70B Instruct", "vendor": "meta-llama"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 59.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-07-17T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "kimi-k2", "name": "KIMI K2", "vendor": "moonshotai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 58.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-3-1-70b-instruct", "name": "Llama 3.1 70b Instruct", "vendor": "meta-llama"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 58.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-13T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "grok-2", "name": "GROK 2", "vendor": "xAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 58.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-20T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-32b-instruct-q8-0", "name": "Qwen2.5 32b Instruct Q8 0", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 57.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-16T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-7b-instruct", "name": "Qwen2.5 Coder 7B Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 57.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-exp-1121", "name": "Gemini Exp 1121", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 57.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-12T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 56.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-20T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-r1", "name": "Deepseek R1", "vendor": "deepseek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 56.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-3-5", "name": "GPT-3.5", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 55.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-72b-instruct", "name": "Qwen2 72B Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 55.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-18T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-mini", "name": "GPT-4o mini", "vendor": "openai"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 55.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-v3-0324", "name": "DeepSeek V3 0324", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 55.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-2-5-flash-preview", "name": "Gemini 2.5 Flash Preview", "vendor": "Google"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 54.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-03-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-sonnet-20240229", "name": "Claude 3 Sonnet (2024-02-29)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 54.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-13T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "grok-2-mini", "name": "GROK 2 Mini", "vendor": "xAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 54.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-16T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-3-1-nemotron-70b-instruct-hf", "name": "Llama 3.1 Nemotron 70B Instruct HF", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 54.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "quasar-alpha", "name": "Quasar Alpha", "vendor": "OpenRouter"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 54.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-32b", "name": "Qwen2.5 32b", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 54.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "nova-pro", "name": "Nova Pro", "vendor": "Amazon"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 53.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-31T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o3-mini", "name": "o3 mini", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 53.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "grok-3-beta", "name": "GROK 3 Beta", "vendor": "xAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 52.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "optimus-alpha", "name": "Optimus Alpha", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 52.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-14T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 51.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-19T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-7b-instruct-q8-0", "name": "Qwen2.5 Coder 7b Instruct Q8 0", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 51.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-17T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 51.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-09T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codestral-2405", "name": "Codestral 2405", "vendor": "Mistral"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 50.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-1106", "name": "GPT-4.1106", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 49.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-72b-instruct-q8-0", "name": "Qwen2 72b Instruct Q8 0", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 49.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-1-5-pro", "name": "Gemini 1.5 Pro", "vendor": "Google"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 49.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-08-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 49.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "grok-3-mini-beta-high", "name": "GROK 3 Mini Beta (high)", "vendor": "xAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 49.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-03T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama3-70b-8192", "name": "Llama3 70b 8192", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 48.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-chat-v3-prev", "name": "DeepSeek Chat V3 (prev)", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 48.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codestral-22b-v0-1-q8-0", "name": "Codestral 22b V0.1 Q8 0", "vendor": "Mistral"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 48.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codestral-22b-v0-1-q4-k-m", "name": "Codestral 22B V0.1 Q4 K M", "vendor": "Mistral"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 47.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-01T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-haiku-20240307", "name": "Claude 3 Haiku (2024-03-07)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 45.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-28T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codestral", "name": "Codestral", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 45.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-29T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "chatgpt-4o-20250329", "name": "ChatGPT-4o (2025-03-29)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 45.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "yi-coder-9b-chat-q4-0", "name": "Yi Coder 9b Chat Q4 0", "vendor": "01.AI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 44.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-27T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-5", "name": "GPT-4.5", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 44.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1-mini", "name": "o1 mini", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 44.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-07T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "wizardlm-2-8x22b", "name": "WizardLM 2 8x22B", "vendor": "microsoft"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 43.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-01T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "yi-coder-9b-chat", "name": "Yi Coder 9b Chat", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 42.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "reflection-70b", "name": "Reflection 70B", "vendor": "Reflection"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 41.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-08-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-oss-120b-high", "name": "GPT Oss 120b High", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 40.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-05-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen3-32b", "name": "Qwen3 32B", "vendor": "qwen"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 39.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-04T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-3b-instruct", "name": "Qwen2.5 Coder 3B Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 38.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-1-5-flash", "name": "Gemini 1.5 Flash", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 38.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-11T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "command-r-08-24", "name": "Command R (08 24)", "vendor": "Cohere"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 38.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-01T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "mistral-small", "name": "Mistral Small", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 38.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-exp-1206", "name": "Gemini Exp 1206", "vendor": "Google"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 37.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-07T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen1-5-110b-chat", "name": "Qwen1.5 110b Chat", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 37.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-3-1-8b-instruct", "name": "Llama 3.1 8b Instruct", "vendor": "meta-llama"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 36.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-09T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemma2-27b-instruct-q8-0", "name": "Gemma2 27b Instruct Q8 0", "vendor": "Google"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 35.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-25T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-2-0-pro", "name": "Gemini 2.0 Pro", "vendor": "Google"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 34.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "grok-3-mini-beta-low", "name": "GROK 3 Mini Beta (low)", "vendor": "xAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 34.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codeqwen-7b-chat-v1-5-q8-0", "name": "Codeqwen 7b Chat V1.5 Q8 0", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 33.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-08T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-0125", "name": "GPT-4.0125", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 33.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "mistral-nemo-12b-instruct-2407-q4-k-m", "name": "Mistral Nemo 12b Instruct 2407 Q4 K M", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 32.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1-mini", "name": "o1 mini", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 32.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-coder-v2-0724-deprecated", "name": "DeepSeek Coder V2 0724 (deprecated)", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 32.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-14T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-1-mini", "name": "GPT-4.1 mini", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 32.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-29T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codegeex4", "name": "Codegeex4", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 31.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "command-r-plus", "name": "Command R Plus", "vendor": "Cohere"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 31.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-11T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-1-5b-instruct", "name": "Qwen2.5 Coder 1.5B Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 31.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-05T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-chat-v2-5", "name": "DeepSeek Chat V2.5", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 30.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "wojtek-opencodeinterpreter-6-7b", "name": "wojtek/opencodeinterpreter 6.7b", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 30.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "hermes3-8b-llama3-1", "name": "Hermes3 8b Llama3.1", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 28.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "claude-3-5-haiku-20241022", "name": "Claude 3.5 Haiku (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 27.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-02-15T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "chatgpt-4o-20250215", "name": "ChatGPT-4o (2025-02-15)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 27.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "o1-mini", "name": "o1 mini", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 26.33, "resolved_count": 79, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/paul-gauthier/aider", "observed_at": "2024-05-23T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "ensemble-gpt-4o-claude-3-opus", "name": "Ensemble (GPT-4o, Claude 3 Opus)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 26.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama3-2-3b-instruct", "name": "Llama3.2 3b Instruct", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 26.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-28T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "tulu3", "name": "Tulu3", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 26.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-07T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "ensemble-qwq-32b-qwen-2-5-coder-instruct", "name": "Ensemble (QwQ 32B, QWEN 2.5 Coder Instruct)", "vendor": "Qwen"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 23.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 22.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-01T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "hermes3", "name": "Hermes3", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 21.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-28T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen-max-2025-01-25", "name": "QWEN Max 2025 01 25", "vendor": "Qwen"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 21.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 20.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwq-32b", "name": "QwQ 32B", "vendor": "Qwen"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 20.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-28T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "granite3-dense-8b", "name": "Granite3 Dense 8b", "vendor": "Meta"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 18.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-30T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-20241120", "name": "GPT-4o (2024-11-20)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 18.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 17.8, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "deepseek-chat-v2-5", "name": "DeepSeek Chat V2.5", "vendor": "DeepSeek"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 16.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-26T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder 32B Instruct", "vendor": "qwen"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 15.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-06T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "llama-4-maverick", "name": "Llama 4 Maverick", "vendor": "meta-llama"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 14.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-29T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "qwen2-5-coder-0-5b-instruct", "name": "Qwen2.5 Coder 0.5B Instruct", "vendor": "Qwen"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 12.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-23T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "yi-lightning", "name": "Yi Lightning", "vendor": "01.AI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 12.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-14T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "command-a-03-2025-quality", "name": "Command A 03 2025 Quality", "vendor": "Cohere"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 11.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-13T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "codestral-25-01", "name": "Codestral 25.01", "vendor": "Mistral"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 10.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-19T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "openhands-lm-32b-v0-1", "name": "openhands-lm-32b-v0.1", "vendor": "All Hands AI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 8.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-04-14T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4-1-nano", "name": "GPT-4.1 nano", "vendor": "openai"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 4.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-03-15T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gemma-3-27b-it", "name": "Gemma 3 27b It", "vendor": "google"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 3.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, "model": {"slug": "gpt-4o-mini-20240718", "name": "GPT-4o mini (2024-07-18)", "vendor": "OpenAI"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}], "benchmarks": [{"slug": "aider-edit", "name": "Aider Edit", "category": "eval"}, {"slug": "aider-polyglot", "name": "Aider Polyglot", "category": "eval"}, {"slug": "aider-refactor", "name": "Aider Refactor", "category": "eval"}, {"slug": "swe-bench-lite", "name": "SWE-bench Lite", "category": "eval"}]}