{"benchmark": {"slug": "aider-edit", "name": "Aider Edit", "description": "Aider Edit leaderboard tracks edit-focused coding tasks via Aider (`aider_leaderboard` source). Public YAML in the Aider repository.", "category": "eval", "split": null, "language": null, "task_count": null, "homepage_url": "https://aider.chat/docs/leaderboards/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://aider.chat/docs/leaderboards/", "is_public_leaderboard": true, "harness_count": 1, "model_count": 84, "score_count": 84, "best_success_rate": 84.2}, "protocol": {"eval_protocol": "Aider edit leaderboard", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 84}, "rankings": {"benchmark": {"slug": "aider-edit", "name": "Aider Edit", "description": "Aider Edit leaderboard tracks edit-focused coding tasks via Aider (`aider_leaderboard` source). Public YAML in the Aider repository.", "category": "eval", "split": null, "language": null, "task_count": null, "homepage_url": "https://aider.chat/docs/leaderboards/", "paper_url": null, "hf_dataset_id": null, "official_leaderboard_url": "https://aider.chat/docs/leaderboards/", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 84.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1", "name": "o1", "vendor": "openai"}, "success_rate": 84.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-18T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-exp-1206-whole", "name": "Gemini Exp 1206 (whole)", "vendor": "Google"}, "success_rate": 80.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-08T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1-preview", "name": "o1 preview", "vendor": "OpenAI"}, "success_rate": 79.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-21T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-sonnet-20240620", "name": "Claude 3.5 Sonnet (2024-06-20)", "vendor": "Anthropic"}, "success_rate": 77.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-04T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-5-haiku-20241022", "name": "Claude 3.5 Haiku (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 75.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-04T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 72.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-13T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-coder-v2-0724", "name": "DeepSeek Coder V2 0724", "vendor": "DeepSeek"}, "success_rate": 72.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-32b", "name": "Qwen2.5 Coder 32b", "vendor": "Qwen"}, "success_rate": 72.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-28T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-v2-5", "name": "DeepSeek V2.5", "vendor": "DeepSeek"}, "success_rate": 72.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-05T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "chatgpt-4o", "name": "ChatGPT-4o", "vendor": "OpenAI"}, "success_rate": 72.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-04T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-v2-5-1210", "name": "DeepSeek V2.5 1210", "vendor": "DeepSeek"}, "success_rate": 72.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-10T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-20240806", "name": "GPT-4o (2024-08-06)", "vendor": "OpenAI"}, "success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-08-06T00:00:00+00:00"}, {"rank": 14, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-32b-instruct", "name": "Qwen2.5 Coder 32B Instruct", "vendor": "qwen"}, "success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-09T00:00:00+00:00"}, {"rank": 15, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-20241120", "name": "GPT-4o (2024-11-20)", "vendor": "OpenAI"}, "success_rate": 71.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-20T00:00:00+00:00"}, {"rank": 16, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "deepseek-chat-v2-0628", "name": "DeepSeek Chat V2 0628", "vendor": "DeepSeek"}, "success_rate": 69.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-19T00:00:00+00:00"}, {"rank": 17, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "success_rate": 69.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-11T00:00:00+00:00"}, {"rank": 18, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-14b-instruct", "name": "Qwen2.5 Coder 14B Instruct", "vendor": "Qwen"}, "success_rate": 69.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-29T00:00:00+00:00"}, {"rank": 19, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-exp-1206-diff", "name": "Gemini Exp 1206 (diff)", "vendor": "Google"}, "success_rate": 69.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-06T00:00:00+00:00"}, {"rank": 20, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-opus-20240229", "name": "Claude 3 Opus (2024-02-29)", "vendor": "Anthropic"}, "success_rate": 68.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-01T00:00:00+00:00"}, {"rank": 21, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-0613", "name": "GPT-4.0613", "vendor": "OpenAI"}, "success_rate": 67.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2023-12-16T00:00:00+00:00"}, {"rank": 22, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "dracarys2-72b-instruct", "name": "Dracarys2 72B Instruct", "vendor": "Abacus"}, "success_rate": 66.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-05T00:00:00+00:00"}, {"rank": 23, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-0125", "name": "GPT-4.0125", "vendor": "OpenAI"}, "success_rate": 66.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-01-25T00:00:00+00:00"}, {"rank": 24, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-0314", "name": "GPT-4.0314", "vendor": "OpenAI"}, "success_rate": 66.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-04T00:00:00+00:00"}, {"rank": 25, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-3-1-405b-instruct-whole", "name": "Llama 3.1 405b Instruct (whole)", "vendor": "Meta"}, "success_rate": 66.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00"}, {"rank": 26, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-1-5-pro", "name": "Gemini 1.5 Pro", "vendor": "Google"}, "success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-24T00:00:00+00:00"}, {"rank": 27, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4-1106", "name": "GPT-4.1106", "vendor": "OpenAI"}, "success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-08T00:00:00+00:00"}, {"rank": 28, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen-2-5-72b-instruct-bf16", "name": "QWEN 2.5 72b Instruct (bf16)", "vendor": "Qwen"}, "success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-20T00:00:00+00:00"}, {"rank": 29, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "mistral-large-2411", "name": "Mistral Large (2411)", "vendor": "Mistral"}, "success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-20T00:00:00+00:00"}, {"rank": 30, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen-qwen-2-5-coder-32b-instruct", "name": "Qwen/qwen 2.5 Coder 32b Instruct", "vendor": "Qwen"}, "success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-20T00:00:00+00:00"}, {"rank": 31, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "yi-lightning", "name": "Yi Lightning", "vendor": "01.AI"}, "success_rate": 65.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-11T00:00:00+00:00"}, {"rank": 32, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-3-1-405b-instruct-diff", "name": "Llama 3.1 405b Instruct (diff)", "vendor": "Meta"}, "success_rate": 63.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-23T00:00:00+00:00"}, {"rank": 33, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "nousresearch-hermes-3-llama-3-1-405b", "name": "Nousresearch/hermes 3 Llama 3.1 405b", "vendor": "nousresearch"}, "success_rate": 63.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-08-30T00:00:00+00:00"}, {"rank": 34, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-1-coder-7b-instruct-gguf-q8-0-32k", "name": "Qwen2.5.1 Coder 7B Instruct GGUF Q8 0 32k", "vendor": "Qwen"}, "success_rate": 63.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-07T00:00:00+00:00"}, {"rank": 35, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-14b", "name": "Qwen2.5 Coder 14b", "vendor": "Qwen"}, "success_rate": 61.7, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-27T00:00:00+00:00"}, {"rank": 36, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-exp-1114", "name": "Gemini Exp 1114", "vendor": "Google"}, "success_rate": 60.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-15T00:00:00+00:00"}, {"rank": 37, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "mistral-large-2-2407", "name": "Mistral Large 2 (2407)", "vendor": "Mistral"}, "success_rate": 60.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-24T00:00:00+00:00"}, {"rank": 38, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-3-3-70b-instruct", "name": "Llama 3.3 70B Instruct", "vendor": "meta-llama"}, "success_rate": 59.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-06T00:00:00+00:00"}, {"rank": 39, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-3-1-70b-instruct", "name": "Llama 3.1 70b Instruct", "vendor": "meta-llama"}, "success_rate": 58.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-25T00:00:00+00:00"}, {"rank": 40, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "grok-2", "name": "GROK 2", "vendor": "xAI"}, "success_rate": 58.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-13T00:00:00+00:00"}, {"rank": 41, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-32b-instruct-q8-0", "name": "Qwen2.5 32b Instruct Q8 0", "vendor": "Qwen"}, "success_rate": 58.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-20T00:00:00+00:00"}, {"rank": 42, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-7b-instruct", "name": "Qwen2.5 Coder 7B Instruct", "vendor": "Qwen"}, "success_rate": 57.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-16T00:00:00+00:00"}, {"rank": 43, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-exp-1121", "name": "Gemini Exp 1121", "vendor": "Google"}, "success_rate": 57.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-21T00:00:00+00:00"}, {"rank": 44, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 57.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-12T00:00:00+00:00"}, {"rank": 45, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-3-5", "name": "GPT-3.5", "vendor": "OpenAI"}, "success_rate": 56.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-30T00:00:00+00:00"}, {"rank": 46, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-72b-instruct", "name": "Qwen2 72B Instruct", "vendor": "Qwen"}, "success_rate": 55.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00"}, {"rank": 47, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gpt-4o-mini", "name": "GPT-4o mini", "vendor": "openai"}, "success_rate": 55.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-18T00:00:00+00:00"}, {"rank": 48, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-sonnet-20240229", "name": "Claude 3 Sonnet (2024-02-29)", "vendor": "Anthropic"}, "success_rate": 54.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-03-06T00:00:00+00:00"}, {"rank": 49, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "grok-2-mini", "name": "GROK 2 Mini", "vendor": "xAI"}, "success_rate": 54.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-13T00:00:00+00:00"}, {"rank": 50, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-3-1-nemotron-70b-instruct-hf", "name": "Llama 3.1 Nemotron 70B Instruct HF", "vendor": "Meta"}, "success_rate": 54.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-16T00:00:00+00:00"}, {"rank": 51, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-32b", "name": "Qwen2.5 32b", "vendor": "Qwen"}, "success_rate": 54.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00"}, {"rank": 52, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "nova-pro", "name": "Nova Pro", "vendor": "Amazon"}, "success_rate": 54.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-04T00:00:00+00:00"}, {"rank": 53, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-7b-instruct-q8-0", "name": "Qwen2.5 Coder 7b Instruct Q8 0", "vendor": "Qwen"}, "success_rate": 51.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-19T00:00:00+00:00"}, {"rank": 54, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codestral-2405", "name": "Codestral 2405", "vendor": "Mistral"}, "success_rate": 51.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-09T00:00:00+00:00"}, {"rank": 55, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-72b-instruct-q8-0", "name": "Qwen2 72b Instruct Q8 0", "vendor": "Qwen"}, "success_rate": 49.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00"}, {"rank": 56, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama3-70b-8192", "name": "Llama3 70b 8192", "vendor": "Meta"}, "success_rate": 49.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-03T00:00:00+00:00"}, {"rank": 57, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codestral-22b-v0-1-q8-0", "name": "Codestral 22b V0.1 Q8 0", "vendor": "Mistral"}, "success_rate": 48.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00"}, {"rank": 58, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codestral-22b-v0-1-q4-k-m", "name": "Codestral 22B V0.1 Q4 K M", "vendor": "Mistral"}, "success_rate": 48.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-21T00:00:00+00:00"}, {"rank": 59, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "claude-3-haiku-20240307", "name": "Claude 3 Haiku (2024-03-07)", "vendor": "Anthropic"}, "success_rate": 47.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-01T00:00:00+00:00"}, {"rank": 60, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codestral", "name": "Codestral", "vendor": "Meta"}, "success_rate": 45.9, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-28T00:00:00+00:00"}, {"rank": 61, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "yi-coder-9b-chat-q4-0", "name": "Yi Coder 9b Chat Q4 0", "vendor": "01.AI"}, "success_rate": 45.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-04T00:00:00+00:00"}, {"rank": 62, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "wizardlm-2-8x22b", "name": "WizardLM 2 8x22B", "vendor": "microsoft"}, "success_rate": 44.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-07T00:00:00+00:00"}, {"rank": 63, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "yi-coder-9b-chat", "name": "Yi Coder 9b Chat", "vendor": "Meta"}, "success_rate": 43.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-01T00:00:00+00:00"}, {"rank": 64, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "reflection-70b", "name": "Reflection 70B", "vendor": "Reflection"}, "success_rate": 42.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-06T00:00:00+00:00"}, {"rank": 65, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-3b-instruct", "name": "Qwen2.5 Coder 3B Instruct", "vendor": "Qwen"}, "success_rate": 39.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-04T00:00:00+00:00"}, {"rank": 66, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemini-1-5-flash", "name": "Gemini 1.5 Flash", "vendor": "Google"}, "success_rate": 38.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-24T00:00:00+00:00"}, {"rank": 67, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "command-r-08-24", "name": "Command R (08 24)", "vendor": "Cohere"}, "success_rate": 38.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-11T00:00:00+00:00"}, {"rank": 68, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "mistral-small", "name": "Mistral Small", "vendor": "Meta"}, "success_rate": 38.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-01T00:00:00+00:00"}, {"rank": 69, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen1-5-110b-chat", "name": "Qwen1.5 110b Chat", "vendor": "Qwen"}, "success_rate": 37.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-07T00:00:00+00:00"}, {"rank": 70, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama-3-1-8b-instruct", "name": "Llama 3.1 8b Instruct", "vendor": "meta-llama"}, "success_rate": 37.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-25T00:00:00+00:00"}, {"rank": 71, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "gemma2-27b-instruct-q8-0", "name": "Gemma2 27b Instruct Q8 0", "vendor": "Google"}, "success_rate": 36.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-07-09T00:00:00+00:00"}, {"rank": 72, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codeqwen-7b-chat-v1-5-q8-0", "name": "Codeqwen 7b Chat V1.5 Q8 0", "vendor": "Qwen"}, "success_rate": 34.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-06-08T00:00:00+00:00"}, {"rank": 73, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "mistral-nemo-12b-instruct-2407-q4-k-m", "name": "Mistral Nemo 12b Instruct 2407 Q4 K M", "vendor": "Meta"}, "success_rate": 33.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00"}, {"rank": 74, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "codegeex4", "name": "Codegeex4", "vendor": "Meta"}, "success_rate": 32.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-29T00:00:00+00:00"}, {"rank": 75, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "command-r-plus", "name": "Command R Plus", "vendor": "Cohere"}, "success_rate": 31.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-05-06T00:00:00+00:00"}, {"rank": 76, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-1-5b-instruct", "name": "Qwen2.5 Coder 1.5B Instruct", "vendor": "Qwen"}, "success_rate": 31.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-11T00:00:00+00:00"}, {"rank": 77, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "wojtek-opencodeinterpreter-6-7b", "name": "wojtek/opencodeinterpreter 6.7b", "vendor": "Meta"}, "success_rate": 30.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00"}, {"rank": 78, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "hermes3-8b-llama3-1", "name": "Hermes3 8b Llama3.1", "vendor": "Meta"}, "success_rate": 30.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00"}, {"rank": 79, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "o1-mini", "name": "o1 mini", "vendor": "OpenAI"}, "success_rate": 27.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-12-21T00:00:00+00:00"}, {"rank": 80, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "llama3-2-3b-instruct", "name": "Llama3.2 3b Instruct", "vendor": "Meta"}, "success_rate": 26.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-09-30T00:00:00+00:00"}, {"rank": 81, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "tulu3", "name": "Tulu3", "vendor": "Meta"}, "success_rate": 26.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-28T00:00:00+00:00"}, {"rank": 82, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "hermes3", "name": "Hermes3", "vendor": "Meta"}, "success_rate": 22.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-01T00:00:00+00:00"}, {"rank": 83, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "granite3-dense-8b", "name": "Granite3 Dense 8b", "vendor": "Meta"}, "success_rate": 20.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-11-28T00:00:00+00:00"}, {"rank": 84, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "qwen2-5-coder-0-5b-instruct", "name": "Qwen2.5 Coder 0.5B Instruct", "vendor": "Qwen"}, "success_rate": 14.3, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-29T00:00:00+00:00"}]}}