{"model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic", "model_family": "claude-3-5-sonnet", "parameter_count_b": null, "context_length": 200000, "homepage_url": null, "hf_model_id": null, "openrouter_id": null, "card_summary": "Anthropic Claude 3.5 Sonnet (2024-10-22) on SWE-bench Verified leaderboards.", "is_rl_checkpoint": false, "score_count": 20, "best_success_rate": 92.1, "input_price_per_million": null, "output_price_per_million": null}, "scores": [{"success_rate": 92.1, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 84.2, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 53.0, "resolved_count": 265, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-10-29T00:00:00+00:00", "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 51.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider polyglot leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2025-01-17T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "aider-polyglot", "name": "Aider Polyglot"}}, {"success_rate": 51.33, "resolved_count": 154, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2025-09-11T00:00:00+00:00", "harness": {"slug": "isea", "name": "Isea", "vendor": "ISEA", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 50.8, "resolved_count": 254, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-12-02T00:00:00+00:00", "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 49.0, "resolved_count": 245, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.anthropic.com/", "observed_at": "2024-10-22T00:00:00+00:00", "harness": {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 48.74, "resolved_count": null, "total_count": 1055, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "LiveCodeBench generation pass@1 mean", "source": "livecodebench", "source_url": "https://livecodebench.github.io/leaderboard.html", "observed_at": null, "harness": {"slug": "livecodebench", "name": "Livecodebench", "vendor": "LiveCodeBench", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "livecodebench-generation", "name": "LiveCodeBench (code generation)"}}, {"success_rate": 46.2, "resolved_count": 231, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.autocoderover.net/", "observed_at": "2024-11-08T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 46.0, "resolved_count": 138, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/GLEAM-Lab/KGCompass", "observed_at": "2025-06-19T00:00:00+00:00", "harness": {"slug": "kgcompass", "name": "KGCompass", "vendor": "Gleam Lab", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 44.67, "resolved_count": 134, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.kodu.ai/", "observed_at": "2024-12-07T00:00:00+00:00", "harness": {"slug": "kodu-v1", "name": "Kodu V1", "vendor": "Kodu", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 42.67, "resolved_count": 128, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/nimasteryang/Lingxi", "observed_at": "2025-05-09T00:00:00+00:00", "harness": {"slug": "lingxi", "name": "Lingxi", "vendor": "Nimasteryang", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 41.67, "resolved_count": 125, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-10-25T00:00:00+00:00", "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 41.33, "resolved_count": 124, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2024-12-20T00:00:00+00:00", "harness": {"slug": "patchkitty-0-9", "name": "Patchkitty 0 9", "vendor": "PatchKitty", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 41.0, "resolved_count": 123, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/fishmingyu/OrcarLLM", "observed_at": "2025-01-13T00:00:00+00:00", "harness": {"slug": "orcaloca", "name": "OrcaLoca", "vendor": "Fishmingyu", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 40.67, "resolved_count": 122, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-12-02T00:00:00+00:00", "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 38.33, "resolved_count": 115, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/aorwall/moatless-tools", "observed_at": "2024-11-17T00:00:00+00:00", "harness": {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 37.0, "resolved_count": 111, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/patched-codes/patchwork", "observed_at": "2025-01-04T00:00:00+00:00", "harness": {"slug": "patched-codes-patchwork", "name": "Patched.Codes Patchwork", "vendor": "Patched Codes", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 29.38, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-11-03T00:00:00+00:00", "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test"}}, {"success_rate": 24.89, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://www.autocoderover.net/", "observed_at": "2024-11-21T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test"}}], "harnesses": [{"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "repo_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null}, {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, {"slug": "isea", "name": "Isea", "vendor": "ISEA", "repo_url": null, "openrouter_icon_url": null}, {"slug": "kgcompass", "name": "KGCompass", "vendor": "Gleam Lab", "repo_url": "https://github.com/GLEAM-Lab/KGCompass", "openrouter_icon_url": null}, {"slug": "kodu-v1", "name": "Kodu V1", "vendor": "Kodu", "repo_url": null, "openrouter_icon_url": null}, {"slug": "lingxi", "name": "Lingxi", "vendor": "Nimasteryang", "repo_url": "https://github.com/nimasteryang/Lingxi", "openrouter_icon_url": null}, {"slug": "livecodebench", "name": "Livecodebench", "vendor": "LiveCodeBench", "repo_url": "https://github.com/LiveCodeBench/LiveCodeBench", "openrouter_icon_url": null}, {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "repo_url": "https://github.com/aorwall/moatless-tools", "openrouter_icon_url": null}, {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico"}, {"slug": "orcaloca", "name": "OrcaLoca", "vendor": "Fishmingyu", "repo_url": "https://github.com/fishmingyu/OrcarLLM", "openrouter_icon_url": null}, {"slug": "patched-codes-patchwork", "name": "Patched.Codes Patchwork", "vendor": "Patched Codes", "repo_url": "https://github.com/patched-codes/patchwork", "openrouter_icon_url": null}, {"slug": "patchkitty-0-9", "name": "Patchkitty 0 9", "vendor": "PatchKitty", "repo_url": null, "openrouter_icon_url": null}, {"slug": "tools", "name": "Tools", "vendor": "Anthropic", "repo_url": "https://github.com/anthropics", "openrouter_icon_url": null}]}