{"model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai", "model_family": "gpt-4", "parameter_count_b": null, "context_length": 8191, "homepage_url": null, "hf_model_id": null, "openrouter_id": "openai/gpt-4", "card_summary": "openai GPT-4 on SWE-bench Verified leaderboards.", "is_rl_checkpoint": false, "score_count": 19, "best_success_rate": 87.5, "input_price_per_million": 30.0, "output_price_per_million": 60.0}, "scores": [{"success_rate": 87.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 79.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 76.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 57.6, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider edit leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-12T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "aider-edit", "name": "Aider Edit"}}, {"success_rate": 47.2, "resolved_count": 236, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-11-06T00:00:00+00:00", "harness": {"slug": "appmap-navie-v2", "name": "Appmap Navie V2", "vendor": "AppMap", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 39.67, "resolved_count": 119, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://opencsg.com/starship", "observed_at": "2025-01-13T00:00:00+00:00", "harness": {"slug": "opencsg-starship-agentic-coder", "name": "Opencsg Starship Agentic Coder", "vendor": "OpenCSG", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 37.32, "resolved_count": null, "total_count": 1055, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "LiveCodeBench generation pass@1 mean", "source": "livecodebench", "source_url": "https://livecodebench.github.io/leaderboard.html", "observed_at": null, "harness": {"slug": "livecodebench", "name": "Livecodebench", "vendor": "LiveCodeBench", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "livecodebench-generation", "name": "LiveCodeBench (code generation)"}}, {"success_rate": 36.0, "resolved_count": 108, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-11-13T00:00:00+00:00", "harness": {"slug": "appmap-navie-v2", "name": "Appmap Navie V2", "vendor": "AppMap", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 30.0, "resolved_count": 90, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://aigcode.net/", "observed_at": "2024-09-08T00:00:00+00:00", "harness": {"slug": "aigcode-infant-coder-2024-08-30", "name": "Aigcode Infant Coder 2024 08 30", "vendor": "AIGCode", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 28.33, "resolved_count": 85, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/NL2Code/CodeR", "observed_at": "2024-06-04T00:00:00+00:00", "harness": {"slug": "coder", "name": "CodeR", "vendor": "Nl2Code", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 28.0, "resolved_count": 84, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2024-11-17T00:00:00+00:00", "harness": {"slug": "reproducedrg", "name": "Reproducedrg", "vendor": "Research", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 24.0, "resolved_count": 120, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.epam.com/services/artificial-intelligence", "observed_at": "2024-08-20T00:00:00+00:00", "harness": {"slug": "epam-ai-run-developer-agent", "name": "EPAM AI Run", "vendor": "EPAM Systems, Inc.", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 23.67, "resolved_count": 71, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://opencsg.com/product?class=StarShip", "observed_at": "2024-05-24T00:00:00+00:00", "harness": {"slug": "opencsg-starship-codegenagent", "name": "Opencsg Starship Codegenagent", "vendor": "OpenCSG", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 22.4, "resolved_count": 112, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00", "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 21.67, "resolved_count": 65, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2024-08-28T00:00:00+00:00", "harness": {"slug": "bytedance-autose-based-on-swe-agent", "name": "Bytedance AutoSE (based On SWE Agent)", "vendor": "Bytedance", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 21.4, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Aider refactor leaderboard", "source": "aider_leaderboard", "source_url": "https://aider.chat/docs/leaderboards/", "observed_at": "2024-04-10T00:00:00+00:00", "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "aider-refactor", "name": "Aider Refactor"}}, {"success_rate": 19.0, "resolved_count": 57, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/nus-apr/auto-code-rover", "observed_at": "2024-05-30T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 18.0, "resolved_count": 54, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00", "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 12.47, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00", "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "repo_url": null, "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test"}}], "harnesses": [{"slug": "aider", "name": "Aider", "vendor": "Aider", "repo_url": "https://github.com/Aider-AI/aider", "openrouter_icon_url": "https://aider.chat/favicon.ico"}, {"slug": "aigcode-infant-coder-2024-08-30", "name": "Aigcode Infant Coder 2024 08 30", "vendor": "AIGCode", "repo_url": null, "openrouter_icon_url": null}, {"slug": "appmap-navie-v2", "name": "Appmap Navie V2", "vendor": "AppMap", "repo_url": "https://github.com/applandeo/appmap", "openrouter_icon_url": null}, {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, {"slug": "bytedance-autose-based-on-swe-agent", "name": "Bytedance AutoSE (based On SWE Agent)", "vendor": "Bytedance", "repo_url": null, "openrouter_icon_url": null}, {"slug": "coder", "name": "CodeR", "vendor": "Nl2Code", "repo_url": "https://github.com/NL2Code/CodeR", "openrouter_icon_url": "https://coder.techcamp.org.uk/"}, {"slug": "epam-ai-run-developer-agent", "name": "EPAM AI Run", "vendor": "EPAM Systems, Inc.", "repo_url": "https://github.com/epam/AI-RUN", "openrouter_icon_url": null}, {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, {"slug": "livecodebench", "name": "Livecodebench", "vendor": "LiveCodeBench", "repo_url": "https://github.com/LiveCodeBench/LiveCodeBench", "openrouter_icon_url": null}, {"slug": "opencsg-starship-agentic-coder", "name": "Opencsg Starship Agentic Coder", "vendor": "OpenCSG", "repo_url": null, "openrouter_icon_url": null}, {"slug": "opencsg-starship-codegenagent", "name": "Opencsg Starship Codegenagent", "vendor": "OpenCSG", "repo_url": null, "openrouter_icon_url": null}, {"slug": "reproducedrg", "name": "Reproducedrg", "vendor": "Research", "repo_url": null, "openrouter_icon_url": null}, {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "repo_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null}]}