{"harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "description": "Codex (OpenAI) is a coding agent oriented around terminal workflows: reading context, proposing edits, and executing commands against a repository. Leaderboard rows labeled Codex typically pair this harness with OpenAI frontier models on SWE-bench Verified and Artificial Analysis coding-agent benchmarks.\n\nUnlike minimal bash-only scaffolds, Codex ships vendor tooling and prompts tuned for product use. Cross-harness comparisons on the same model should use the same scaffold (for example mini-SWE-agent on the SWE-bench bash-only board) rather than mixing Codex with community SWE-agent forks.\n\nPublic scores here come from `artificialanalysis` (Coding Agent Index components) and community `swe_bench_site` submissions. Check each row's source URL for the exact leaderboard snapshot and model pairing.", "card_summary": "OpenAI's Codex agent for terminal-based coding, repo navigation, and patch application.", "homepage_url": "https://openai.com/codex", "repo_url": "https://github.com/openai/codex", "docs_url": null, "scaffold_code": null, "harbor_agent_name": null, "api_protocol": null, "status": "validated", "supports_rl": true, "score_count": 37, "best_success_rate": 83.15, "github_stars": null, "popularity_tier": "popular", "hrl_score": 69.0, "hrl_rank": 5, "catalog_token_volume": 2011461977785, "openrouter_icon_url": "https://openai.com/favicon.ico", "leaderboard_icon_url": null, "aa_coding_index": 0.6505240024334414, "aa_mean_cost_usd": 1.5618390533742323, "aa_mean_total_tokens": 5600275}, "scores": [{"success_rate": 83.15, "resolved_count": null, "total_count": null, "cost_usd": 4.627393258426967, "input_tokens": 336797311, "output_tokens": 5966373, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 445, "is_harnessrl_measured": false, "eval_protocol": null, "source": "tbench_official", "source_url": "https://github.com/harbor-framework/terminal-bench-2-1/pull/45", "observed_at": "2026-05-01T00:00:00+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5", "name": "GPT-5.5", "vendor": "openai"}, "benchmark": {"slug": "terminal-bench-2-1-official", "name": "Terminal-Bench 2.1 (official)"}}, {"success_rate": 82.77, "resolved_count": null, "total_count": null, "cost_usd": 4.7523700715746395, "input_tokens": 6164087, "output_tokens": 25286, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5-xhigh", "name": "GPT-5.5 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 78.65, "resolved_count": null, "total_count": null, "cost_usd": 2.6525358742331298, "input_tokens": 3492715, "output_tokens": 13239, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5-medium", "name": "GPT-5.5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 78.43, "resolved_count": null, "total_count": null, "cost_usd": 0.9464044943820225, "input_tokens": 29803660, "output_tokens": 8707506, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 445, "is_harnessrl_measured": false, "eval_protocol": null, "source": "tbench_official", "source_url": "https://github.com/harbor-framework/terminal-bench-2-1/pull/115", "observed_at": "2026-07-11T00:00:00+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6", "name": "GPT-5.6", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-2-1-official", "name": "Terminal-Bench 2.1 (official)"}}, {"success_rate": 78.28, "resolved_count": null, "total_count": null, "cost_usd": 2.695350269529651, "input_tokens": 4933733, "output_tokens": 37445, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-max", "name": "GPT-5.6 (max)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 77.9, "resolved_count": null, "total_count": null, "cost_usd": 1.2924588445807745, "input_tokens": 1658130, "output_tokens": 8962, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-low", "name": "GPT-5.6 (low)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 72.28, "resolved_count": null, "total_count": null, "cost_usd": 1.5618390533742323, "input_tokens": 2877096, "output_tokens": 20902, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-high", "name": "GPT-5.6 (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 72.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-rebench resolved rate (best rolling window)", "source": "swe_rebench_site", "source_url": "https://swe-rebench.com/", "observed_at": null, "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "swe-rebench-agent-product", "name": "SWE-rebench agent (product)", "vendor": null}, "benchmark": {"slug": "swe-rebench", "name": "SWE-rebench"}}, {"success_rate": 70.79, "resolved_count": null, "total_count": null, "cost_usd": 1.2093171048466254, "input_tokens": 6575104, "output_tokens": 30840, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-xhigh", "name": "GPT-5.6 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 67.79, "resolved_count": null, "total_count": null, "cost_usd": 0.05949917044417178, "input_tokens": 10443571, "output_tokens": 58411, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "deepseek-v4-flash-0731-max", "name": "Deepseek V4 Flash 0731 Max", "vendor": "DeepSeek"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 66.96, "resolved_count": null, "total_count": null, "cost_usd": 2.695350269529651, "input_tokens": 4933733, "output_tokens": 37445, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-max", "name": "GPT-5.6 (max)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 64.31, "resolved_count": null, "total_count": null, "cost_usd": 4.7523700715746395, "input_tokens": 6164087, "output_tokens": 25286, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5-xhigh", "name": "GPT-5.5 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 62.92, "resolved_count": null, "total_count": null, "cost_usd": 0.09081146755368102, "input_tokens": 6378205, "output_tokens": 37873, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "deepseek-v4-pro-0813-max", "name": "Deepseek V4 Pro 0813 Max", "vendor": "DeepSeek"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 62.17, "resolved_count": null, "total_count": null, "cost_usd": 0.44305367719836414, "input_tokens": 2221162, "output_tokens": 11663, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-medium", "name": "GPT-5.6 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 60.47, "resolved_count": null, "total_count": null, "cost_usd": 1.5618390533742323, "input_tokens": 2877096, "output_tokens": 20902, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-high", "name": "GPT-5.6 (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 60.3, "resolved_count": null, "total_count": null, "cost_usd": 1.0868546433537856, "input_tokens": 1731981, "output_tokens": 7689, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6", "name": "GPT-5.6", "vendor": "OpenAI"}, "benchmark": {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)"}}, {"success_rate": 56.8, "resolved_count": null, "total_count": null, "cost_usd": 0.57, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 60, "is_harnessrl_measured": false, "eval_protocol": "WildClawBench overall score (60 tasks)", "source": "wildclawbench_site", "source_url": "https://github.com/internlm/WildClawBench", "observed_at": null, "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-4", "name": "GPT-5.4", "vendor": "openai"}, "benchmark": {"slug": "wildclawbench", "name": "WildClawBench"}}, {"success_rate": 56.64, "resolved_count": null, "total_count": null, "cost_usd": 2.6525358742331298, "input_tokens": 3492715, "output_tokens": 13239, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5-medium", "name": "GPT-5.5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 56.64, "resolved_count": null, "total_count": null, "cost_usd": 1.2093171048466254, "input_tokens": 6575104, "output_tokens": 30840, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-xhigh", "name": "GPT-5.6 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 53.39, "resolved_count": null, "total_count": null, "cost_usd": 1.2924588445807745, "input_tokens": 1658130, "output_tokens": 8962, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-low", "name": "GPT-5.6 (low)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 49.19, "resolved_count": null, "total_count": null, "cost_usd": 0.09081146755368102, "input_tokens": 6378205, "output_tokens": 37873, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "deepseek-v4-pro-0813-max", "name": "Deepseek V4 Pro 0813 Max", "vendor": "DeepSeek"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 42.77, "resolved_count": null, "total_count": null, "cost_usd": 0.05949917044417178, "input_tokens": 10443571, "output_tokens": 58411, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "deepseek-v4-flash-0731-max", "name": "Deepseek V4 Flash 0731 Max", "vendor": "DeepSeek"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 38.9, "resolved_count": null, "total_count": null, "cost_usd": 0.13, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 60, "is_harnessrl_measured": false, "eval_protocol": "WildClawBench overall score (60 tasks)", "source": "wildclawbench_site", "source_url": "https://github.com/internlm/WildClawBench", "observed_at": null, "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "glm-5", "name": "GLM 5", "vendor": "z-ai"}, "benchmark": {"slug": "wildclawbench", "name": "WildClawBench"}}, {"success_rate": 38.71, "resolved_count": null, "total_count": null, "cost_usd": 0.05949917044417178, "input_tokens": 10443571, "output_tokens": 58411, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "deepseek-v4-flash-0731-max", "name": "Deepseek V4 Flash 0731 Max", "vendor": "DeepSeek"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 36.58, "resolved_count": null, "total_count": null, "cost_usd": 0.44305367719836414, "input_tokens": 2221162, "output_tokens": 11663, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-medium", "name": "GPT-5.6 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 36.02, "resolved_count": null, "total_count": null, "cost_usd": 2.695350269529651, "input_tokens": 4933733, "output_tokens": 37445, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-max", "name": "GPT-5.6 (max)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 36.02, "resolved_count": null, "total_count": null, "cost_usd": 4.7523700715746395, "input_tokens": 6164087, "output_tokens": 25286, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5-xhigh", "name": "GPT-5.5 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 35.8, "resolved_count": null, "total_count": null, "cost_usd": 0.06, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 60, "is_harnessrl_measured": false, "eval_protocol": "WildClawBench overall score (60 tasks)", "source": "wildclawbench_site", "source_url": "https://github.com/internlm/WildClawBench", "observed_at": null, "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "minimax-m2-7", "name": "MiniMax M2.7", "vendor": "minimax"}, "benchmark": {"slug": "wildclawbench", "name": "WildClawBench"}}, {"success_rate": 35.4, "resolved_count": null, "total_count": null, "cost_usd": 1.0868546433537856, "input_tokens": 1731981, "output_tokens": 7689, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6", "name": "GPT-5.6", "vendor": "OpenAI"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}, {"success_rate": 35.3, "resolved_count": null, "total_count": null, "cost_usd": 0.15, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 60, "is_harnessrl_measured": false, "eval_protocol": "WildClawBench overall score (60 tasks)", "source": "wildclawbench_site", "source_url": "https://github.com/internlm/WildClawBench", "observed_at": null, "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "mimo-v2-pro", "name": "MiMo V2 Pro", "vendor": "Xiaomi"}, "benchmark": {"slug": "wildclawbench", "name": "WildClawBench"}}, {"success_rate": 34.41, "resolved_count": null, "total_count": null, "cost_usd": 1.2924588445807745, "input_tokens": 1658130, "output_tokens": 8962, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-low", "name": "GPT-5.6 (low)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 34.41, "resolved_count": null, "total_count": null, "cost_usd": 1.0868546433537856, "input_tokens": 1731981, "output_tokens": 7689, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6", "name": "GPT-5.6", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 31.45, "resolved_count": null, "total_count": null, "cost_usd": 1.2093171048466254, "input_tokens": 6575104, "output_tokens": 30840, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-xhigh", "name": "GPT-5.6 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 31.18, "resolved_count": null, "total_count": null, "cost_usd": 1.5618390533742323, "input_tokens": 2877096, "output_tokens": 20902, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-high", "name": "GPT-5.6 (high)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 30.65, "resolved_count": null, "total_count": null, "cost_usd": 2.6525358742331298, "input_tokens": 3492715, "output_tokens": 13239, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-5-medium", "name": "GPT-5.5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 27.15, "resolved_count": null, "total_count": null, "cost_usd": 0.44305367719836414, "input_tokens": 2221162, "output_tokens": 11663, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "gpt-5-6-medium", "name": "GPT-5.6 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)"}}, {"success_rate": 16.22, "resolved_count": null, "total_count": null, "cost_usd": 0.09081146755368102, "input_tokens": 6378205, "output_tokens": 37873, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "Artificial Analysis Coding Agent Index component (pass@1)", "source": "artificialanalysis", "source_url": "https://artificialanalysis.ai/agents/coding-agents", "observed_at": "2026-08-27T21:54:11.189561+00:00", "harness": {"slug": "codex", "name": "Codex", "vendor": "OpenAI", "repo_url": "https://github.com/openai/codex", "openrouter_icon_url": "https://openai.com/favicon.ico"}, "model": {"slug": "deepseek-v4-pro-0813-max", "name": "Deepseek V4 Pro 0813 Max", "vendor": "DeepSeek"}, "benchmark": {"slug": "deepswe", "name": "DeepSWE (AA component)"}}], "benchmarks": [{"slug": "deepswe", "name": "DeepSWE (AA component)", "category": "eval"}, {"slug": "swe-atlas-qna", "name": "SWE-Atlas-QnA (AA component)", "category": "eval"}, {"slug": "swe-rebench", "name": "SWE-rebench", "category": "eval"}, {"slug": "terminal-bench-2-1-official", "name": "Terminal-Bench 2.1 (official)", "category": "eval"}, {"slug": "terminal-bench-v2-1", "name": "Terminal-Bench v2.1 (AA component)", "category": "eval"}, {"slug": "wildclawbench", "name": "WildClawBench", "category": "eval"}]}