{"benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite", "description": "SWE-bench Lite is the 300-task subset of the original SWE-bench suite used for faster eval. The public leaderboard on swebench.com accepts community harness\u00d7model submissions (`source = swe_bench_site`).\n\nDistinct from Verified (500 tasks) and from training-only indices. Useful for historical comparisons and lighter eval cycles.", "category": "eval", "split": "lite", "language": "python", "task_count": 300, "homepage_url": "https://www.swebench.com/lite", "paper_url": "https://arxiv.org/abs/2310.06770", "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/lite", "is_public_leaderboard": true, "harness_count": 36, "model_count": 18, "score_count": 50, "best_success_rate": 60.33}, "protocol": {"eval_protocol": "SWE-bench Lite public submission", "temperature": null, "max_turns": null, "context_budget_tokens": null, "n": 50}, "rankings": {"benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite", "description": "SWE-bench Lite is the 300-task subset of the original SWE-bench suite used for faster eval. The public leaderboard on swebench.com accepts community harness\u00d7model submissions (`source = swe_bench_site`).\n\nDistinct from Verified (500 tasks) and from training-only indices. Useful for historical comparisons and lighter eval cycles.", "category": "eval", "split": "lite", "language": "python", "task_count": 300, "homepage_url": "https://www.swebench.com/lite", "paper_url": "https://arxiv.org/abs/2310.06770", "hf_dataset_id": null, "official_leaderboard_url": "https://www.swebench.com/lite", "is_public_leaderboard": true, "harness_count": null, "model_count": null, "score_count": null, "best_success_rate": null}, "sort": "success_rate", "rows": [{"rank": 1, "harness": {"slug": "experepair-v1-0", "name": "ExpeRepair V1.0", "vendor": "Experepair", "scaffold_code": null, "repo_url": "https://github.com/ExpeRepair/ExpeRepair", "homepage_url": "https://github.com/ExpeRepair/ExpeRepair", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 60.33, "resolved_count": 181, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/ExpeRepair/ExpeRepair", "observed_at": "2025-06-25T00:00:00+00:00"}, {"rank": 2, "harness": {"slug": "kgcompass", "name": "KGCompass", "vendor": "Gleam Lab", "scaffold_code": null, "repo_url": "https://github.com/GLEAM-Lab/KGCompass", "homepage_url": "https://github.com/GLEAM-Lab/KGCompass", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250609_KGCompass_deepseek-v3.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 58.33, "resolved_count": 175, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/GLEAM-Lab/KGCompass", "observed_at": "2025-09-06T00:00:00+00:00"}, {"rank": 3, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "success_rate": 56.67, "resolved_count": 170, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2025-05-26T00:00:00+00:00"}, {"rank": 4, "harness": {"slug": "semagent-multi-v1-0", "name": "SemAgent Multi V1.0", "vendor": "Columbia", "scaffold_code": null, "repo_url": null, "homepage_url": "https://arxiv.org/abs/2506.16650", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250625_SemAgent_Multi-v1_Claude3.7Sonnet_Gemini2.5Pro.jpg"}, "model": {"slug": "gemini-2-5-pro", "name": "Gemini 2.5 Pro", "vendor": "google"}, "success_rate": 51.67, "resolved_count": 155, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://arxiv.org/abs/2506.16650", "observed_at": "2025-06-25T00:00:00+00:00"}, {"rank": 5, "harness": {"slug": "isea", "name": "Isea", "vendor": "ISEA", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/lite", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 51.33, "resolved_count": 154, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2025-09-11T00:00:00+00:00"}, {"rank": 6, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "success_rate": 48.0, "resolved_count": 144, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2025-02-26T00:00:00+00:00"}, {"rank": 7, "harness": {"slug": "dars-agent", "name": "DARS Agent", "vendor": "Darsagent", "scaffold_code": null, "repo_url": "https://github.com/darsagent/DARS-Agent", "homepage_url": "https://github.com/darsagent/DARS-Agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250205_dars_agent_claude_3.5_sonnet_deepseek_r1.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 47.0, "resolved_count": 141, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/darsagent/DARS-Agent", "observed_at": "2025-02-05T00:00:00+00:00"}, {"rank": 8, "harness": {"slug": "kgcompass", "name": "KGCompass", "vendor": "Gleam Lab", "scaffold_code": null, "repo_url": "https://github.com/GLEAM-Lab/KGCompass", "homepage_url": "https://github.com/GLEAM-Lab/KGCompass", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250609_KGCompass_deepseek-v3.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 46.0, "resolved_count": 138, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/GLEAM-Lab/KGCompass", "observed_at": "2025-06-19T00:00:00+00:00"}, {"rank": 9, "harness": {"slug": "entropo", "name": "Entropo", "vendor": "Sherdencooper", "scaffold_code": null, "repo_url": "https://github.com/sherdencooper/R2E-Gym", "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250901_entroPO_R2E_QwenCoder30BA3B.png"}, "model": {"slug": "qwen3-coder-30b-a3b-instruct", "name": "Qwen3 Coder 30B A3B Instruct", "vendor": "qwen"}, "success_rate": 45.0, "resolved_count": 135, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/sherdencooper/R2E-Gym", "observed_at": "2025-09-01T00:00:00+00:00"}, {"rank": 10, "harness": {"slug": "kodu-v1", "name": "Kodu V1", "vendor": "Kodu", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.kodu.ai/", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241207_kodu_sonnet_v1.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 44.67, "resolved_count": 134, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.kodu.ai/", "observed_at": "2024-12-07T00:00:00+00:00"}, {"rank": 11, "harness": {"slug": "codestory-midwit-agent", "name": "Codestory Midwit Agent", "vendor": "CodeStory", "scaffold_code": null, "repo_url": null, "homepage_url": "https://codestory.ai", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240702_codestory_aide_mixed.png"}, "model": {"slug": "mixed-models", "name": "mixed-models", "vendor": "Mixed"}, "success_rate": 43.0, "resolved_count": 129, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://aide.dev/", "observed_at": "2024-07-02T00:00:00+00:00"}, {"rank": 12, "harness": {"slug": "lingxi", "name": "Lingxi", "vendor": "Nimasteryang", "scaffold_code": null, "repo_url": "https://github.com/nimasteryang/Lingxi", "homepage_url": "https://github.com/nimasteryang/Lingxi", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250509_Lingxi_claude-3-5-sonnet-20241022.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 42.67, "resolved_count": 128, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/nimasteryang/Lingxi", "observed_at": "2025-05-09T00:00:00+00:00"}, {"rank": 13, "harness": {"slug": "openhands-sdk", "name": "OpenHands SDK", "vendor": "All Hands AI", "scaffold_code": "ohsdk", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": "https://www.all-hands.dev/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241103_OpenHands-CodeAct-2.1-sonnet-20241022.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 41.67, "resolved_count": 125, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-10-25T00:00:00+00:00"}, {"rank": 14, "harness": {"slug": "patchkitty-0-9", "name": "Patchkitty 0 9", "vendor": "PatchKitty", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/lite", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241220_PatchKitty-0.9_claude-3.5-sonnet-20241022.gif"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 41.33, "resolved_count": 124, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2024-12-20T00:00:00+00:00"}, {"rank": 15, "harness": {"slug": "orcaloca", "name": "OrcaLoca", "vendor": "Fishmingyu", "scaffold_code": null, "repo_url": "https://github.com/fishmingyu/OrcarLLM", "homepage_url": "https://github.com/fishmingyu/OrcarLLM", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 41.0, "resolved_count": 123, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/fishmingyu/OrcarLLM", "observed_at": "2025-01-13T00:00:00+00:00"}, {"rank": 16, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 40.67, "resolved_count": 122, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-12-02T00:00:00+00:00"}, {"rank": 17, "harness": {"slug": "opencsg-starship-agentic-coder", "name": "Opencsg Starship Agentic Coder", "vendor": "OpenCSG", "scaffold_code": null, "repo_url": null, "homepage_url": "https://opencsg.com/starship", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250113_OpenCSG-Starship-Agentic-Coder_gpt4o.svg"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 39.67, "resolved_count": 119, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://opencsg.com/starship", "observed_at": "2025-01-13T00:00:00+00:00"}, {"rank": 18, "harness": {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "scaffold_code": null, "repo_url": "https://github.com/aorwall/moatless-tools", "homepage_url": "https://github.com/aorwall/moatless-tools", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240617_moatless_gpt4o.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 38.33, "resolved_count": 115, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/aorwall/moatless-tools", "observed_at": "2024-11-17T00:00:00+00:00"}, {"rank": 19, "harness": {"slug": "abanteai-mentatbot", "name": "Abanteai Mentatbot", "vendor": "AbanteAI", "scaffold_code": null, "repo_url": "https://github.com/AbanteAI/mentat", "homepage_url": "https://mentat.ai/blog/mentatbot-sota-coding-agent", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 38.0, "resolved_count": 114, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://mentat.ai/blog/mentatbot-sota-coding-agent", "observed_at": "2024-06-27T00:00:00+00:00"}, {"rank": 20, "harness": {"slug": "patched-codes-patchwork", "name": "Patched.Codes Patchwork", "vendor": "Patched Codes", "scaffold_code": null, "repo_url": "https://github.com/patched-codes/patchwork", "homepage_url": "https://github.com/patched-codes/patchwork", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250104_patched_codes_claude-3.5-sonnet-20241022.png"}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "success_rate": 37.0, "resolved_count": 111, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/patched-codes/patchwork", "observed_at": "2025-01-04T00:00:00+00:00"}, {"rank": 21, "harness": {"slug": "kgcompass", "name": "KGCompass", "vendor": "Gleam Lab", "scaffold_code": null, "repo_url": "https://github.com/GLEAM-Lab/KGCompass", "homepage_url": "https://github.com/GLEAM-Lab/KGCompass", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20250609_KGCompass_deepseek-v3.png"}, "model": {"slug": "deepseek-v3", "name": "DeepSeek V3", "vendor": "DeepSeek"}, "success_rate": 36.67, "resolved_count": 110, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/GLEAM-Lab/KGCompass", "observed_at": "2025-06-09T00:00:00+00:00"}, {"rank": 22, "harness": {"slug": "appmap-navie-v2", "name": "Appmap Navie V2", "vendor": "AppMap", "scaffold_code": null, "repo_url": "https://github.com/applandeo/appmap", "homepage_url": "https://appmap.io", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241113_navie-2-gpt4o-sonnet.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 36.0, "resolved_count": 108, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-11-13T00:00:00+00:00"}, {"rank": 23, "harness": {"slug": "bytedance-marscode-agent", "name": "Bytedance Marscode Agent", "vendor": "ByteDance", "scaffold_code": null, "repo_url": "https://github.com/bytedance/trae-agent", "homepage_url": "https://www.marscode.com", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240723_marscode-agent-dev.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 34.0, "resolved_count": 102, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.marscode.com/", "observed_at": "2024-07-23T00:00:00+00:00"}, {"rank": 24, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "o3-mini", "name": "o3 mini", "vendor": "openai"}, "success_rate": 32.33, "resolved_count": 97, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/sorendunn/Agentless-Lite", "observed_at": "2025-02-14T00:00:00+00:00"}, {"rank": 25, "harness": {"slug": "codeshelltester", "name": "CodeShellTester", "vendor": "Wisdomshell", "scaffold_code": null, "repo_url": "https://github.com/WisdomShell/codeshell", "homepage_url": "https://github.com/WisdomShell/codeshell", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241111_codeshelltester_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 31.33, "resolved_count": 94, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/WisdomShell/codeshell", "observed_at": "2024-11-11T00:00:00+00:00"}, {"rank": 26, "harness": {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "scaffold_code": null, "repo_url": "https://github.com/aorwall/moatless-tools", "homepage_url": "https://github.com/aorwall/moatless-tools", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240617_moatless_gpt4o.png"}, "model": {"slug": "deepseek-v3", "name": "DeepSeek V3", "vendor": "DeepSeek"}, "success_rate": 30.67, "resolved_count": 92, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/aorwall/moatless-tools", "observed_at": "2025-01-11T00:00:00+00:00"}, {"rank": 27, "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "scaffold_code": null, "repo_url": "https://github.com/nus-apr/auto-code-rover", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 30.67, "resolved_count": 92, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://autocoderover.dev/", "observed_at": "2024-06-21T00:00:00+00:00"}, {"rank": 28, "harness": {"slug": "aegis-o3-mini-1-0", "name": "Aegis O3 Mini 1.0", "vendor": "Evandiewald", "scaffold_code": null, "repo_url": "https://github.com/evandiewald/aegis", "homepage_url": "https://github.com/evandiewald/aegis", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "o3-mini", "name": "o3 mini", "vendor": "openai"}, "success_rate": 30.33, "resolved_count": 91, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/evandiewald/aegis", "observed_at": "2025-02-07T00:00:00+00:00"}, {"rank": 29, "harness": {"slug": "aigcode-infant-coder-2024-08-30", "name": "Aigcode Infant Coder 2024 08 30", "vendor": "AIGCode", "scaffold_code": null, "repo_url": null, "homepage_url": "https://aigcode.net/", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 30.0, "resolved_count": 90, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://aigcode.net/", "observed_at": "2024-09-08T00:00:00+00:00"}, {"rank": 30, "harness": {"slug": "kortix-ai", "name": "Kortix AI", "vendor": null, "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.kortix.ai/", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241203_KortixAI-AgentPress-sonnet-20241022.png"}, "model": {"slug": "sonnet-20241022", "name": "Sonnet 20241022", "vendor": "Anthropic"}, "success_rate": 30.0, "resolved_count": 90, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.kortix.ai/", "observed_at": "2024-12-03T00:00:00+00:00"}, {"rank": 31, "harness": {"slug": "agentless", "name": "Agentless", "vendor": "Ozyyshr", "scaffold_code": null, "repo_url": "https://github.com/ozyyshr/RepoGraph", "homepage_url": "https://github.com/ozyyshr/RepoGraph", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240808_RepoGraph_gpt4o.png"}, "model": {"slug": "gpt-4o", "name": "GPT-4o", "vendor": "openai"}, "success_rate": 29.67, "resolved_count": 89, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/ozyyshr/RepoGraph", "observed_at": "2024-08-08T00:00:00+00:00"}, {"rank": 32, "harness": {"slug": "coder", "name": "CodeR", "vendor": "Nl2Code", "scaffold_code": null, "repo_url": "https://github.com/NL2Code/CodeR", "homepage_url": "https://github.com/NL2Code/CodeR", "openrouter_icon_url": "https://coder.techcamp.org.uk/", "leaderboard_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 28.33, "resolved_count": 85, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/NL2Code/CodeR", "observed_at": "2024-06-04T00:00:00+00:00"}, {"rank": 33, "harness": {"slug": "reproducedrg", "name": "Reproducedrg", "vendor": "Research", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/lite", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 28.0, "resolved_count": 84, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2024-11-17T00:00:00+00:00"}, {"rank": 34, "harness": {"slug": "sima", "name": "SIMA", "vendor": "Swe Bench", "scaffold_code": null, "repo_url": "https://github.com/swe-bench/experiments/tree/main/evaluation/lite/20240706_sima_gpt4o", "homepage_url": "https://github.com/swe-bench/experiments/tree/main/evaluation/lite/20240706_sima_gpt4o", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 27.67, "resolved_count": 83, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/swe-bench/experiments/tree/main/evaluation/lite/20240706_sima_gpt4o", "observed_at": "2024-07-06T00:00:00+00:00"}, {"rank": 35, "harness": {"slug": "agentless-lite", "name": "Agentless Lite", "vendor": "OpenAutoCoder", "scaffold_code": null, "repo_url": "https://github.com/OpenAutoCoder/Agentless", "homepage_url": "https://github.com/OpenAutoCoder/Agentless", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241006_Agentless_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 27.33, "resolved_count": 82, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/OpenAutoCoder/Agentless", "observed_at": "2024-06-30T00:00:00+00:00"}, {"rank": 36, "harness": {"slug": "masai", "name": "Masai", "vendor": "Masai", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/verified", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 27.33, "resolved_count": 82, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://masai-dev-agent.github.io/", "observed_at": "2024-06-12T00:00:00+00:00"}, {"rank": 37, "harness": {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "scaffold_code": null, "repo_url": "https://github.com/aorwall/moatless-tools", "homepage_url": "https://github.com/aorwall/moatless-tools", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240617_moatless_gpt4o.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 26.67, "resolved_count": 80, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/aorwall/moatless-tools", "observed_at": "2024-06-23T00:00:00+00:00"}, {"rank": 38, "harness": {"slug": "openhands", "name": "OpenHands (legacy)", "vendor": "All Hands AI", "scaffold_code": "oh", "repo_url": "https://github.com/All-Hands-AI/OpenHands", "homepage_url": "https://github.com/All-Hands-AI/OpenHands", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240725_opendevin_codeact_v1.8_claude35sonnet.png"}, "model": {"slug": "codeact-v1-8", "name": "CodeAct v1.8", "vendor": null}, "success_rate": 26.67, "resolved_count": 80, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://docs.all-hands.dev/", "observed_at": "2024-07-25T00:00:00+00:00"}, {"rank": 39, "harness": {"slug": "aider", "name": "Aider", "vendor": "Aider", "scaffold_code": null, "repo_url": "https://github.com/Aider-AI/aider", "homepage_url": "https://aider.chat/", "openrouter_icon_url": "https://aider.chat/favicon.ico", "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240523_aider.png"}, "model": {"slug": "ensemble-gpt-4o-claude-3-opus", "name": "Ensemble (GPT-4o, Claude 3 Opus)", "vendor": "Anthropic"}, "success_rate": 26.33, "resolved_count": 79, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/paul-gauthier/aider", "observed_at": "2024-05-23T00:00:00+00:00"}, {"rank": 40, "harness": {"slug": "swe-fixer", "name": "SWE Fixer", "vendor": "Internlm", "scaffold_code": null, "repo_url": "https://github.com/InternLM/SWE-Fixer", "homepage_url": "https://github.com/InternLM/SWE-Fixer", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128.png"}, "model": {"slug": "ensemble-swe-fixer-qwen2-5-7b-retriever-qwen2-5-72b-editor", "name": "Ensemble (SWE Fixer (Qwen2.5 7b Retriever, Qwen2.5 72b Editor)", "vendor": "Qwen"}, "success_rate": 24.67, "resolved_count": 74, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/InternLM/SWE-Fixer", "observed_at": "2025-03-06T00:00:00+00:00"}, {"rank": 41, "harness": {"slug": "moatless-tools", "name": "Moatless Tools", "vendor": "Aorwall", "scaffold_code": null, "repo_url": "https://github.com/aorwall/moatless-tools", "homepage_url": "https://github.com/aorwall/moatless-tools", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240617_moatless_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 24.67, "resolved_count": 74, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/aorwall/moatless-tools", "observed_at": "2024-06-17T00:00:00+00:00"}, {"rank": 42, "harness": {"slug": "opencsg-starship-codegenagent", "name": "Opencsg Starship Codegenagent", "vendor": "OpenCSG", "scaffold_code": null, "repo_url": null, "homepage_url": "https://opencsg.com/product?class=StarShip", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240524_opencsg_starship_gpt4.svg"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 23.67, "resolved_count": 71, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://opencsg.com/product?class=StarShip", "observed_at": "2024-05-24T00:00:00+00:00"}, {"rank": 43, "harness": {"slug": "swe-fixer", "name": "SWE Fixer", "vendor": "Internlm", "scaffold_code": null, "repo_url": "https://github.com/InternLM/SWE-Fixer", "homepage_url": "https://github.com/InternLM/SWE-Fixer", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20241128_SWE-Fixer_Qwen2.5-7b-retriever_Qwen2.5-72b-editor_20241128.png"}, "model": {"slug": "ensemble-swe-fixer-qwen2-5-7b-retriever-qwen2-5-72b-editor-20241128", "name": "Ensemble (SWE Fixer (Qwen2.5 7b Retriever, Qwen2.5 72b Editor) 20241128)", "vendor": "Qwen"}, "success_rate": 23.33, "resolved_count": 70, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/InternLM/SWE-Fixer", "observed_at": "2024-11-28T00:00:00+00:00"}, {"rank": 44, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-5-sonnet", "name": "Claude 3.5 Sonnet", "vendor": "Anthropic"}, "success_rate": 23.0, "resolved_count": 69, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-06-20T00:00:00+00:00"}, {"rank": 45, "harness": {"slug": "appmap-navie", "name": "Appmap Navie", "vendor": "AppMap", "scaffold_code": null, "repo_url": "https://github.com/applandeo/appmap", "homepage_url": "https://appmap.io", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240615_appmap-navie_gpt4o.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 21.67, "resolved_count": 65, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://appmap.io/navie", "observed_at": "2024-06-15T00:00:00+00:00"}, {"rank": 46, "harness": {"slug": "bytedance-autose-based-on-swe-agent", "name": "Bytedance AutoSE (based On SWE Agent)", "vendor": "Bytedance", "scaffold_code": null, "repo_url": null, "homepage_url": "https://www.swebench.com/lite", "openrouter_icon_url": null, "leaderboard_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 21.67, "resolved_count": 65, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://www.swebench.com/lite", "observed_at": "2024-08-28T00:00:00+00:00"}, {"rank": 47, "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "scaffold_code": null, "repo_url": "https://github.com/nus-apr/auto-code-rover", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 19.0, "resolved_count": 57, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/nus-apr/auto-code-rover", "observed_at": "2024-05-30T00:00:00+00:00"}, {"rank": 48, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "success_rate": 18.33, "resolved_count": 55, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-07-28T00:00:00+00:00"}, {"rank": 49, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "success_rate": 18.0, "resolved_count": 54, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00"}, {"rank": 50, "harness": {"slug": "swe-agent", "name": "SWE-agent", "vendor": "SWE-agent", "scaffold_code": null, "repo_url": "https://github.com/SWE-agent/SWE-agent", "homepage_url": "https://github.com/SWE-agent/SWE-agent", "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240402_sweagent_claude3opus.png"}, "model": {"slug": "claude-3-opus", "name": "Claude 3 Opus", "vendor": "Anthropic"}, "success_rate": 11.67, "resolved_count": 35, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/SWE-agent/SWE-agent", "observed_at": "2024-04-02T00:00:00+00:00"}]}}