{"harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "description": "AutoCodeRover is an open-source software engineering agent that combines retrieval over the codebase with structured patch proposals. The catalog tracks versioned submission labels (v2.0, v2.1, dated builds) as separate harness slugs when leaderboard metadata distinguishes them.\n\nScores are community SWE-bench Verified submissions. Each version may change prompts, retrieval, or tool interfaces \u2014 do not treat all AutoCodeRover slugs as interchangeable.\n\nUpstream repository: `AutoCodeRover/AutoCodeRover`. EPAM AI RUN and other forks sometimes appear as distinct slugs when submissions use different harness names.", "card_summary": "AutoCodeRover open agent \u2014 iterative retrieval and patch application for issue fixing.", "homepage_url": "https://github.com/AutoCodeRover/AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "docs_url": null, "scaffold_code": null, "harbor_agent_name": null, "api_protocol": null, "status": "validated", "supports_rl": true, "score_count": 6, "best_success_rate": 46.2, "github_stars": null, "popularity_tier": null, "hrl_score": null, "hrl_rank": null, "catalog_token_volume": null, "openrouter_icon_url": null, "leaderboard_icon_url": "https://www.swebench.com/img/logos/20240628_autocoderover-v20240620.jpg", "aa_coding_index": null, "aa_mean_cost_usd": null, "aa_mean_total_tokens": null}, "scores": [{"success_rate": 46.2, "resolved_count": 231, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://www.autocoderover.net/", "observed_at": "2024-11-08T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 38.4, "resolved_count": 192, "total_count": 500, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Verified public submission", "source": "swe_bench_site", "source_url": "https://autocoderover.dev/", "observed_at": "2024-06-28T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-bench-verified", "name": "SWE-bench Verified"}}, {"success_rate": 30.67, "resolved_count": 92, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://autocoderover.dev/", "observed_at": "2024-06-21T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 24.89, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://www.autocoderover.net/", "observed_at": "2024-11-21T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-sonnet-20241022", "name": "Claude 3.5 Sonnet (2024-10-22)", "vendor": "Anthropic"}, "benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test"}}, {"success_rate": 19.0, "resolved_count": 57, "total_count": 300, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Lite public submission", "source": "swe_bench_site", "source_url": "https://github.com/nus-apr/auto-code-rover", "observed_at": "2024-05-30T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "swe-bench-lite", "name": "SWE-bench Lite"}}, {"success_rate": 18.83, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "SWE-bench Test public submission", "source": "swe_bench_site", "source_url": "https://autocoderover.dev/", "observed_at": "2024-06-28T00:00:00+00:00", "harness": {"slug": "autocoderover-v20240620", "name": "Autocoderover V20240620", "vendor": "AutoCodeRover", "repo_url": "https://github.com/nus-apr/auto-code-rover", "openrouter_icon_url": null}, "model": {"slug": "gpt-4o-20240513", "name": "GPT-4o (2024-05-13)", "vendor": "OpenAI"}, "benchmark": {"slug": "swe-bench-test", "name": "SWE-bench Test"}}], "benchmarks": [{"slug": "swe-bench-lite", "name": "SWE-bench Lite", "category": "eval"}, {"slug": "swe-bench-test", "name": "SWE-bench Test", "category": "eval"}, {"slug": "swe-bench-verified", "name": "SWE-bench Verified", "category": "eval"}]}