{"harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "description": "HAL Generalist Agent is tracked in this catalog as a software-engineering agent harness. Public benchmark rows use this slug on leaderboards such as SWE-bench Verified or Artificial Analysis coding-agent evals.\n\nCheck each score row's `source` and `source_url` for the authoritative snapshot. Harness-level metadata here does not invent scores \u2014 only documents what the harness is and where published results come from.", "card_summary": "Princeton PLI HAL Generalist Agent coding agent harness.", "homepage_url": "https://hal.cs.princeton.edu/", "repo_url": "https://github.com/princeton-pli/hal-harness", "docs_url": null, "scaffold_code": null, "harbor_agent_name": null, "api_protocol": null, "status": "validated", "supports_rl": true, "score_count": 124, "best_success_rate": 96.25, "github_stars": null, "popularity_tier": null, "hrl_score": null, "hrl_rank": null, "catalog_token_volume": null, "openrouter_icon_url": null, "leaderboard_icon_url": null, "aa_coding_index": null, "aa_mean_cost_usd": null, "aa_mean_total_tokens": null}, "scores": [{"success_rate": 96.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 93.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 93.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-5", "name": "GPT-5.5", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 93.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-7-opus", "name": "Claude 4.7 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 92.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 91.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-sonnet", "name": "Claude 4 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 90.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-7-opus", "name": "Claude 4.7 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 90.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 90.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o1", "name": "o1", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 90.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-3-1-pro", "name": "Gemini 3.1 Pro", "vendor": "Google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 89.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-3-1-pro", "name": "Gemini 3.1 Pro", "vendor": "Google"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 89.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-5", "name": "GPT-5.5", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 89.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-haiku", "name": "Claude 3.5 Haiku", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 89.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-2", "name": "GPT-5.2", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 88.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-3-5-flash", "name": "Gemini 3.5 Flash", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 88.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-2-medium", "name": "GPT-5.2 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 88.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-pro", "name": "Gemini 2.5 Pro", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 88.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-pro", "name": "Gemini 2.5 Pro", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 87.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o1", "name": "o1", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 87.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 87.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 87.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 86.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-2-xhigh", "name": "GPT-5.2 (xhigh)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 86.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o1", "name": "o1", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 85.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-flash", "name": "Gemini 2.5 Flash", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 85.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-3-0-pro", "name": "Gemini 3.0 Pro", "vendor": "Google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 84.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4o-mini", "name": "GPT-4o mini", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 83.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-2-medium", "name": "GPT-5.2 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 83.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-2", "name": "GPT-5.2", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 81.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-pro", "name": "Gemini 2.5 Pro", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 81.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-flash", "name": "Gemini 2.5 Flash", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 81.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 80.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-2", "name": "GPT-5.2", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 79.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 78.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/gaia/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-haiku", "name": "Claude 3 Haiku", "vendor": "anthropic"}, "benchmark": {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA"}}, {"success_rate": 78.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-haiku", "name": "Claude 3.5 Haiku", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 77.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-flash", "name": "Gemini 2.5 Flash", "vendor": "google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 76.5, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4", "name": "GPT-4", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 75.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 75.25, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4o-mini", "name": "GPT-4o mini", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 74.55, "resolved_count": null, "total_count": null, "cost_usd": 1.08, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 72.75, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-5-haiku", "name": "Claude 3.5 Haiku", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 72.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline_original/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4o-mini", "name": "GPT-4o mini", "vendor": "openai"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)"}}, {"success_rate": 70.91, "resolved_count": null, "total_count": null, "cost_usd": 1.0901, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet-high", "name": "Claude 4.5 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 68.48, "resolved_count": null, "total_count": null, "cost_usd": 3.4075, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus-high", "name": "Claude 4.1 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 66.0, "resolved_count": null, "total_count": null, "cost_usd": null, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": null, "is_harnessrl_measured": false, "eval_protocol": "HAL reliability composite (mean of 4 dimension aggregates)", "source": "hal_reliability_site", "source_url": "https://hal.cs.princeton.edu/reliability/benchmark/taubench_airline/", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-haiku", "name": "Claude 3 Haiku", "vendor": "anthropic"}, "benchmark": {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)"}}, {"success_rate": 64.85, "resolved_count": null, "total_count": null, "cost_usd": 4.0357, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-opus-high", "name": "Claude 4 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 64.24, "resolved_count": null, "total_count": null, "cost_usd": 0.7424, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet-high", "name": "Claude 3.7 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 64.24, "resolved_count": null, "total_count": null, "cost_usd": 3.8901, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus", "name": "Claude 4.1 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 59.39, "resolved_count": null, "total_count": null, "cost_usd": 0.6348, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 56.36, "resolved_count": null, "total_count": null, "cost_usd": 0.792, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 56.36, "resolved_count": null, "total_count": null, "cost_usd": 0.7928, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-haiku", "name": "Claude 4.5 Haiku", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 56.0, "resolved_count": null, "total_count": null, "cost_usd": 0.8422, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 54.55, "resolved_count": null, "total_count": null, "cost_usd": 0.3599, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 54.0, "resolved_count": null, "total_count": null, "cost_usd": 3.6098, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus", "name": "Claude 4.1 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 49.7, "resolved_count": null, "total_count": null, "cost_usd": 0.4496, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 46.0, "resolved_count": null, "total_count": null, "cost_usd": 8.1618, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus-high", "name": "Claude 4.1 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 44.0, "resolved_count": null, "total_count": null, "cost_usd": 1.3329, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-haiku-high", "name": "Claude 4.5 Haiku (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 44.0, "resolved_count": null, "total_count": null, "cost_usd": 0.6916, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet-high", "name": "Claude 3.7 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 44.0, "resolved_count": null, "total_count": null, "cost_usd": 3.003, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-opus", "name": "Claude 4 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 44.0, "resolved_count": null, "total_count": null, "cost_usd": 3.0058, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-opus-high", "name": "Claude 4 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 42.0, "resolved_count": null, "total_count": null, "cost_usd": 9.748, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus", "name": "Claude 4.1 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 40.0, "resolved_count": null, "total_count": null, "cost_usd": 1.9586, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet-high", "name": "Claude 4.5 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 37.78, "resolved_count": null, "total_count": null, "cost_usd": 1.47, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet-high", "name": "Claude 3.7 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 35.56, "resolved_count": null, "total_count": null, "cost_usd": 2.2504, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-3-pro-preview", "name": "Gemini 3 Pro Preview", "vendor": "Google"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 35.56, "resolved_count": null, "total_count": null, "cost_usd": 8.3358, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus", "name": "Claude 4.1 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 34.0, "resolved_count": null, "total_count": null, "cost_usd": 2.6161, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 34.0, "resolved_count": null, "total_count": null, "cost_usd": 7.8039, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-opus", "name": "Claude 4 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 33.33, "resolved_count": null, "total_count": null, "cost_usd": 1.8931, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet", "name": "Claude 4.5 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 33.33, "resolved_count": null, "total_count": null, "cost_usd": 2.8313, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-opus", "name": "Claude 4.5 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 33.33, "resolved_count": null, "total_count": null, "cost_usd": 7.966, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus-high", "name": "Claude 4.1 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 32.73, "resolved_count": null, "total_count": null, "cost_usd": 0.0473, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 32.0, "resolved_count": null, "total_count": null, "cost_usd": 2.8056, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-1-opus-high", "name": "Claude 4.1 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 31.11, "resolved_count": null, "total_count": null, "cost_usd": 1.2587, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 31.11, "resolved_count": null, "total_count": null, "cost_usd": 2.4973, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-opus-high", "name": "Claude 4.5 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 30.3, "resolved_count": null, "total_count": null, "cost_usd": 0.4436, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-r1-january-2025", "name": "DeepSeek R1 (January 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 30.3, "resolved_count": null, "total_count": null, "cost_usd": 1.6531, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-opus", "name": "Claude 4 Opus", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 30.0, "resolved_count": null, "total_count": null, "cost_usd": 8.2331, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-opus-high", "name": "Claude 4 Opus (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 30.0, "resolved_count": null, "total_count": null, "cost_usd": 1.0556, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 29.39, "resolved_count": null, "total_count": null, "cost_usd": 0.1055, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-v3-march-2025", "name": "DeepSeek V3 (March 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 28.89, "resolved_count": null, "total_count": null, "cost_usd": 1.9504, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-sonnet-high", "name": "Claude 4.5 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 28.48, "resolved_count": null, "total_count": null, "cost_usd": 17.1427, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 165, "is_harnessrl_measured": false, "eval_protocol": "HAL GAIA (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/gaia", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "hal-gaia", "name": "GAIA (HAL)"}}, {"success_rate": 26.0, "resolved_count": null, "total_count": null, "cost_usd": 2.3965, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 25.41, "resolved_count": null, "total_count": null, "cost_usd": 1.4617, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 135, "is_harnessrl_measured": false, "eval_protocol": "HAL USACO (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/usaco", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-usaco", "name": "USACO (HAL)"}}, {"success_rate": 24.0, "resolved_count": null, "total_count": null, "cost_usd": 1.4894, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet-high", "name": "Claude 3.7 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 24.0, "resolved_count": null, "total_count": null, "cost_usd": 3.0182, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-4-5-haiku", "name": "Claude 4.5 Haiku", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 22.22, "resolved_count": null, "total_count": null, "cost_usd": 1.296, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 22.22, "resolved_count": null, "total_count": null, "cost_usd": 1.9631, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 22.0, "resolved_count": null, "total_count": null, "cost_usd": 0.04, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 20.0, "resolved_count": null, "total_count": null, "cost_usd": 0.9006, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 19.61, "resolved_count": null, "total_count": null, "cost_usd": 0.758, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 102, "is_harnessrl_measured": false, "eval_protocol": "HAL ScienceAgentBench (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scienceagentbench", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "benchmark": {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)"}}, {"success_rate": 18.0, "resolved_count": null, "total_count": null, "cost_usd": 0.4114, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 18.0, "resolved_count": null, "total_count": null, "cost_usd": 0.2146, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-v3-march-2025", "name": "DeepSeek V3 (March 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 17.65, "resolved_count": null, "total_count": null, "cost_usd": 0.4733, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 102, "is_harnessrl_measured": false, "eval_protocol": "HAL ScienceAgentBench (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scienceagentbench", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet-high", "name": "Claude 3.7 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)"}}, {"success_rate": 16.0, "resolved_count": null, "total_count": null, "cost_usd": 0.357, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 15.56, "resolved_count": null, "total_count": null, "cost_usd": 0.5, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 12.0, "resolved_count": null, "total_count": null, "cost_usd": 1.1751, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 11.11, "resolved_count": null, "total_count": null, "cost_usd": 0.6611, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-5-medium", "name": "GPT-5 (medium)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 10.78, "resolved_count": null, "total_count": null, "cost_usd": 0.4041, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 102, "is_harnessrl_measured": false, "eval_protocol": "HAL ScienceAgentBench (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scienceagentbench", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)"}}, {"success_rate": 10.0, "resolved_count": null, "total_count": null, "cost_usd": 0.6157, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-v3-march-2025", "name": "DeepSeek V3 (March 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 10.0, "resolved_count": null, "total_count": null, "cost_usd": 0.6036, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 50, "is_harnessrl_measured": false, "eval_protocol": "HAL TAU-bench Airline (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/taubench_airline", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-r1-january-2025", "name": "DeepSeek R1 (January 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)"}}, {"success_rate": 9.8, "resolved_count": null, "total_count": null, "cost_usd": 0.3047, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 102, "is_harnessrl_measured": false, "eval_protocol": "HAL ScienceAgentBench (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scienceagentbench", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)"}}, {"success_rate": 8.89, "resolved_count": null, "total_count": null, "cost_usd": 0.0456, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-oss-120b-high-august-2025", "name": "GPT OSS 120B High (August 2025)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 8.89, "resolved_count": null, "total_count": null, "cost_usd": 0.062, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-oss-120b-august-2025", "name": "GPT OSS 120B (August 2025)", "vendor": "OpenAI"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 8.89, "resolved_count": null, "total_count": null, "cost_usd": 0.1042, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-v3-march-2025", "name": "DeepSeek V3 (March 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 8.89, "resolved_count": null, "total_count": null, "cost_usd": 0.1727, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-r1-may-2025", "name": "DeepSeek R1 (May 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 6.86, "resolved_count": null, "total_count": null, "cost_usd": 0.676, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 102, "is_harnessrl_measured": false, "eval_protocol": "HAL ScienceAgentBench (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scienceagentbench", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)"}}, {"success_rate": 6.0, "resolved_count": null, "total_count": null, "cost_usd": 2.9941, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-r1-january-2025", "name": "DeepSeek R1 (January 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 4.45, "resolved_count": null, "total_count": null, "cost_usd": 0.5544, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-r1-january-2025", "name": "DeepSeek R1 (January 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 4.44, "resolved_count": null, "total_count": null, "cost_usd": 0.1569, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 4.44, "resolved_count": null, "total_count": null, "cost_usd": 0.6751, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 45, "is_harnessrl_measured": false, "eval_protocol": "HAL CORE-Bench Hard (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/corebench_hard", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-5-pro-preview", "name": "Gemini 2.5 Pro Preview", "vendor": "google"}, "benchmark": {"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)"}}, {"success_rate": 3.08, "resolved_count": null, "total_count": null, "cost_usd": 0.755, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 3.08, "resolved_count": null, "total_count": null, "cost_usd": 0.8373, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 3.08, "resolved_count": null, "total_count": null, "cost_usd": 2.3519, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "claude-3-7-sonnet-high", "name": "Claude 3.7 Sonnet (high)", "vendor": "Anthropic"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 2.0, "resolved_count": null, "total_count": null, "cost_usd": 0.6535, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 2.0, "resolved_count": null, "total_count": null, "cost_usd": 0.1496, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 2.0, "resolved_count": null, "total_count": null, "cost_usd": 1.0571, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}, {"success_rate": 1.54, "resolved_count": null, "total_count": null, "cost_usd": 1.1512, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o4-mini", "name": "o4 mini", "vendor": "openai"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 1.54, "resolved_count": null, "total_count": null, "cost_usd": 0.9234, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gpt-4-1", "name": "GPT-4.1", "vendor": "openai"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 0.98, "resolved_count": null, "total_count": null, "cost_usd": 0.5464, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 102, "is_harnessrl_measured": false, "eval_protocol": "HAL ScienceAgentBench (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scienceagentbench", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-v3-march-2025", "name": "DeepSeek V3 (March 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)"}}, {"success_rate": 0.0, "resolved_count": null, "total_count": null, "cost_usd": 0.7686, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "gemini-2-0-flash", "name": "Gemini 2.0 Flash", "vendor": "Google"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 0.0, "resolved_count": null, "total_count": null, "cost_usd": 2.742, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-v3-march-2025", "name": "DeepSeek V3 (March 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 0.0, "resolved_count": null, "total_count": null, "cost_usd": 6.0847, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 80, "is_harnessrl_measured": false, "eval_protocol": "HAL SciCode (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/scicode", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "deepseek-r1-january-2025", "name": "DeepSeek R1 (January 2025)", "vendor": "DeepSeek"}, "benchmark": {"slug": "hal-scicode", "name": "SciCode (HAL)"}}, {"success_rate": 0.0, "resolved_count": null, "total_count": null, "cost_usd": 11.9533, "input_tokens": null, "output_tokens": null, "latency_ms": null, "temperature": null, "max_turns": null, "context_budget_tokens": null, "harness_version": null, "trial_count": 49, "is_harnessrl_measured": false, "eval_protocol": "HAL SWE-bench Verified Mini (HAL) accuracy", "source": "hal_site", "source_url": "https://hal.cs.princeton.edu/swebench_verified_mini", "observed_at": null, "harness": {"slug": "hal-generalist-agent", "name": "HAL Generalist Agent", "vendor": "Princeton PLI", "repo_url": "https://github.com/princeton-pli/hal-harness", "openrouter_icon_url": null}, "model": {"slug": "o3", "name": "o3", "vendor": "openai"}, "benchmark": {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)"}}], "benchmarks": [{"slug": "hal-corebench-hard", "name": "CORE-Bench Hard (HAL)", "category": "eval"}, {"slug": "hal-gaia", "name": "GAIA (HAL)", "category": "eval"}, {"slug": "hal-reliability-gaia", "name": "HAL Reliability \u2014 GAIA", "category": "eval"}, {"slug": "hal-reliability-tau-bench-airline", "name": "HAL Reliability \u2014 \u03c4-bench Airline (clean)", "category": "eval"}, {"slug": "hal-reliability-tau-bench-airline-original", "name": "HAL Reliability \u2014 \u03c4-bench Airline (original)", "category": "eval"}, {"slug": "hal-swe-bench-verified-mini", "name": "SWE-bench Verified Mini (HAL)", "category": "eval"}, {"slug": "hal-scicode", "name": "SciCode (HAL)", "category": "eval"}, {"slug": "hal-scienceagentbench", "name": "ScienceAgentBench (HAL)", "category": "eval"}, {"slug": "hal-tau-bench-airline", "name": "TAU-bench Airline (HAL)", "category": "eval"}, {"slug": "hal-usaco", "name": "USACO (HAL)", "category": "eval"}]}