{
  "description": "Task IDs + run configuration for reproducing the ARC-AGI v1 and v2 pilot results published on the benchmarks page. Both runs used Claude Opus 4.6 via claude_code_headless; v1 draws from fchollet/ARC-AGI (800 tasks), v2 from arcprize/ARC-AGI-2 (1120 tasks). Sampling is rng.sample(seed=master_seed, k=25) from the concatenated training+evaluation pool.",
  "arms": [
    "baseline",
    "g6_no_memory",
    "g6_full"
  ],
  "model": "claude-opus-4-6 (via Claude Code headless subprocess)",
  "per_arm_policy": {
    "baseline": {
      "max_iterations": 3,
      "request_timeout_sec": 1800,
      "include_tools": false
    },
    "g6_no_memory": {
      "max_iterations": 3,
      "request_timeout_sec": 1800,
      "include_tools": true
    },
    "g6_full": {
      "max_iterations": 3,
      "request_timeout_sec": 1800,
      "include_tools": true,
      "memory": "AdaptMemoryBlock (rule_sig retrieval)"
    }
  },
  "v1": {
    "dataset": "fchollet/ARC-AGI",
    "source": "data/ARC-AGI/data/{training,evaluation}",
    "master_seed": 10668641,
    "n": 25,
    "task_ids": [
      "025d127b",
      "0607ce86",
      "15696249",
      "310f3251",
      "32e9702f",
      "6cf79266",
      "6d0160f0",
      "7bb29440",
      "7f4411dc",
      "93b4f4b3",
      "992798f6",
      "99fa7670",
      "9a4bb226",
      "9dfd6313",
      "b9630600",
      "bc4146bd",
      "d19f7514",
      "ddf7fa4f",
      "e26a3af2",
      "e345f17b",
      "e4075551",
      "e760a62e",
      "eb5a1d5d",
      "ed36ccf7",
      "f15e1fac"
    ],
    "run_dir": "benchmarking/arc_pilot/runs/20260414T232841Z",
    "git_sha": "653e75a3efa3176896b1304faa4b6442a73edbee",
    "started_at": "2026-04-14T23:28:41.126645Z",
    "finished_at": "2026-04-15T03:16:00.779857Z",
    "status": "complete (75 rows, 3 arms x 25 tasks)"
  },
  "v2": {
    "dataset": "arcprize/ARC-AGI-2",
    "source": "data/ARC-AGI-2/data/{training,evaluation}",
    "master_seed": 10668641,
    "n": 25,
    "task_ids": [
      "03560426",
      "1190bc91",
      "14754a24",
      "1b2d62fb",
      "4c5c2cf0",
      "56ff96f3",
      "575b1a71",
      "67a423a3",
      "692cd3b6",
      "6e19193c",
      "7e576d6e",
      "810b9b61",
      "b7256dcd",
      "b8cdaf2b",
      "c1d99e64",
      "c4d1a9ae",
      "c61be7dc",
      "d23f8c26",
      "d255d7a7",
      "d749d46f",
      "d90796e8",
      "df8cc377",
      "f25fbde4",
      "f3e62deb",
      "fd02da9e"
    ],
    "run_dir": "benchmarking/arc_pilot/runs/20260415T060545Z",
    "git_sha": "729042a746ac968d5536fd3fa5f6d12851bf6c2d",
    "started_at": "2026-04-15T06:05:45.715992Z",
    "finished_at": null,
    "status": "partial (49 rows; 16 paired tasks complete, 9 abandoned for cost)"
  },
  "reproduction": {
    "pilot_runner": "python -m benchmarking.arc_pilot --version v1 --n 25 --arms baseline,g6_no_memory,g6_full",
    "cost_reconstruction": "python benchmarks/test_sets/arc_agi_pilot_reconstruct.py --run <run_dir> --session-dir ~/.claude/projects/<workspace-slug>",
    "pricing_reference": "Opus 4.6: 15/75/1.5/18.75/30 USD per MTok (input/output/cache_read/cache_write_5m/cache_write_1h)"
  }
}