{ "generatedAt": "2026-08-08T10:21:51.656Z", "generator": { "name": "agent-bench/aggregate.mjs", "sourceCommit": "6633ce37071e3276e7312439907d24043e3f7c5c" }, "methodology": "docs/AGENT_BENCH_METHODOLOGY.md", "runtime": [ "claude-sonnet-5 maxTurns=30 repairLoop=true" ], "resultFiles": [ "2026-08-04T15-05-04-claude.json", "2026-08-04T15-06-36-claude.json", "2026-08-04T15-07-35-claude.json", "2026-08-04T15-26-06-claude.json", "2026-08-04T21-25-31-claude.json", "2026-08-06T20-20-46-claude.json", "2026-08-08T10-20-51-claude.json" ], "totalRuns": 90, "totalCostUsd": 78.57, "notes": [ "Arms are identical except the workspace guidance file: WITH documents the dt CLI affordances; WITHOUT is generic with the same repository access. The control can discover the affordances on its own, so measured lift under-claims.", "Acceptance is affordance-neutral (semantics, not implementation); design-system reuse is reported separately and never gates a pass.", "The migration category is from a re-run after a task-spec fix (#1406): the original brief enumerated 4 files while acceptance swept wider. The corrected brief demands all consumers and acceptance matches; the confounded original migration results are excluded.", "On the corrected migration task both arms passed 3/3; the affordance effect there is efficiency (WITH mean ~$1.18 and ~25 turns vs WITHOUT ~$1.73 and ~34 turns), with one WITHOUT run needing the repair loop.", "num_turns counts top-level turns only; runs that delegate to subagents can show few turns at normal cost.", "Batch 2 (2026-08-06, 30 runs, $26.87) doubles every cell to n=6 under the identical pinned methodology; cost figures are mean +/- sample sd across both batches.", "Batch 3 (2026-08-08, 30 runs): identical pinned methodology, run against the post-Phase-4 HEAD where the affordance layer includes the seven per-publish evidence artifacts, enriched playground defaults, and the parity-repaired components. First batch with 30/30 passes in both arms." ], "arms": { "with": { "runs": 45, "firstTryPass": 45, "finalPass": 45, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.8183, "sd": 0.2868, "n": 45, "min": 0.5022, "max": 2.0067 }, "initialTurns": { "mean": 17.3333, "sd": 6.3996, "n": 45, "min": 9, "max": 33 }, "dsReuse": { "hits": 14, "eligible": 27 } }, "without": { "runs": 45, "firstTryPass": 42, "finalPass": 45, "passViaRepairLoop": 3, "costUsdPerRun": { "mean": 0.9277, "sd": 0.5445, "n": 45, "min": 0.4534, "max": 2.1216 }, "initialTurns": { "mean": 16.6, "sd": 10.0349, "n": 45, "min": 1, "max": 41 }, "dsReuse": { "hits": 2, "eligible": 27 } } }, "tasks": [ { "id": "forced-colors-chip", "category": "forced-colors", "with": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.988, "sd": 0.4289, "n": 9, "min": 0.563, "max": 2.0067 }, "initialTurns": { "mean": 20.6667, "sd": 6.7823, "n": 9, "min": 12, "max": 33 }, "dsReuse": { "hits": 1, "eligible": 9 } }, "without": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 1.238, "sd": 0.4343, "n": 9, "min": 0.7265, "max": 2.0977 }, "initialTurns": { "mean": 21.1111, "sd": 9.1576, "n": 9, "min": 1, "max": 31 }, "dsReuse": { "hits": 1, "eligible": 9 } } }, { "id": "migration-badge-rename", "category": "migration", "with": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 1.0947, "sd": 0.1941, "n": 9, "min": 0.8474, "max": 1.3702 }, "initialTurns": { "mean": 24.4444, "sd": 5.2467, "n": 9, "min": 17, "max": 31 }, "dsReuse": { "hits": 0, "eligible": 0 } }, "without": { "runs": 9, "firstTryPass": 6, "finalPass": 9, "passViaRepairLoop": 3, "costUsdPerRun": { "mean": 1.7199, "sd": 0.3867, "n": 9, "min": 1.2209, "max": 2.1216 }, "initialTurns": { "mean": 32.1111, "sd": 3.3333, "n": 9, "min": 31, "max": 41 }, "dsReuse": { "hits": 0, "eligible": 0 } } }, { "id": "repair-status-panel", "category": "repair", "with": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.5518, "sd": 0.0448, "n": 9, "min": 0.5086, "max": 0.662 }, "initialTurns": { "mean": 10.5556, "sd": 0.8819, "n": 9, "min": 9, "max": 12 }, "dsReuse": { "hits": 0, "eligible": 0 } }, "without": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.5884, "sd": 0.0767, "n": 9, "min": 0.4889, "max": 0.712 }, "initialTurns": { "mean": 9.7778, "sd": 2.6822, "n": 9, "min": 5, "max": 13 }, "dsReuse": { "hits": 0, "eligible": 0 } } }, { "id": "table-species", "category": "table", "with": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.7191, "sd": 0.0861, "n": 9, "min": 0.5022, "max": 0.7839 }, "initialTurns": { "mean": 16.1111, "sd": 2.8916, "n": 9, "min": 9, "max": 19 }, "dsReuse": { "hits": 8, "eligible": 9 } }, "without": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.5184, "sd": 0.0366, "n": 9, "min": 0.4817, "max": 0.5976 }, "initialTurns": { "mean": 9.4444, "sd": 0.8819, "n": 9, "min": 8, "max": 11 }, "dsReuse": { "hits": 0, "eligible": 9 } } }, { "id": "tree-taxonomy", "category": "tree", "with": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.738, "sd": 0.0704, "n": 9, "min": 0.6244, "max": 0.8228 }, "initialTurns": { "mean": 14.8889, "sd": 3.6209, "n": 9, "min": 10, "max": 19 }, "dsReuse": { "hits": 5, "eligible": 9 } }, "without": { "runs": 9, "firstTryPass": 9, "finalPass": 9, "passViaRepairLoop": 0, "costUsdPerRun": { "mean": 0.5738, "sd": 0.1085, "n": 9, "min": 0.4534, "max": 0.7973 }, "initialTurns": { "mean": 10.5556, "sd": 2.5055, "n": 9, "min": 8, "max": 16 }, "dsReuse": { "hits": 1, "eligible": 9 } } } ] }