{ "generated_at": "2026-10-07T02:00:52+00:00", "rerun_bench_version": "0.2.1", "k": 10, "entries": [ { "run_id": "claude-opus-5-5", "agent": "claude", "model": "claude-opus-5-5", "models_reported": [ "claude-opus-5-5" ], "cli_version": "2.1.292 (Claude Code)", "started_at": "2026-10-07T01:26:17+00:00", "simulated": false, "planned_runs": 100, "stopped": null, "metrics": { "n_tasks": 10, "n_runs": 100, "min_runs_per_task": 10, "passes": 100, "agent_error_runs": 0, "timeout_runs": 0, "pass_rate": 1.0, "pass_rate_ci95": [ 0.9630065017930143, 1.0 ], "macro_pass_rate": 1.0, "macro_pass_rate_task_bootstrap_ci95": [ 1.0, 1.0 ], "k": 10, "pass_at_k": 1.0, "pass_hat_k": 1.0, "flip_rate": 0.0, "flaky_task_fraction": 0.0, "total_cost_usd": 9.7213368, "cost_reported_runs": 100, "mean_cost_usd": 0.097213368, "median_cost_usd": 0.0899442, "cost_per_success_usd": 0.097213368, "cost_cv_within_task": 0.043432157377852054, "tokens_mean": 59759.2, "tokens_median": 53367.5, "output_tokens_median": 1015.5, "tokens_cv_within_task": 0.08062932770520388, "wall_time_mean_s": 13.81801, "wall_time_median_s": 12.67, "wall_time_total_s": 1381.801, "wall_time_cv_within_task": 0.1987011385162314, "approach_similarity": 0.8202072157583584, "per_task": { "add-cli-flag": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.09863282, "median": 0.0927327, "std": 0.01080361619930002, "cv": 0.10953368462242101, "min": 0.0878862, "max": 0.11188119999999999, "iqr": 0.02054974999999999 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 60292.4, "median": 52423.5, "std": 10445.000911653595, "cv": 0.1732390966631548, "min": 51992.0, "max": 72945.0, "iqr": 20020.25 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 1214.5, "median": 1149.0, "std": 161.44021941401232, "cv": 0.13292731116839218, "min": 1024.0, "max": 1430.0, "iqr": 284.25 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 13.4091, "median": 13.1985, "std": 1.8792195454496525, "cv": 0.14014509142669176, "min": 10.378, "max": 16.676, "iqr": 1.744250000000001 }, "approach_similarity": 0.7467691567691568, "timeouts": 0, "agent_errors": 0 }, "edit-config": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.08172098, "median": 0.08142930000000001, "std": 0.004703738073253844, "cv": 0.05755851279871881, "min": 0.0727198, "max": 0.0924924, "iqr": 0.0008663000000000143 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 53078.3, "median": 51330.5, "std": 5758.025106077796, "cv": 0.10848171674823413, "min": 50642.0, "max": 69454.0, "iqr": 50.0 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 766.9, "median": 763.0, "std": 122.10237052935905, "cv": 0.15921550466730872, "min": 497.0, "max": 1007.0, "iqr": 43.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 8.734399999999999, "median": 7.577999999999999, "std": 2.304653369935608, "cv": 0.2638593801446703, "min": 6.496, "max": 12.772, "iqr": 2.3522500000000006 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "fix-failing-test": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.08386074, "median": 0.08430780000000002, "std": 0.0013535420530346805, "cv": 0.01614035427107703, "min": 0.08103999999999999, "max": 0.0852124, "iqr": 0.00043590000000000295 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 68636.5, "median": 68690.5, "std": 146.61684759944882, "cv": 0.0021361352574715904, "min": 68340.0, "max": 68760.0, "iqr": 42.75 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 654.4, "median": 657.5, "std": 21.644604151817813, "cv": 0.03307549534201989, "min": 612.0, "max": 694.0, "iqr": 18.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 9.9595, "median": 9.386, "std": 2.362623788644029, "cv": 0.2372231325512354, "min": 7.853, "max": 15.1, "iqr": 2.705499999999999 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "follow-agents-md": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.0850817, "median": 0.0850234, "std": 0.0009556069915097026, "cv": 0.011231639606515886, "min": 0.0833418, "max": 0.0867598, "iqr": 0.0007156499999999982 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 52267.0, "median": 52296.0, "std": 100.95653630261997, "cv": 0.0019315540647563466, "min": 52113.0, "max": 52409.0, "iqr": 156.0 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 875.1, "median": 871.0, "std": 32.63757207745562, "cv": 0.037295819994807015, "min": 829.0, "max": 919.0, "iqr": 49.5 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 10.654900000000001, "median": 9.6245, "std": 2.2681665356259306, "cv": 0.21287544093571317, "min": 8.692, "max": 14.944, "iqr": 1.6575000000000006 }, "approach_similarity": 0.7872571872571873, "timeouts": 0, "agent_errors": 0 }, "implement-lru-cache": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.10116662000000001, "median": 0.10094149999999999, "std": 0.002441400005370327, "cv": 0.02413246588025108, "min": 0.097944, "max": 0.10692860000000001, "iqr": 0.0010137999999999953 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 52980.1, "median": 52936.0, "std": 228.15708526266624, "cv": 0.004306467622044244, "min": 52674.0, "max": 53521.0, "iqr": 188.25 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 1498.7, "median": 1531.5, "std": 106.39554084223225, "cv": 0.07099188686343648, "min": 1311.0, "max": 1626.0, "iqr": 151.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 14.8506, "median": 14.7685, "std": 0.9663927427983577, "cv": 0.06507432311141353, "min": 13.643, "max": 16.348, "iqr": 1.3282500000000006 }, "approach_similarity": 0.8162806061356787, "timeouts": 0, "agent_errors": 0 }, "implement-slugify": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.09095432, "median": 0.0909947, "std": 0.00176623114682082, "cv": 0.019418881333188132, "min": 0.0881304, "max": 0.0938994, "iqr": 0.0021582499999999866 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 51946.3, "median": 51971.5, "std": 108.65649236623338, "cv": 0.0020917080209029975, "min": 51803.0, "max": 52107.0, "iqr": 180.5 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 1254.0, "median": 1243.0, "std": 75.86977146546721, "cv": 0.0605022101000536, "min": 1124.0, "max": 1388.0, "iqr": 76.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 13.9299, "median": 13.5265, "std": 2.4315210351638843, "cv": 0.17455409121127102, "min": 11.059, "max": 19.382, "iqr": 2.651500000000002 }, "approach_similarity": 0.5837551990493167, "timeouts": 0, "agent_errors": 0 }, "minimal-fix": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.08492066000000001, "median": 0.0839943, "std": 0.0020692178684495995, "cv": 0.024366483591267418, "min": 0.0832788, "max": 0.0898614, "iqr": 0.00202380000000002 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 70310.7, "median": 68549.0, "std": 5535.175437348144, "cv": 0.07872451045641908, "min": 68374.0, "max": 86057.0, "iqr": 302.75 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 775.6, "median": 748.5, "std": 44.71688719041163, "cv": 0.05765457347912794, "min": 732.0, "max": 860.0, "iqr": 67.75 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 12.0709, "median": 9.6495, "std": 4.69052123258528, "cv": 0.38858090387504496, "min": 8.734, "max": 22.658, "iqr": 5.156000000000001 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "multi-file-rename": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.08758462, "median": 0.0897299, "std": 0.005807651183625495, "cv": 0.06630902986877713, "min": 0.07824400000000001, "max": 0.09413919999999999, "iqr": 0.008092100000000005 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 64338.6, "median": 69723.5, "std": 8874.780550651505, "cv": 0.1379386643578117, "min": 51356.0, "max": 70320.0, "iqr": 13751.5 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 792.6, "median": 819.5, "std": 87.48231567325796, "cv": 0.11037385272931864, "min": 664.0, "max": 901.0, "iqr": 139.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 10.3191, "median": 9.649000000000001, "std": 3.261092129605936, "cv": 0.3160248596879511, "min": 7.663, "max": 18.657, "iqr": 1.5417500000000004 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "refactor-extract-helper": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.10720904, "median": 0.10735030000000001, "std": 0.004270912987797031, "cv": 0.03983724681983003, "min": 0.1012232, "max": 0.11377000000000001, "iqr": 0.004616250000000016 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 57243.0, "median": 53717.0, "std": 7673.704161326234, "cv": 0.13405489162563516, "min": 53167.0, "max": 72129.0, "iqr": 870.5 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 1587.6, "median": 1596.0, "std": 124.42061279743356, "cv": 0.07837025245492163, "min": 1395.0, "max": 1772.0, "iqr": 112.5 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 16.279, "median": 15.9555, "std": 2.0318997131857772, "cv": 0.12481723159811887, "min": 14.031, "max": 21.093, "iqr": 1.6872500000000006 }, "approach_similarity": 0.9714285714285714, "timeouts": 0, "agent_errors": 0 }, "write-tests": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 10, "n_missing": 0, "mean": 0.15100218, "median": 0.15220109999999998, "std": 0.009934927952297052, "cv": 0.06579327498647404, "min": 0.136321, "max": 0.1657922, "iqr": 0.014411900000000005 }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 66499.1, "median": 66370.0, "std": 10865.190343988968, "cv": 0.1633885322356087, "min": 55283.0, "max": 77773.0, "iqr": 20822.0 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 3137.9, "median": 3106.0, "std": 207.15718669647936, "cv": 0.06601777835382878, "min": 2831.0, "max": 3476.0, "iqr": 294.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 27.972699999999996, "median": 28.2265, "std": 1.7862507631597777, "cv": 0.06385693062020391, "min": 24.927, "max": 30.31, "iqr": 2.0702500000000015 }, "approach_similarity": 0.29658143694367334, "timeouts": 0, "agent_errors": 0 } } } }, { "run_id": "codex", "agent": "codex", "model": "gpt-6-luna", "models_reported": [ "gpt-6-luna" ], "cli_version": "codex-cli 0.160.0", "started_at": "2026-10-06T17:43:42+00:00", "simulated": false, "planned_runs": 100, "stopped": null, "metrics": { "n_tasks": 10, "n_runs": 100, "min_runs_per_task": 10, "passes": 96, "agent_error_runs": 0, "timeout_runs": 0, "pass_rate": 0.96, "pass_rate_ci95": [ 0.9016292856411208, 0.9843366960084523 ], "macro_pass_rate": 0.96, "macro_pass_rate_task_bootstrap_ci95": [ 0.9199999999999999, 0.99 ], "k": 10, "pass_at_k": 1.0, "pass_hat_k": 0.7, "flip_rate": 0.07555555555555556, "flaky_task_fraction": 0.3, "total_cost_usd": null, "cost_reported_runs": 0, "mean_cost_usd": null, "median_cost_usd": null, "cost_per_success_usd": null, "cost_cv_within_task": null, "tokens_mean": 60819.4, "tokens_median": 56552.0, "output_tokens_median": 552.0, "tokens_cv_within_task": 0.15934344217569843, "wall_time_mean_s": 26.2629, "wall_time_median_s": 16.819, "wall_time_total_s": 2626.29, "wall_time_cv_within_task": 0.9033985003148407, "approach_similarity": 0.7547088559680863, "per_task": { "add-cli-flag": { "n": 10, "passes": 9, "pass_rate": 0.9, "pass_rate_ci95": [ 0.5958499732047615, 0.9821237869049271 ], "pass_at_k": { "1": 0.9, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 0.9, "2": 0.8, "3": 0.7, "4": 0.6, "5": 0.5, "6": 0.4, "7": 0.3, "8": 0.2, "9": 0.1, "10": 0.0 }, "flip_rate": 0.2, "outcomes": [ true, true, true, true, true, false, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 58109.6, "median": 56109.0, "std": 9073.150184289174, "cv": 0.15613857579968154, "min": 41462.0, "max": 73910.0, "iqr": 1168.75 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 481.2, "median": 509.5, "std": 101.2629141284102, "cv": 0.2104383086625316, "min": 205.0, "max": 552.0, "iqr": 51.5 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 18.038400000000003, "median": 13.434, "std": 17.809562757125736, "cv": 0.9873138835554003, "min": 6.984, "max": 67.296, "iqr": 8.2975 }, "approach_similarity": 0.6152477152477153, "timeouts": 0, "agent_errors": 0 }, "edit-config": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 48197.7, "median": 42391.5, "std": 8373.09235388376, "cv": 0.17372389873134528, "min": 41469.0, "max": 60782.0, "iqr": 15424.0 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 313.8, "median": 296.5, "std": 43.84771627145731, "cv": 0.13973140940553636, "min": 270.0, "max": 397.0, "iqr": 70.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 28.4875, "median": 12.911, "std": 24.605566187122964, "cv": 0.863732029385624, "min": 7.147, "max": 63.903, "iqr": 43.049 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "fix-failing-test": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 80622.6, "median": 84724.0, "std": 6949.231247651306, "cv": 0.08619458126693143, "min": 70068.0, "max": 85226.0, "iqr": 10838.0 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 439.5, "median": 416.0, "std": 100.84779510617858, "cv": 0.22946028465569643, "min": 332.0, "max": 638.0, "iqr": 41.75 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 26.5334, "median": 16.625999999999998, "std": 24.990533995450715, "cv": 0.9418519298488213, "min": 9.717, "max": 87.293, "iqr": 9.685499999999998 }, "approach_similarity": 0.8, "timeouts": 0, "agent_errors": 0 }, "follow-agents-md": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 56841.8, "median": 56601.5, "std": 11931.317500129182, "cv": 0.20990393513451688, "min": 42192.0, "max": 72858.0, "iqr": 21229.25 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 543.4, "median": 550.0, "std": 85.67794997028764, "cv": 0.15767013244440126, "min": 400.0, "max": 647.0, "iqr": 129.75 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 21.3217, "median": 16.011, "std": 16.039461143968925, "cv": 0.7522599578818258, "min": 11.301, "max": 62.569, "iqr": 4.551750000000002 }, "approach_similarity": 0.624027084027084, "timeouts": 0, "agent_errors": 0 }, "implement-lru-cache": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 59702.0, "median": 56637.5, "std": 11542.305026871078, "cv": 0.19333196587838058, "min": 42583.0, "max": 86505.0, "iqr": 741.25 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 760.4, "median": 744.5, "std": 98.04103449293282, "cv": 0.12893350143731303, "min": 649.0, "max": 995.0, "iqr": 82.25 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 27.6096, "median": 18.0475, "std": 29.788999722865636, "cv": 1.0789363019698088, "min": 11.081, "max": 110.673, "iqr": 11.587750000000002 }, "approach_similarity": 0.6762869604898591, "timeouts": 0, "agent_errors": 0 }, "implement-slugify": { "n": 10, "passes": 9, "pass_rate": 0.9, "pass_rate_ci95": [ 0.5958499732047615, 0.9821237869049271 ], "pass_at_k": { "1": 0.9, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 0.9, "2": 0.8, "3": 0.7, "4": 0.6, "5": 0.5, "6": 0.4, "7": 0.3, "8": 0.2, "9": 0.1, "10": 0.0 }, "flip_rate": 0.2, "outcomes": [ true, true, true, true, true, true, true, false, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 50628.9, "median": 56081.5, "std": 11950.023834936714, "cv": 0.23603167034908348, "min": 27546.0, "max": 70300.0, "iqr": 13977.0 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 520.2, "median": 558.0, "std": 146.44361827452002, "cv": 0.2815140681940023, "min": 115.0, "max": 624.0, "iqr": 40.5 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 30.058999999999997, "median": 15.891, "std": 27.434565432031984, "cv": 0.9126905563069958, "min": 8.31, "max": 81.669, "iqr": 37.565 }, "approach_similarity": 0.6372709874548109, "timeouts": 0, "agent_errors": 0 }, "minimal-fix": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 53329.2, "median": 56028.0, "std": 6175.3146892373925, "cv": 0.11579612462285939, "min": 41621.0, "max": 57239.0, "iqr": 392.75 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 320.2, "median": 297.5, "std": 76.05378213986322, "cv": 0.23751961942493197, "min": 237.0, "max": 448.0, "iqr": 120.5 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 21.3208, "median": 12.5115, "std": 21.77930992580905, "cv": 1.0215052871284873, "min": 6.781, "max": 70.447, "iqr": 5.827000000000002 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "multi-file-rename": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 75719.0, "median": 72283.5, "std": 11391.33233842488, "cv": 0.1504421920313908, "min": 60085.0, "max": 101035.0, "iqr": 1998.5 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 693.6, "median": 678.0, "std": 99.70422926502833, "cv": 0.14374888879040992, "min": 583.0, "max": 959.0, "iqr": 25.75 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 26.667700000000004, "median": 21.639499999999998, "std": 25.685505813590666, "cv": 0.9631691452052732, "min": 10.907, "max": 98.33, "iqr": 7.341999999999995 }, "approach_similarity": 1.0, "timeouts": 0, "agent_errors": 0 }, "refactor-extract-helper": { "n": 10, "passes": 8, "pass_rate": 0.8, "pass_rate_ci95": [ 0.49016247153664183, 0.9433178485456247 ], "pass_at_k": { "1": 0.8, "2": 0.9777777777777777, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 0.8, "2": 0.6222222222222222, "3": 0.4666666666666667, "4": 0.3333333333333333, "5": 0.2222222222222222, "6": 0.13333333333333333, "7": 0.06666666666666667, "8": 0.022222222222222223, "9": 0.0, "10": 0.0 }, "flip_rate": 0.35555555555555557, "outcomes": [ true, true, true, true, true, true, true, false, true, false ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 52075.4, "median": 56320.0, "std": 6911.344445507289, "cv": 0.13271802896391172, "min": 41509.0, "max": 56455.0, "iqr": 10053.5 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 528.2, "median": 615.0, "std": 184.86319265878754, "cv": 0.3499871121900559, "min": 176.0, "max": 654.0, "iqr": 85.25 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 20.9255, "median": 17.0385, "std": 19.261291835353795, "cv": 0.9204698494828699, "min": 8.852, "max": 74.105, "iqr": 9.818500000000002 }, "approach_similarity": 0.9807692307692308, "timeouts": 0, "agent_errors": 0 }, "write-tests": { "n": 10, "passes": 10, "pass_rate": 1.0, "pass_rate_ci95": [ 0.7224672001371107, 1.0 ], "pass_at_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "pass_hat_k": { "1": 1.0, "2": 1.0, "3": 1.0, "4": 1.0, "5": 1.0, "6": 1.0, "7": 1.0, "8": 1.0, "9": 1.0, "10": 1.0 }, "flip_rate": 0.0, "outcomes": [ true, true, true, true, true, true, true, true, true, true ], "cost_usd": { "n": 0, "n_missing": 10, "mean": null, "median": null, "std": null, "cv": null, "min": null, "max": null, "iqr": null }, "total_tokens": { "n": 10, "n_missing": 0, "mean": 72967.8, "median": 72748.0, "std": 10153.721034401351, "cv": 0.13915344897888315, "min": 57548.0, "max": 88510.0, "iqr": 1973.25 }, "output_tokens": { "n": 10, "n_missing": 0, "mean": 1249.7, "median": 1270.0, "std": 89.12294878424973, "cv": 0.0713154747413377, "min": 1070.0, "max": 1372.0, "iqr": 92.0 }, "wall_time_s": { "n": 10, "n_missing": 0, "mean": 41.6654, "median": 33.769, "std": 24.668252661625182, "cv": 0.5920560623833009, "min": 18.31, "max": 79.062, "iqr": 39.342 }, "approach_similarity": 0.2134865816921625, "timeouts": 0, "agent_errors": 0 } } } } ], "tasks": [ "add-cli-flag", "edit-config", "fix-failing-test", "follow-agents-md", "implement-lru-cache", "implement-slugify", "minimal-fix", "multi-file-rename", "refactor-extract-helper", "write-tests" ], "comparison": [ { "a": "claude-opus-5-5", "b": "codex", "intervals_overlap": true, "same_tasks": true, "same_runs_per_task": true } ], "lab": { "title": "Claude Code and Codex CLI, 10 tasks x 10 runs", "pilot": false, "date": "2026-10-06", "source_url": "https://github.com/Abelo9996/rerun-bench/tree/main/docs/run-2026-10-06", "harness_commit": "ea1934d", "machine": "macOS 26.2, Apple Silicon (arm64), Python 3.13.15", "notes": [ "Claude Code was pinned with --model claude-opus-5-5. With no model flag, Claude Code on this account reported claude-opus-4-8 on 2026-10-06 (it reported claude-opus-5-5 on 2026-10-03, also with the same CLI version 2.1.288), so the pinned set is the one compared here. The 100 default-model runs (100 of 100 passed) are in the source directory.", "Codex CLI ran 2026-10-06 17:43 to 19:02 UTC; Claude Code (pinned) ran 2026-10-07 01:26 to 01:49 UTC.", "One Codex run (write-tests run 3) hung after the Mac went to sleep mid-run and was replaced with a fresh run of the same task and index; the original record is kept in the source directory.", "All 4 failures were Codex CLI runs that read the task files, changed nothing and exited 0 after 7 to 10 seconds. In 3 of them the final message said the change had been made.", "Claude Code reports cost as total_cost_usd at list prices. On a subscription login this is quota used, not money billed. Codex CLI reports tokens only, so its cost is not reported." ] } }