{ "schema_version": 1, "attempts": [ { "attempt_id": "a0001", "group": "medium_8k", "case": "cache", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 247.292, "native_output_tokens": 8550.0, "opencode_output_tokens": 358, "tool_calls": 8, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 247.292, "exit_code": 0, "termination": "completed", "tool_calls": 8, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0002", "group": "medium_8k", "case": "wallet", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 248.808, "native_output_tokens": 8546.0, "opencode_output_tokens": 354, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 248.808, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0003", "group": "medium_8k", "case": "eventstream", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 246.291, "native_output_tokens": 8513.0, "opencode_output_tokens": 321, "tool_calls": 8, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 246.291, "exit_code": 0, "termination": "completed", "tool_calls": 8, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0004", "group": "medium_8k", "case": "boltons_paths", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 390.574, "native_output_tokens": 11360.0, "opencode_output_tokens": 10512, "tool_calls": 36, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 390.574, "exit_code": 0, "termination": "completed", "tool_calls": 36, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 11 } ] }, { "attempt_id": "a0005", "group": "medium_8k", "case": "async_pool", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 525.707, "native_output_tokens": 17817.0, "opencode_output_tokens": 9625, "tool_calls": 15, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 525.707, "exit_code": 0, "termination": "completed", "tool_calls": 15, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 3 } ] }, { "attempt_id": "a0006", "group": "medium_8k", "case": "ledger", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 246.296, "native_output_tokens": 8610.0, "opencode_output_tokens": 418, "tool_calls": 11, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 246.296, "exit_code": 0, "termination": "completed", "tool_calls": 11, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0007", "group": "medium_8k", "case": "pagination", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 296.862, "native_output_tokens": 10393.0, "opencode_output_tokens": 2201, "tool_calls": 11, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 296.862, "exit_code": 0, "termination": "completed", "tool_calls": 11, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0008", "group": "medium_8k", "case": "buildplan", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 245.291, "native_output_tokens": 8593.0, "opencode_output_tokens": 401, "tool_calls": 11, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 245.291, "exit_code": 0, "termination": "completed", "tool_calls": 11, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0009", "group": "medium_8k", "case": "pagination", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 240.262, "native_output_tokens": 8477.0, "opencode_output_tokens": 285, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 240.262, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0010", "group": "medium_8k", "case": "eventstream", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 236.733, "native_output_tokens": 8379.0, "opencode_output_tokens": 187, "tool_calls": 5, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 236.733, "exit_code": 0, "termination": "completed", "tool_calls": 5, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0011", "group": "medium_8k", "case": "wallet", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 243.273, "native_output_tokens": 8496.0, "opencode_output_tokens": 304, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 243.273, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0012", "group": "medium_8k", "case": "ledger", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 243.777, "native_output_tokens": 8525.0, "opencode_output_tokens": 333, "tool_calls": 10, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 243.777, "exit_code": 0, "termination": "completed", "tool_calls": 10, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0013", "group": "medium_8k", "case": "buildplan", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 241.765, "native_output_tokens": 8488.0, "opencode_output_tokens": 296, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 241.765, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0014", "group": "medium_8k", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 258.688, "native_output_tokens": 6204.0, "opencode_output_tokens": 5750, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 258.688, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0015", "group": "medium_8k", "case": "async_pool", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 244.785, "native_output_tokens": 8474.0, "opencode_output_tokens": 282, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 244.785, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0016", "group": "medium_8k", "case": "cache", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 237.741, "native_output_tokens": 8373.0, "opencode_output_tokens": 181, "tool_calls": 5, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 237.741, "exit_code": 0, "termination": "completed", "tool_calls": 5, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0017", "group": "medium_8k", "case": "ledger", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 245.796, "native_output_tokens": 8589.0, "opencode_output_tokens": 397, "tool_calls": 11, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 245.796, "exit_code": 0, "termination": "completed", "tool_calls": 11, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0018", "group": "medium_8k", "case": "eventstream", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 243.783, "native_output_tokens": 8431.0, "opencode_output_tokens": 239, "tool_calls": 7, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 243.783, "exit_code": 0, "termination": "completed", "tool_calls": 7, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0019", "group": "medium_8k", "case": "async_pool", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 240.758, "native_output_tokens": 8480.0, "opencode_output_tokens": 288, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 240.758, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0020", "group": "medium_8k", "case": "pagination", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 242.265, "native_output_tokens": 8530.0, "opencode_output_tokens": 338, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 242.265, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0021", "group": "medium_8k", "case": "wallet", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 456.218, "native_output_tokens": 15283.0, "opencode_output_tokens": 14583, "tool_calls": 26, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 456.218, "exit_code": 0, "termination": "completed", "tool_calls": 26, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0022", "group": "medium_8k", "case": "boltons_paths", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 246.495, "native_output_tokens": 8074.0, "opencode_output_tokens": 7480, "tool_calls": 26, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 246.495, "exit_code": 0, "termination": "completed", "tool_calls": 26, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0023", "group": "medium_8k", "case": "cache", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 304.953, "native_output_tokens": 10325.0, "opencode_output_tokens": 9459, "tool_calls": 22, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 304.953, "exit_code": 0, "termination": "completed", "tool_calls": 22, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0024", "group": "medium_8k", "case": "buildplan", "repeat": "repeat-3", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 242.773, "native_output_tokens": 8519.0, "opencode_output_tokens": 327, "tool_calls": 10, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 242.773, "exit_code": 0, "termination": "completed", "tool_calls": 10, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0025", "group": "medium_16k", "case": "cache", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 428.532, "native_output_tokens": 14149.0, "opencode_output_tokens": 13618, "tool_calls": 12, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 428.532, "exit_code": 0, "termination": "completed", "tool_calls": 12, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0026", "group": "medium_16k", "case": "wallet", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 771.177, "native_output_tokens": 25351.0, "opencode_output_tokens": 24599, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 771.177, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0027", "group": "medium_16k", "case": "eventstream", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 947.233, "native_output_tokens": 30301.0, "opencode_output_tokens": 28655, "tool_calls": 31, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 947.233, "exit_code": 0, "termination": "completed", "tool_calls": 31, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0028", "group": "medium_16k", "case": "boltons_paths", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 395.431, "native_output_tokens": 11521.0, "opencode_output_tokens": 10831, "tool_calls": 26, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 395.431, "exit_code": 0, "termination": "completed", "tool_calls": 26, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0029", "group": "medium_16k", "case": "async_pool", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 825.791, "native_output_tokens": 27049.0, "opencode_output_tokens": 26343, "tool_calls": 22, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 825.791, "exit_code": 0, "termination": "completed", "tool_calls": 22, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0030", "group": "medium_16k", "case": "ledger", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 893.712, "native_output_tokens": 29586.0, "opencode_output_tokens": 13202, "tool_calls": 13, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 893.712, "exit_code": 0, "termination": "completed", "tool_calls": 13, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 2 } ] }, { "attempt_id": "a0031", "group": "medium_16k", "case": "pagination", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 611.123, "native_output_tokens": 20445.0, "opencode_output_tokens": 19936, "tool_calls": 17, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 611.123, "exit_code": 0, "termination": "completed", "tool_calls": 17, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0032", "group": "medium_16k", "case": "buildplan", "repeat": "repeat-1", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 664.031, "native_output_tokens": 22151.0, "opencode_output_tokens": 21247, "tool_calls": 25, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "turns": [ { "seconds": 664.031, "exit_code": 0, "termination": "completed", "tool_calls": 25, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0033", "group": "medium_16k", "case": "pagination", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 480.02, "native_output_tokens": 16669.0, "opencode_output_tokens": 285, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 480.02, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0034", "group": "medium_16k", "case": "eventstream", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 544.161, "native_output_tokens": 19252.0, "opencode_output_tokens": 18549, "tool_calls": 17, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 544.161, "exit_code": 0, "termination": "completed", "tool_calls": 17, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0035", "group": "medium_16k", "case": "wallet", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 465.528, "native_output_tokens": 16688.0, "opencode_output_tokens": 304, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 465.528, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0036", "group": "medium_16k", "case": "ledger", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 466.031, "native_output_tokens": 16717.0, "opencode_output_tokens": 333, "tool_calls": 10, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 466.031, "exit_code": 0, "termination": "completed", "tool_calls": 10, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0037", "group": "medium_16k", "case": "buildplan", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 669.347, "native_output_tokens": 22247.0, "opencode_output_tokens": 21414, "tool_calls": 20, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "turns": [ { "seconds": 669.347, "exit_code": 0, "termination": "completed", "tool_calls": 20, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0038", "group": "medium_16k", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 296.613, "native_output_tokens": 10028.0, "opencode_output_tokens": 9189, "tool_calls": 28, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 296.613, "exit_code": 0, "termination": "completed", "tool_calls": 28, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0039", "group": "medium_16k", "case": "async_pool", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 703.668, "native_output_tokens": 24497.0, "opencode_output_tokens": 23601, "tool_calls": 28, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 703.668, "exit_code": 0, "termination": "completed", "tool_calls": 28, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0040", "group": "medium_16k", "case": "cache", "repeat": "repeat-2", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 353.545, "native_output_tokens": 12703.0, "opencode_output_tokens": 11948, "tool_calls": 18, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 353.545, "exit_code": 0, "termination": "completed", "tool_calls": 18, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0041", "group": "medium_16k", "case": "ledger", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 468.034, "native_output_tokens": 16787.0, "opencode_output_tokens": 403, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 468.034, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0042", "group": "medium_16k", "case": "eventstream", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 458.465, "native_output_tokens": 16594.0, "opencode_output_tokens": 210, "tool_calls": 5, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 458.465, "exit_code": 0, "termination": "completed", "tool_calls": 5, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0043", "group": "medium_16k", "case": "async_pool", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1027.922, "native_output_tokens": 34569.0, "opencode_output_tokens": 18185, "tool_calls": 14, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 1027.922, "exit_code": 0, "termination": "completed", "tool_calls": 14, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 0 } ] }, { "attempt_id": "a0044", "group": "medium_16k", "case": "pagination", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 570.35, "native_output_tokens": 19982.0, "opencode_output_tokens": 3598, "tool_calls": 17, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 570.35, "exit_code": 0, "termination": "completed", "tool_calls": 17, "observed_tests_passed": false, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 3 } ] }, { "attempt_id": "a0045", "group": "medium_16k", "case": "wallet", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 510.851, "native_output_tokens": 17924.0, "opencode_output_tokens": 16692, "tool_calls": 32, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 510.851, "exit_code": 0, "termination": "completed", "tool_calls": 32, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 9 } ] }, { "attempt_id": "a0046", "group": "medium_16k", "case": "boltons_paths", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 228.528, "native_output_tokens": 6600.0, "opencode_output_tokens": 5926, "tool_calls": 23, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 228.528, "exit_code": 0, "termination": "completed", "tool_calls": 23, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0047", "group": "medium_16k", "case": "cache", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 369.509, "native_output_tokens": 13055.0, "opencode_output_tokens": 12498, "tool_calls": 17, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 369.509, "exit_code": 0, "termination": "completed", "tool_calls": 17, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0048", "group": "medium_16k", "case": "buildplan", "repeat": "repeat-3", "profile": "medium_16k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 637.841, "native_output_tokens": 21769.0, "opencode_output_tokens": 20903, "tool_calls": 23, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 637.841, "exit_code": 0, "termination": "completed", "tool_calls": 23, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 7 } ] }, { "attempt_id": "a0049", "group": "medium_32k", "case": "cache", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 345.483, "native_output_tokens": 12440.0, "opencode_output_tokens": 11429, "tool_calls": 18, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 345.483, "exit_code": 0, "termination": "completed", "tool_calls": 18, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0050", "group": "medium_32k", "case": "wallet", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 422.552, "native_output_tokens": 15233.0, "opencode_output_tokens": 13815, "tool_calls": 18, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 422.552, "exit_code": 0, "termination": "completed", "tool_calls": 18, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0051", "group": "medium_32k", "case": "eventstream", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 488.673, "native_output_tokens": 17302.0, "opencode_output_tokens": 16494, "tool_calls": 19, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 488.673, "exit_code": 0, "termination": "completed", "tool_calls": 19, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0052", "group": "medium_32k", "case": "boltons_paths", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 265.104, "native_output_tokens": 8048.0, "opencode_output_tokens": 6990, "tool_calls": 26, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 265.104, "exit_code": 0, "termination": "completed", "tool_calls": 26, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 6 } ] }, { "attempt_id": "a0053", "group": "medium_32k", "case": "async_pool", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 706.076, "native_output_tokens": 24825.0, "opencode_output_tokens": 23424, "tool_calls": 29, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 706.076, "exit_code": 0, "termination": "completed", "tool_calls": 29, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 8 } ] }, { "attempt_id": "a0054", "group": "medium_32k", "case": "ledger", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 1689.465, "native_output_tokens": 53573.0, "opencode_output_tokens": 52122, "tool_calls": 39, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "turns": [ { "seconds": 1689.465, "exit_code": 0, "termination": "completed", "tool_calls": 39, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 7 } ] }, { "attempt_id": "a0055", "group": "medium_32k", "case": "pagination", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 528.201, "native_output_tokens": 18660.0, "opencode_output_tokens": 17824, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 528.201, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0056", "group": "medium_32k", "case": "buildplan", "repeat": "repeat-1", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 672.636, "native_output_tokens": 23487.0, "opencode_output_tokens": 22686, "tool_calls": 32, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 672.636, "exit_code": 0, "termination": "completed", "tool_calls": 32, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0057", "group": "medium_32k", "case": "pagination", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 1076.064, "native_output_tokens": 36532.0, "opencode_output_tokens": 35767, "tool_calls": 35, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 1076.064, "exit_code": 0, "termination": "completed", "tool_calls": 35, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 11 } ] }, { "attempt_id": "a0058", "group": "medium_32k", "case": "eventstream", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 323.91, "native_output_tokens": 11609.0, "opencode_output_tokens": 10973, "tool_calls": 19, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 323.91, "exit_code": 0, "termination": "completed", "tool_calls": 19, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0059", "group": "medium_32k", "case": "wallet", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 835.129, "native_output_tokens": 28370.0, "opencode_output_tokens": 27628, "tool_calls": 30, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 835.129, "exit_code": 0, "termination": "completed", "tool_calls": 30, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0060", "group": "medium_32k", "case": "ledger", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 1800.189, "native_output_tokens": 44261.0, "opencode_output_tokens": 44261, "tool_calls": 20, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 1800.189, "exit_code": -15, "termination": "wall_time_limit", "tool_calls": 20, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0061", "group": "medium_32k", "case": "buildplan", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 764.014, "native_output_tokens": 26300.0, "opencode_output_tokens": 25447, "tool_calls": 20, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 764.014, "exit_code": 0, "termination": "completed", "tool_calls": 20, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0062", "group": "medium_32k", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 299.277, "native_output_tokens": 7733.0, "opencode_output_tokens": 6933, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 299.277, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0063", "group": "medium_32k", "case": "async_pool", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1056.693, "native_output_tokens": 35631.0, "opencode_output_tokens": 34876, "tool_calls": 19, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 1056.693, "exit_code": 0, "termination": "completed", "tool_calls": 19, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0064", "group": "medium_32k", "case": "cache", "repeat": "repeat-2", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 427.108, "native_output_tokens": 15032.0, "opencode_output_tokens": 14355, "tool_calls": 17, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 427.108, "exit_code": 0, "termination": "completed", "tool_calls": 17, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0065", "group": "medium_32k", "case": "ledger", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 1044.87, "native_output_tokens": 34534.0, "opencode_output_tokens": 33719, "tool_calls": 24, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "turns": [ { "seconds": 1044.87, "exit_code": 0, "termination": "completed", "tool_calls": 24, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0066", "group": "medium_32k", "case": "eventstream", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 588.322, "native_output_tokens": 19563.0, "opencode_output_tokens": 18946, "tool_calls": 24, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 588.322, "exit_code": 0, "termination": "completed", "tool_calls": 24, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0067", "group": "medium_32k", "case": "async_pool", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 944.602, "native_output_tokens": 30248.0, "opencode_output_tokens": 29177, "tool_calls": 24, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 944.602, "exit_code": 0, "termination": "completed", "tool_calls": 24, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0068", "group": "medium_32k", "case": "pagination", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 584.877, "native_output_tokens": 19589.0, "opencode_output_tokens": 18815, "tool_calls": 20, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 584.877, "exit_code": 0, "termination": "completed", "tool_calls": 20, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0069", "group": "medium_32k", "case": "wallet", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 653.478, "native_output_tokens": 22174.0, "opencode_output_tokens": 21422, "tool_calls": 29, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 653.478, "exit_code": 0, "termination": "completed", "tool_calls": 29, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0070", "group": "medium_32k", "case": "boltons_paths", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 220.828, "native_output_tokens": 7087.0, "opencode_output_tokens": 6405, "tool_calls": 16, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 220.828, "exit_code": 0, "termination": "completed", "tool_calls": 16, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0071", "group": "medium_32k", "case": "cache", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 250.29, "native_output_tokens": 8726.0, "opencode_output_tokens": 8191, "tool_calls": 13, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 250.29, "exit_code": 0, "termination": "completed", "tool_calls": 13, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 0 } ] }, { "attempt_id": "a0072", "group": "medium_32k", "case": "buildplan", "repeat": "repeat-3", "profile": "medium_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 567.817, "native_output_tokens": 20103.0, "opencode_output_tokens": 19243, "tool_calls": 27, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 567.817, "exit_code": 0, "termination": "completed", "tool_calls": 27, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0073", "group": "xhigh_32k", "case": "cache", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 463.189, "native_output_tokens": 14662.0, "opencode_output_tokens": 13940, "tool_calls": 13, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 463.189, "exit_code": 0, "termination": "completed", "tool_calls": 13, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0074", "group": "xhigh_32k", "case": "wallet", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1188.266, "native_output_tokens": 36039.0, "opencode_output_tokens": 35009, "tool_calls": 24, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 1188.266, "exit_code": 0, "termination": "completed", "tool_calls": 24, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0075", "group": "xhigh_32k", "case": "eventstream", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 954.686, "native_output_tokens": 28963.0, "opencode_output_tokens": 28229, "tool_calls": 13, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 954.686, "exit_code": 0, "termination": "completed", "tool_calls": 13, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0076", "group": "xhigh_32k", "case": "boltons_paths", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 369.89, "native_output_tokens": 10381.0, "opencode_output_tokens": 9672, "tool_calls": 27, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 369.89, "exit_code": 0, "termination": "completed", "tool_calls": 27, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0077", "group": "xhigh_32k", "case": "async_pool", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1086.027, "native_output_tokens": 33088.0, "opencode_output_tokens": 32034, "tool_calls": 41, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 1086.027, "exit_code": 0, "termination": "completed", "tool_calls": 41, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 9 } ] }, { "attempt_id": "a0078", "group": "xhigh_32k", "case": "ledger", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 1364.913, "native_output_tokens": 42255.0, "opencode_output_tokens": 41447, "tool_calls": 30, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 1364.913, "exit_code": 0, "termination": "completed", "tool_calls": 30, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0079", "group": "xhigh_32k", "case": "pagination", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 749.327, "native_output_tokens": 25565.0, "opencode_output_tokens": 24702, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 749.327, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0080", "group": "xhigh_32k", "case": "buildplan", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 685.047, "native_output_tokens": 23497.0, "opencode_output_tokens": 22338, "tool_calls": 25, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 685.047, "exit_code": 0, "termination": "completed", "tool_calls": 25, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0081", "group": "xhigh_32k", "case": "pagination", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 828.939, "native_output_tokens": 28071.0, "opencode_output_tokens": 27227, "tool_calls": 33, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 828.939, "exit_code": 0, "termination": "completed", "tool_calls": 33, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0082", "group": "xhigh_32k", "case": "eventstream", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 467.651, "native_output_tokens": 16513.0, "opencode_output_tokens": 15436, "tool_calls": 23, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 467.651, "exit_code": 0, "termination": "completed", "tool_calls": 23, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0083", "group": "xhigh_32k", "case": "wallet", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1688.381, "native_output_tokens": 51541.0, "opencode_output_tokens": 50355, "tool_calls": 29, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 1688.381, "exit_code": 0, "termination": "completed", "tool_calls": 29, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0084", "group": "xhigh_32k", "case": "ledger", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1337.023, "native_output_tokens": 42651.0, "opencode_output_tokens": 41493, "tool_calls": 30, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 1337.023, "exit_code": 0, "termination": "completed", "tool_calls": 30, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0085", "group": "xhigh_32k", "case": "buildplan", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 694.09, "native_output_tokens": 23969.0, "opencode_output_tokens": 23113, "tool_calls": 27, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 694.09, "exit_code": 0, "termination": "completed", "tool_calls": 27, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0086", "group": "xhigh_32k", "case": "boltons_paths", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 379.694, "native_output_tokens": 9724.0, "opencode_output_tokens": 8978, "tool_calls": 25, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 379.694, "exit_code": 0, "termination": "completed", "tool_calls": 25, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0087", "group": "xhigh_32k", "case": "async_pool", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 878.349, "native_output_tokens": 28425.0, "opencode_output_tokens": 27617, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 878.349, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0088", "group": "xhigh_32k", "case": "cache", "repeat": "repeat-2", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 421.012, "native_output_tokens": 15213.0, "opencode_output_tokens": 14458, "tool_calls": 15, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 421.012, "exit_code": 0, "termination": "completed", "tool_calls": 15, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0089", "group": "xhigh_32k", "case": "ledger", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1115.964, "native_output_tokens": 36782.0, "opencode_output_tokens": 35927, "tool_calls": 33, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 1115.964, "exit_code": 0, "termination": "completed", "tool_calls": 33, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0090", "group": "xhigh_32k", "case": "eventstream", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 901.336, "native_output_tokens": 29749.0, "opencode_output_tokens": 28705, "tool_calls": 19, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 901.336, "exit_code": 0, "termination": "completed", "tool_calls": 19, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0091", "group": "xhigh_32k", "case": "async_pool", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 553.796, "native_output_tokens": 19455.0, "opencode_output_tokens": 18704, "tool_calls": 23, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 553.796, "exit_code": 0, "termination": "completed", "tool_calls": 23, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0092", "group": "xhigh_32k", "case": "pagination", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 655.936, "native_output_tokens": 22029.0, "opencode_output_tokens": 21236, "tool_calls": 24, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 655.936, "exit_code": 0, "termination": "completed", "tool_calls": 24, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0093", "group": "xhigh_32k", "case": "wallet", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 802.238, "native_output_tokens": 26994.0, "opencode_output_tokens": 26105, "tool_calls": 18, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 802.238, "exit_code": 0, "termination": "completed", "tool_calls": 18, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0094", "group": "xhigh_32k", "case": "boltons_paths", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 371.586, "native_output_tokens": 9511.0, "opencode_output_tokens": 8625, "tool_calls": 26, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 371.586, "exit_code": 0, "termination": "completed", "tool_calls": 26, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0095", "group": "xhigh_32k", "case": "cache", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 476.164, "native_output_tokens": 16467.0, "opencode_output_tokens": 15618, "tool_calls": 19, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 476.164, "exit_code": 0, "termination": "completed", "tool_calls": 19, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0096", "group": "xhigh_32k", "case": "buildplan", "repeat": "repeat-3", "profile": "xhigh_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 870.267, "native_output_tokens": 28716.0, "opencode_output_tokens": 27389, "tool_calls": 24, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 870.267, "exit_code": 0, "termination": "completed", "tool_calls": 24, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0097", "group": "temperature08_targeted", "case": "ledger", "repeat": "repeat-1", "profile": "xhigh_32k_temperature08", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 1547.169, "native_output_tokens": 49295.0, "opencode_output_tokens": 48479, "tool_calls": 39, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 1547.169, "exit_code": 0, "termination": "completed", "tool_calls": 39, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 7 } ] }, { "attempt_id": "a0098", "group": "temperature08_targeted", "case": "pagination", "repeat": "repeat-1", "profile": "xhigh_32k_temperature08", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 647.091, "native_output_tokens": 21661.0, "opencode_output_tokens": 20933, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 647.091, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0099", "group": "temperature08_targeted", "case": "pagination", "repeat": "repeat-2", "profile": "xhigh_32k_temperature08", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 716.573, "native_output_tokens": 24524.0, "opencode_output_tokens": 23673, "tool_calls": 20, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 716.573, "exit_code": 0, "termination": "completed", "tool_calls": 20, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 3 } ] }, { "attempt_id": "a0100", "group": "temperature08_targeted", "case": "ledger", "repeat": "repeat-2", "profile": "xhigh_32k_temperature08", "status": "complete", "event": null, "functional_success": true, "delivered_success": false, "seconds": 1800.017, "native_output_tokens": 54797.0, "opencode_output_tokens": 54797, "tool_calls": 26, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 1800.017, "exit_code": -15, "termination": "wall_time_limit", "tool_calls": 26, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 3 } ] }, { "attempt_id": "a0101", "group": "temperature08_targeted", "case": "ledger", "repeat": "repeat-3", "profile": "xhigh_32k_temperature08", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 1674.277, "native_output_tokens": 52654.0, "opencode_output_tokens": 51688, "tool_calls": 27, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": true, "syntax": true }, "solved": false }, "turns": [ { "seconds": 1674.277, "exit_code": 0, "termination": "completed", "tool_calls": 27, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 4 } ] }, { "attempt_id": "a0102", "group": "temperature08_targeted", "case": "pagination", "repeat": "repeat-3", "profile": "xhigh_32k_temperature08", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 795.783, "native_output_tokens": 26492.0, "opencode_output_tokens": 25650, "tool_calls": 22, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 795.783, "exit_code": 0, "termination": "completed", "tool_calls": 22, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0103", "group": "extra_8k_excluded", "case": "cache", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 248.796, "native_output_tokens": 8550.0, "opencode_output_tokens": 358, "tool_calls": 8, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 248.796, "exit_code": 0, "termination": "completed", "tool_calls": 8, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0104", "group": "extra_8k_excluded", "case": "wallet", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 249.307, "native_output_tokens": 8546.0, "opencode_output_tokens": 354, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 249.307, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0105", "group": "extra_8k_excluded", "case": "eventstream", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 245.79, "native_output_tokens": 8513.0, "opencode_output_tokens": 321, "tool_calls": 8, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 245.79, "exit_code": 0, "termination": "completed", "tool_calls": 8, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0106", "group": "extra_8k_excluded", "case": "boltons_paths", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 272.13, "native_output_tokens": 7693.0, "opencode_output_tokens": 6897, "tool_calls": 27, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 272.13, "exit_code": 0, "termination": "completed", "tool_calls": 27, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0107", "group": "extra_8k_excluded", "case": "async_pool", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 516.765, "native_output_tokens": 17520.0, "opencode_output_tokens": 16831, "tool_calls": 19, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "typecheck": true, "consumer_typecheck": true }, "solved": true }, "turns": [ { "seconds": 516.765, "exit_code": 0, "termination": "completed", "tool_calls": 19, "observed_tests_passed": true, "observed_typecheck_passed": true, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0108", "group": "extra_8k_excluded", "case": "ledger", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 246.298, "native_output_tokens": 8610.0, "opencode_output_tokens": 418, "tool_calls": 11, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 246.298, "exit_code": 0, "termination": "completed", "tool_calls": 11, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0109", "group": "extra_8k_excluded", "case": "pagination", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 244.276, "native_output_tokens": 8579.0, "opencode_output_tokens": 387, "tool_calls": 10, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 244.276, "exit_code": 0, "termination": "completed", "tool_calls": 10, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0110", "group": "extra_8k_excluded", "case": "buildplan", "repeat": "repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 245.293, "native_output_tokens": 8593.0, "opencode_output_tokens": 401, "tool_calls": 11, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 245.293, "exit_code": 0, "termination": "completed", "tool_calls": 11, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0111", "group": "extra_8k_excluded", "case": "pagination", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 239.764, "native_output_tokens": 8477.0, "opencode_output_tokens": 285, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 239.764, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0112", "group": "extra_8k_excluded", "case": "eventstream", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 236.735, "native_output_tokens": 8379.0, "opencode_output_tokens": 187, "tool_calls": 5, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 236.735, "exit_code": 0, "termination": "completed", "tool_calls": 5, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0113", "group": "extra_8k_excluded", "case": "wallet", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 233.762, "native_output_tokens": 8496.0, "opencode_output_tokens": 304, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 233.762, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0114", "group": "extra_8k_excluded", "case": "ledger", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 234.27, "native_output_tokens": 8525.0, "opencode_output_tokens": 333, "tool_calls": 10, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 234.27, "exit_code": 0, "termination": "completed", "tool_calls": 10, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0115", "group": "extra_8k_excluded", "case": "buildplan", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 241.273, "native_output_tokens": 8488.0, "opencode_output_tokens": 296, "tool_calls": 9, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "syntax": true }, "solved": false }, "turns": [ { "seconds": 241.273, "exit_code": 0, "termination": "completed", "tool_calls": 9, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 0 } ] }, { "attempt_id": "a0116", "group": "extra_8k_excluded", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 258.263, "native_output_tokens": 6204.0, "opencode_output_tokens": 5750, "tool_calls": 21, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 258.263, "exit_code": 0, "termination": "completed", "tool_calls": 21, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 5 } ] }, { "attempt_id": "a0117", "group": "pilots", "case": "cache", "repeat": "pilot-repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 263.327, "native_output_tokens": 9393.0, "opencode_output_tokens": 8854, "tool_calls": 15, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 263.327, "exit_code": 0, "termination": "completed", "tool_calls": 15, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 2 } ] }, { "attempt_id": "a0118", "group": "pilots", "case": "async_pool", "repeat": "pilot-repeat-1", "profile": "medium_8k", "status": "complete", "event": null, "functional_success": false, "delivered_success": false, "seconds": 245.787, "native_output_tokens": 8718.0, "opencode_output_tokens": 526, "tool_calls": 10, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": false, "regression": true, "public": false, "typecheck": true, "consumer_typecheck": true }, "solved": false }, "turns": [ { "seconds": 245.787, "exit_code": 0, "termination": "completed", "tool_calls": 10, "observed_tests_passed": false, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": false, "tool_error_count": 1 } ] }, { "attempt_id": "a0119", "group": "thinking_off_cancelled", "case": "cache", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "complete", "event": null, "functional_success": true, "delivered_success": true, "seconds": 134.695, "native_output_tokens": 4433.0, "opencode_output_tokens": 3836, "tool_calls": 16, "config_unchanged": true, "phase_one_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "final_grade": { "protected_changes_count": 0, "missing_starter_files_count": 0, "checks": { "acceptance": true, "regression": true, "public": true, "syntax": true }, "solved": true }, "turns": [ { "seconds": 134.695, "exit_code": 0, "termination": "completed", "tool_calls": 16, "observed_tests_passed": true, "observed_typecheck_passed": false, "finish_reasons": [ "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls", "tool-calls" ], "has_final_text_after_tools": true, "tool_error_count": 1 } ] }, { "attempt_id": "a0120", "group": "thinking_off_cancelled", "case": "wallet", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_after_start", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0121", "group": "thinking_off_cancelled", "case": "eventstream", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0122", "group": "thinking_off_cancelled", "case": "boltons_paths", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0123", "group": "thinking_off_cancelled", "case": "async_pool", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0124", "group": "thinking_off_cancelled", "case": "ledger", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0125", "group": "thinking_off_cancelled", "case": "pagination", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0126", "group": "thinking_off_cancelled", "case": "buildplan", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0127", "group": "thinking_off_cancelled", "case": "pagination", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0128", "group": "thinking_off_cancelled", "case": "eventstream", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0129", "group": "thinking_off_cancelled", "case": "wallet", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0130", "group": "thinking_off_cancelled", "case": "ledger", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0131", "group": "thinking_off_cancelled", "case": "buildplan", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0132", "group": "thinking_off_cancelled", "case": "boltons_paths", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0133", "group": "thinking_off_cancelled", "case": "async_pool", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0134", "group": "thinking_off_cancelled", "case": "cache", "repeat": "repeat-2", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0135", "group": "thinking_off_cancelled", "case": "ledger", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0136", "group": "thinking_off_cancelled", "case": "eventstream", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0137", "group": "thinking_off_cancelled", "case": "async_pool", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0138", "group": "thinking_off_cancelled", "case": "pagination", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0139", "group": "thinking_off_cancelled", "case": "wallet", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0140", "group": "thinking_off_cancelled", "case": "boltons_paths", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0141", "group": "thinking_off_cancelled", "case": "cache", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0142", "group": "thinking_off_cancelled", "case": "buildplan", "repeat": "repeat-3", "profile": "thinking_off_32k", "status": "cancelled_unstarted", "event": "direction_changed", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0143", "group": "infrastructure_excluded", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_16k", "status": "infrastructure_no_result", "event": "gpu_fault", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0144", "group": "infrastructure_excluded", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_16k", "status": "infrastructure_no_result", "event": "sandbox_setup_failure", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0145", "group": "infrastructure_excluded", "case": "boltons_paths", "repeat": "repeat-2", "profile": "medium_16k", "status": "infrastructure_no_result", "event": "historical_temperature_cutoff", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0146", "group": "configuration_only", "case": "cache", "repeat": "repeat-1", "profile": "medium_32k", "status": "not_inference", "event": "configuration_preflight", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0147", "group": "configuration_only", "case": "ledger", "repeat": "repeat-1", "profile": "xhigh_32k_temperature08", "status": "not_inference", "event": "configuration_preflight", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0148", "group": "configuration_only", "case": "cache", "repeat": "repeat-1", "profile": "thinking_off_32k", "status": "not_inference", "event": "configuration_preflight", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0149", "group": "configuration_only", "case": "cache", "repeat": "repeat-1", "profile": "xhigh_32k", "status": "not_inference", "event": "configuration_preflight", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0150", "group": "extra_8k_cancelled", "case": "async_pool", "repeat": "repeat-2", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0151", "group": "extra_8k_cancelled", "case": "cache", "repeat": "repeat-2", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0152", "group": "extra_8k_cancelled", "case": "ledger", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0153", "group": "extra_8k_cancelled", "case": "eventstream", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0154", "group": "extra_8k_cancelled", "case": "async_pool", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0155", "group": "extra_8k_cancelled", "case": "pagination", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0156", "group": "extra_8k_cancelled", "case": "wallet", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0157", "group": "extra_8k_cancelled", "case": "boltons_paths", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0158", "group": "extra_8k_cancelled", "case": "cache", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] }, { "attempt_id": "a0159", "group": "extra_8k_cancelled", "case": "buildplan", "repeat": "repeat-3", "profile": "medium_8k", "status": "cancelled_unstarted", "event": "protocol_amendment", "functional_success": null, "delivered_success": null, "seconds": null, "native_output_tokens": null, "opencode_output_tokens": null, "tool_calls": null, "config_unchanged": null, "phase_one_grade": null, "final_grade": null, "turns": [] } ] }