on-policy-self-distillation

gpt-5.6-sol

on-policy-self-distillation — graded by the task's verifier against a measured baseline.

backend: modal
variant: codex azure_ai/gpt-5.6-sol
metric: AIME24 avg@12 accuracy
actions: 428
duration: 9h 21m
recorded: 2026-08-02

run idopsd-aime24-4gpus__eNDoWrD

Everything gpt-5.6-sol did on this task, in order.

download JSON ↗

codexgpt-5.6-sol444 turns16 narration428 reasoning428 action428 observation1 final

#101

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "const r = await tools.exec_command({cmd:\"cat /workspace/.timer/remaining_secs && cat /workspace/.timer/elapsed_secs\",\"workdir\":\"/workspace\",\"yield_time_ms\":10000,\"max_output_tokens\":1000});\ntext(r.output);"
}

exec result

Script completed
Wall time 0.1 seconds
Output:
18746
2854

#102

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "let out=\"\";for(let i=0;i<7;i++){const r=await tools.write_stdin({session_id:60604,chars:\"\",yield_time_ms:60000,max_output_tokens:150});out+=r.output.slice(-250)+\"\\n\";if(r.exit_code!==undefined){out+=`EXIT${r.exit_code}`;break;}}text(out);\n"
}

exec result

Script running with cell ID 49
Wall time 11.0 seconds
Output:

#103

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "49",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 49
Wall time 61.0 seconds
Output:

#104

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "49",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 49
Wall time 61.0 seconds
Output:

#105

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "49",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 49
Wall time 61.0 seconds
Output:

#106

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "49",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 49
Wall time 61.0 seconds
Output:

#107

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "49",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 49
Wall time 61.0 seconds
Output:

#108

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "49",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script completed
Wall time 31.1 seconds
Output:
Warning: truncated output (original token count: 413)
Total output lines: 15

| 0/16 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s]
Adding requests: 100%|██████████| 4/4 [00:00<00:00, 247.37it/s]

Processed prompts:   0%|          | 0/16 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s]

Processed prompts:  25%|██▌       | 4/16 [01:36<04:50, 24.21s/it, est. speed input: 6.44 toks/s, output: 270.28 toks/s]
Pr…213 tokens truncated…mpts:  75%|███████▌  | 12/16 [05:34<01:55, 28.87s/it, est. speed input: 4.73 toks/s, output: 506.67 toks/s]
prompts: 100%|██████████| 16/16 [06:01<00:00, 22.59s/it, est. speed input: 6.31 toks/s, output: 800.75 toks/s]

FINAL avg@4 accuracy: 60.00% (36/60 solutions correct on 15 problems)
Results written to /workspace/eval_results/dev_eval_1785698107.json

EXIT0

#109

narration

The curriculum candidate improves the identical screen from 55% to 60% (36/60). That five-point gap is larger than I’d normally expect from the partial-eval noise, so I’m promoting it to a full 30-problem avg@12 evaluation now. While that runs, I’ll keep the final method conservative unless the full result supports adding another component.

#110

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "const r = await tools.exec_command({cmd:\"bash /workspace/dev_eval.sh /workspace/runs/curriculum/capped_seed42/checkpoint-100\",\"workdir\":\"/workspace\",\"yield_time_ms\":1000,\"max_output_tokens\":500});\ntext(r.output); if(r.session_id) text(`SESSION_ID=${r.session_id}`);"
}

exec result

Script completed
Wall time 1.2 seconds
Output:
SESSION_ID=60475

#111

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "const r = await tools.exec_command({cmd:\"python - <<'PY'\\nimport json,glob,os\\nfs=sorted(glob.glob('/workspace/eval_results/dev_eval_*.json'),key=os.path.getmtime)\\nfor f in fs[-2:]:\\n d=json.load(open(f)); print('\\\\n',f,d.get('average_at_n_pct'))\\n print([(x['problem_id'],x['num_correct']) for x in d['results']])\\nPY\",\"workdir\":\"/workspace\",\"yield_time_ms\":10000,\"max_output_tokens\":3000});\ntext(r.output);"
}

exec result

Script completed
Wall time 0.1 seconds
Output:

 /workspace/eval_results/dev_eval_1785696691.json 55.0
[(60, 3), (61, 2), (62, 0), (63, 0), (64, 0), (65, 2), (66, 3), (67, 4), (68, 2), (69, 4), (70, 2), (71, 4), (72, 4), (73, 0), (74, 3)]

 /workspace/eval_results/dev_eval_1785698107.json 60.0
[(60, 4), (61, 2), (62, 0), (63, 0), (64, 2), (65, 0), (66, 3), (67, 4), (68, 4), (69, 4), (70, 3), (71, 4), (72, 4), (73, 0), (74, 2)]

#112

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "const r = await tools.exec_command({cmd:\"python - <<'PY'\\nimport json\\nD=json.load(open('/workspace/baseline/logs/official_eval_result.json'))\\nfor sl in [D['results'][:15],D['results'][15:]]:\\n print(sum(x['num_correct'] for x in sl),sum(x['n'] for x in sl),[(x['problem_id'],x['num_correct']) for x in sl])\\nPY\",\"workdir\":\"/workspace\",\"yield_time_ms\":10000,\"max_output_tokens\":3000});\ntext(r.output);"
}

exec result

Script completed
Wall time 0.2 seconds
Output:
119 180 [(60, 12), (61, 10), (62, 0), (63, 0), (64, 5), (65, 6), (66, 11), (67, 12), (68, 12), (69, 12), (70, 5), (71, 12), (72, 12), (73, 0), (74, 10)]
77 180 [(75, 2), (76, 10), (77, 1), (78, 5), (79, 12), (80, 2), (81, 0), (82, 7), (83, 11), (84, 11), (85, 0), (86, 12), (87, 4), (88, 0), (89, 0)]

#113

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "const r = await tools.exec_command({cmd:\"python - <<'PY'\\nimport json\\nD=json.load(open('/workspace/baseline/logs/official_eval_result.json'))\\nfor sl in [D['results'][:15],D['results'][15:]]:\\n cs=[]\\n for x in sl:\\n  c=sum(g['correct'] for g in x['generations'][:4]);cs.append((x['problem_id'],c))\\n print(sum(x[1] for x in cs),cs)\\nPY\",\"workdir\":\"/workspace\",\"yield_time_ms\":10000,\"max_output_tokens\":3000});\ntext(r.output);"
}

exec result

Script completed
Wall time 0.1 seconds
Output:
39 [(60, 4), (61, 3), (62, 0), (63, 0), (64, 0), (65, 3), (66, 4), (67, 4), (68, 4), (69, 4), (70, 1), (71, 4), (72, 4), (73, 0), (74, 4)]
29 [(75, 1), (76, 4), (77, 0), (78, 3), (79, 4), (80, 1), (81, 0), (82, 2), (83, 4), (84, 4), (85, 0), (86, 4), (87, 2), (88, 0), (89, 0)]

#114

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "let out=\"\";for(let i=0;i<8;i++){const r=await tools.write_stdin({session_id:60475,chars:\"\",yield_time_ms:60000,max_output_tokens:80});out+=r.output.slice(-160)+\"\\n\";if(r.exit_code!==undefined){out+=`EXIT${r.exit_code}`;break;}}text(out);\n"
}

exec result

Script running with cell ID 54
Wall time 11.0 seconds
Output:

#115

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#116

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#117

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#118

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#119

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#120

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#121

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 200
}

wait result

Script running with cell ID 54
Wall time 61.0 seconds
Output:

#122

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "54",
  "yield_time_ms": 60000,
  "max_tokens": 500
}

wait result

Script completed
Wall time 0.0 seconds
Output:
s truncated…██████| 8/8 [00:00<00:00, 322.79it/s]

Processed prompts:   0%|          | 0/96 [00:00<?, ?it/s, est. speed input: 0.00 toks/s, output: 0.00 toks/s]

: 10.65 toks/s, output: 508.96 toks/s]
Processed prompts:  12%|█▎        | 12/96 [02:50<19:55, 14.23s/it, est. speed input: 10.96 toks/s, output: 393.18 toks/s]


Processed prompts:  25%|██▌       | 24/96 [04:07<11:33,  9.64s/it, est. speed input: 15.64 toks/s, output: 623.72 toks/s]
t: 3.35 toks/s, output: 371.53 toks/s]
Processed prompts:  38%|███▊      | 36/96 [05:55<09:20,  9.34s/it, est. speed input: 13.86 toks/s, output: 845.11 toks/s]
truncated…oks/s, output: 556.29 toks/s]
Processed prompts:  43%|████▎     | 36/84 [06:57<07:03,  8.82s/it, est. speed input: 6.70 toks/s, output: 844.64 toks/s]
cated… toks/s, output: 1640.41 toks/s]
Processed prompts:  25%|██▌       | 24/96 [07:43<24:18, 20.26s/it, est. speed input: 6.86 toks/s, output: 561.73 toks/s]

#123

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolexec

args

{
  "input": "let out=\"\";for(let i=0;i<10;i++){const r=await tools.write_stdin({session_id:60475,chars:\"\",yield_time_ms:60000,max_output_tokens:60});out+=r.output.slice(-120)+\"\\n\";if(r.exit_code!==undefined){out+=`EXIT${r.exit_code}`;break;}}text(out);\n"
}

exec result

Script running with cell ID 55
Wall time 11.0 seconds
Output:

#124

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "55",
  "yield_time_ms": 60000,
  "max_tokens": 100
}

wait result

Script running with cell ID 55
Wall time 61.0 seconds
Output:

#125

reasoning (redacted)

encrypted chain-of-thought — recorded, not exposed

toolwait

args

{
  "cell_id": "55",
  "yield_time_ms": 60000,
  "max_tokens": 100
}

wait result

Script running with cell ID 55
Wall time 61.0 seconds
Output: