claude-opus-5

{
  "command": "tail -3 work/trainE.log work/trainF.log work/trainG.log; nvidia-smi --query-gpu=utilization.gpu,memory.used --format=csv; ps aux|grep -c train_nano",
  "description": "Check E/F/G logs for errors"
}

{
  "command": "for v in E F G; do echo \"=== $v ===\"; cat work/train$v.log | grep -v Warning | tail -5; done",
  "description": "Read E/F/G logs individually"
}

{
  "command": "for v in E F G; do setsid nohup python train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/r$v.json > work/train$v.log 2>&1 < /dev/null & done\ndisown -a 2>/dev/null; sleep 5; ls -la /tmp/trE.npy /tmp/trF.npy /tmp/trG.npy; ps aux|grep train_nano|grep -v grep|wc -l",
  "description": "Relaunch E/F/G detached"
}

{
  "command": "cat > /workspace/work/ngram_close.py <<'EOF'\n\"\"\"Mechanism probe, no training involved: unigram+bigram cross-entropy of the TARGET\ntoken stream under the empirical distribution of a candidate training set.\nLower = the training set literally puts more mass where the target does.\"\"\"\nimport json, sys, numpy as np\nW=\"/workspace/work\"; EOS=50256\ntoks=np.load(f\"{W}/pool_toks.npy\",mmap_mode=\"r\"); off=np.load(f\"{W}/pool_off.npy\")\nlens=np.load(f\"{W}/pool_lens.npy\"); ids=np.load(f\"{W}/pool_ids.npy\")\nrow={int(v):i for i,v in enumerate(ids)}\ntgt=np.load(\"/workspace/data/multi_dev.npy\").astype(np.int64)\nV=50257\ndef stream(sel,budget=12_000_000):\n    parts=[];tot=0\n    for i in sel:\n        r=row[i]; parts.append(np.asarray(toks[off[r]:off[r+1]],dtype=np.int64)); tot+=int(lens[r])+1\n        if tot>=budget: break\n    return np.concatenate(parts)[:budget]\ndef xent(tr):\n    c=np.bincount(tr,minlength=V).astype(np.float64)\n    p=(c+1.0)/(c.sum()+V)                       # add-1 smoothed unigram\n    u=-np.log(p[tgt]).mean()\n    # bigram: hash pairs into 2^22 buckets\n    B=1<<22\n    hb=lambda a: ((a[:-1].astype(np.int64)*50257+a[1:])% B)\n    cb=np.bincount(hb(tr),minlength=B).astype(np.float64)\n    pb=(cb+1.0)/(cb.sum()+B)\n    b=-np.log(pb[hb(tgt)]).mean()\n    return u,b\nfor name,path in [(n,f\"{W}/sel{n}.json\") for n in sys.argv[1:]]:\n    tr=stream(json.load(open(path)))\n    u,b=xent(tr)\n    print(f\"{name}: unigram H(target|train) {u:.4f}  bigram {b:.4f}  ({len(tr)} tok)\",flush=True)\nEOF\nfor v in A E F G; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done",
  "description": "Write n-gram closeness probe; check training"
}

{
  "command": "sleep 110; for v in E F G; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done; cat /workspace/.timer/remaining_secs",
  "description": "Wait for E/F/G"
}

{
  "command": "sleep 100; for v in E F G; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || tail -c 200 work/train$v.log; echo; done",
  "description": "Check E/F/G again"
}

{
  "command": "python - <<'EOF'\nimport numpy as np\nz1=np.load('work/scores.npz'); z2=np.load('work/scores2.npz')\nnp.savez('work/scores4.npz', ids=z1['ids'], wiki=z1['wiki'], web=z1['web'], qa=z2['qa'])\nprint('scores4 = v1 wiki/web + markup-stripped qa')\nEOF\n# H: isolate the markup-stripped qa classifier, web/wiki held at F's signal\npython work/select.py --scores work/scores4.npz --mix wiki:0.25,web:0.5,qa:0.25 --assign argmax --out work/selH.json 2>&1|tail -1\n# I: over-weight the hardest register (technical Q&A) -- PPL is an average over registers\npython work/select.py --scores work/scores.npz --mix wiki:0.22,web:0.43,qa:0.35 --assign argmax --out work/selI.json 2>&1|tail -1\n# J: FALSIFICATION ablation -- single register only (encyclopedic), no mixture matching\npython work/select.py --scores work/scores.npz --mix wiki:1.0 --assign argmax --out work/selJ.json 2>&1|tail -1\nfor v in H I J; do python work/fast_pack.py work/sel$v.json /tmp/tr$v.npy >/dev/null; done\nfor v in H I J; do setsid nohup python train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/r$v.json > work/train$v.log 2>&1 < /dev/null & done\nsleep 3; echo \"launched H I J\"",
  "description": "Build and launch H, I, J"
}

{
  "command": "timeout 300 python work/ngram_close.py _random F D J 2>&1 | tail -6",
  "description": "Measure n-gram closeness (mechanism probe, no training)"
}

{
  "command": "sleep 105; for v in H I J; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done; cat /workspace/.timer/remaining_secs",
  "description": "Collect H, I, J results"
}

{
  "command": "sleep 90; for v in H I J; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done",
  "description": "Collect H, I, J results"
}

{
  "command": "python work/select.py --scores work/scores.npz --mix wiki:0.20,web:0.38,qa:0.42 --assign argmax --out work/selK.json 2>&1|tail -1\npython work/select.py --scores work/scores.npz --mix wiki:0.25,web:0.40,qa:0.35 --assign argmax --out work/selL.json 2>&1|tail -1\npython work/select.py --scores work/scores.npz --mix wiki:0.18,web:0.47,qa:0.35 --assign argmax --out work/selM.json 2>&1|tail -1\nfor v in K L M; do python work/fast_pack.py work/sel$v.json /tmp/tr$v.npy >/dev/null; done\nfor v in K L M; do setsid nohup python train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/r$v.json > work/train$v.log 2>&1 < /dev/null & done\nsleep 2; echo launched",
  "description": "Launch mixture-gradient variants K, L, M"
}

{
  "command": "python - <<'EOF'\np='/workspace/submission/curate.py'; s=open(p).read()\ns=s.replace('''   Positives are surface-normalised first (WikiText detokenisation artefacts such as\n   \"@-@\" / \" , \" and HTML markup such as \"<p>\"/\"&lt;\" are removed), because those cues are\n   present in the target rendering but *never* in the raw-web pool: leaving them in lets\n   the classifier separate the classes on a cue that transfers to no pool document, which\n   we observed producing a degenerate ranking.''',\n'''   Positives are surface-normalised for WikiText detokenisation artefacts (\"@-@\", \" , \")\n   since those cues exist in the target rendering but in no raw-web pool document.\n   NOTE (measured, counter to our first guess): additionally stripping HTML markup from the\n   technical-Q&A positives made things *worse* (dev PPL 329.7 vs 322.1).  The markup tokens\n   evidently act as a useful register indicator, so we do not strip them.''')\ns=s.replace('''3. REGISTER-BALANCED PRIORITY ORDER.''','''3. REGISTER-TILTED PRIORITY ORDER.''')\ns=s.replace('''   output list is produced by a weighted round-robin that keeps the running token mixture\n   at the target's measured mixture (wiki 0.25 / web+news 0.50 / technical Q&A 0.25).\n   Because the trainer consumes the list in order until the budget fills, round-robin makes\n   *every prefix* mixture-matched, so the realised training mix is correct at any cutoff.''',\n'''   output list is produced by a weighted round-robin over a fixed token mixture.  Because the\n   trainer consumes the list in order until the budget fills, round-robin makes *every prefix*\n   mixture-matched, so the realised training mix is correct at whatever cutoff the budget lands.\n   The target's own measured token mixture is wiki 0.25 / web+news 0.50 / Q&A 0.25, but the\n   loss-minimising TRAINING mixture is not the eval mixture: tilting toward the technical\n   register (its per-token loss is highest, so it has the steepest remaining descent) measured\n   better, and MIX below is the tuned optimum.  Covering all registers is what matters most --\n   selecting only the single \"cleanest\" register (encyclopedic) scores 445.9 vs 317.0 tilted.''')\ns=s.replace('''def strip_markup(t):\n    t = TAG.sub(\" \", t)\n    for a, b in ((\"&lt;\", \"<\"), (\"&gt;\", \">\"), (\"&amp;\", \"&\"),\n                 (\"&quot;\", '\"'), (\"&#39;\", \"'\"), (\"&nbsp;\", \" \")):\n        t = t.replace(a, b)\n    return re.sub(r\"[ \\\\t]{2,}\", \" \", t)\n\n''','')\ns=s.replace('        pos[r].append(strip_markup(norm_wikitext(s))); ntok[r] += len(s.split())',\n            '        pos[r].append(norm_wikitext(s)); ntok[r] += len(s.split())')\ns=s.replace('TAG = re.compile(r\"<[^>]{1,40}>\")\\n','')\ns=s.replace('MIX    = {\"wiki\": 0.25, \"web\": 0.50, \"qa\": 0.25}',\n            'MIX    = {\"wiki\": 0.22, \"web\": 0.43, \"qa\": 0.35}   # tuned; see module docstring')\ns=s.replace('    ap.add_argument(\"--out\", default=OUT)',\n            '    ap.add_argument(\"--out\", default=OUT)\\n    ap.add_argument(\"--mix\", default=None,\\n                    help=\"override token mixture, e.g. wiki:0.22,web:0.43,qa:0.35\")')\ns=s.replace('''    a = ap.parse_args()\n''','''    a = ap.parse_args()\n    global MIX\n    if a.mix:\n        MIX = {k: float(v) for k, v in (p.split(\":\") for p in a.mix.split(\",\"))}\n''')\nopen(p,'w').write(s)\nimport ast; ast.parse(s); print('curate.py updated + syntax ok')\nEOF\nfor v in K L M; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done",
  "description": "Align curate.py with winning config"
}

{
  "command": "sleep 115; for v in I K L M; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done; cat /workspace/.timer/remaining_secs",
  "description": "Collect K, L, M"
}

{
  "command": "sleep 95; for v in K L M; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done",
  "description": "Collect K, L, M results"
}

{
  "command": "python - <<'EOF'\ns=open('work/select.py').read()\ns=s.replace('ap.add_argument(\"--assign\",default=\"global\",choices=[\"global\",\"argmax\"])',\n 'ap.add_argument(\"--assign\",default=\"global\",choices=[\"global\",\"argmax\"])\\nap.add_argument(\"--strict\",type=int,default=0)')\ns=s.replace(\"    prose_ok=(alpha>0.65)&(sw>0.06)&(mwl>3.0)&(mwl<10)&(dupline<0.30)&(shortline<0.55)&(ell<0.15)&(bad<0.5)\",\n \"    if a.strict:\\n        prose_ok=(alpha>0.72)&(sw>0.10)&(mwl>3.2)&(mwl<9)&(dupline<0.18)&(shortline<0.38)&(ell<0.08)&(bad<0.5)\\n    else:\\n        prose_ok=(alpha>0.65)&(sw>0.06)&(mwl>3.0)&(mwl<10)&(dupline<0.30)&(shortline<0.55)&(ell<0.15)&(bad<0.5)\")\nopen('work/select.py','w').write(s)\nprint('ok')\nEOF\nM=\"wiki:0.22,web:0.43,qa:0.35\"\npython work/select.py --scores work/scores.npz --mix $M --assign argmax --maxlen 3000 --out work/selN.json 2>&1|tail -1\npython work/select.py --scores work/scores.npz --mix $M --assign argmax --strict 1 --out work/selO.json 2>&1|tail -1\npython work/select.py --scores work/scores.npz --mix $M --assign argmax --minlen 256 --out work/selP.json 2>&1|tail -1\nfor v in N O P; do python work/fast_pack.py work/sel$v.json /tmp/tr$v.npy >/dev/null; done\nfor v in N O P; do setsid nohup python train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/r$v.json > work/train$v.log 2>&1 < /dev/null & done\nsleep 2; echo launched N O P",
  "description": "Test maxlen cap, stricter gate, higher minlen"
}

{
  "command": "cp work/selI.json /workspace/submission/selection.json && python -c \"\nimport json;s=json.load(open('/workspace/submission/selection.json'));print('insurance selection installed:',len(s),'ids, unique:',len(set(s))==len(s))\"\ncat > /workspace/work/per_reg.py <<'EOF'\n\"\"\"Per-register difficulty probe: unigram cross-entropy of EACH target register's tokens\nunder a candidate training set's empirical distribution. No model training involved.\"\"\"\nimport json,sys,re,numpy as np\nW=\"/workspace/work\"\ntoks=np.load(f\"{W}/pool_toks.npy\",mmap_mode=\"r\");off=np.load(f\"{W}/pool_off.npy\")\nlens=np.load(f\"{W}/pool_lens.npy\");ids=np.load(f\"{W}/pool_ids.npy\")\nrow={int(v):i for i,v in enumerate(ids)}\nfrom transformers import GPT2TokenizerFast\ntok=GPT2TokenizerFast.from_pretrained(\"gpt2\")\ndv=np.load(\"/workspace/data/multi_dev.npy\").astype(np.int64);EOS=50256\ndef register(t):\n    if re.search(r\"</?p>|<code>|<pre>|&lt;|&gt;|</a>|</?blockquote>\",t):return \"qa\"\n    if \"@-@\" in t or \"@,@\" in t:return \"wiki\"\n    n=len(t.split())\n    if n and (t.count(\" , \")+t.count(\" . \")+t.count(\" 's \"))/n>0.02:return \"wiki\"\n    return \"web\"\nbuck={\"wiki\":[],\"web\":[],\"qa\":[]};prev=0\nfor i in np.where(dv==EOS)[0]:\n    if i>prev:\n        seg=dv[prev:i];buck[register(tok.decode(seg))].append(seg)\n    prev=i+1\nbuck={k:np.concatenate(v) for k,v in buck.items()}\nV=50257\nsel=json.load(open(sys.argv[1]));parts=[];tot=0\nfor i in sel:\n    r=row[i];parts.append(np.asarray(toks[off[r]:off[r+1]],dtype=np.int64));tot+=int(lens[r])+1\n    if tot>=12_000_000:break\ntr=np.concatenate(parts)[:12_000_000]\nc=np.bincount(tr,minlength=V).astype(np.float64);p=(c+1.0)/(c.sum()+V)\nprint(f\"{sys.argv[1]}: per-register unigram H(target_register | train)\")\nfor k in (\"wiki\",\"web\",\"qa\"):\n    print(f\"   {k:5s} {-np.log(p[buck[k]]).mean():.4f}   ({len(buck[k])} target tokens)\")\nEOF\ntimeout 200 python work/per_reg.py work/selI.json 2>&1|grep -v Warning|tail -5",
  "description": "Install insurance selection; probe per-register difficulty"
}

{
  "command": "for v in N O P; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done; cat /workspace/.timer/remaining_secs",
  "description": "Check N, O, P"
}

{
  "file_path": "/workspace/claim.md",
  "content": "# Curating a raw web pool for a fixed 12M-token budget\n\n**Setup.** 182,016 raw web documents (199.9M GPT-2 tokens). The trainer consumes an\nordered id list until 12M tokens are packed, then trains a frozen 30M GPT and reports\nheld-out perplexity on a broad, equal-parts high-quality English target\n(encyclopedic / general web prose / news / technical Q&A). Only the selection varies.\n\n---\n\n## Hypothesis\n\nAt a budget that is ~6% of the pool, held-out perplexity on a *broad multi-domain* target\nis set mainly by **distributional match to the target**, not by any single scalar notion of\n\"document quality\". Concretely, three things in decreasing order of effect:\n\n1. **Register coverage.** The training set must contain all registers the target contains.\n   Selecting only the register that looks most \"clean and encyclopedic\" should fail badly,\n   even though every selected document is individually high quality.\n2. **Junk removal.** Boilerplate / navigation dumps / duplicate-line spam / non-English\n   pages consume budget while contributing almost no target-relevant probability mass.\n3. **Target-similarity ranking.** The disclosed dev target (`multi_dev.npy`, 1M tokens) is a\n   sample of the target distribution itself; a bag-of-hashed-n-grams classifier separating\n   it from a random pool sample recovers enough of the importance ratio to rank documents\n   usefully, with no labels and no internet.\n\nI further predicted that the loss-minimising *training* mixture is **not** the target's own\nmixture, because registers differ in marginal return per token spent.\n\n## Mechanism — and an observable that is not the final perplexity\n\nThe mechanism is mass placement: selection wins by moving the training set's empirical\nn-gram distribution onto the target's. That is measurable **with no training at all**.\n\n*Pre-registered observable:* the unigram/bigram cross-entropy of the target token stream\nunder a candidate training set's own empirical n-gram distribution,\n`H(target | train)` (add-1 smoothed; `work/ngram_close.py`). Prediction: candidates that\nlower held-out PPL also lower `H(target | train)`, and — the sharp part — a\nsingle-register selection will **not** lower it, landing near random.\n\n| selection | unigram `H` | bigram `H` | trained PPL |\n|---|---|---|---|\n| random (do-nothing baseline) | 8.1343 | 13.6374 | 485.46 |\n| curated, no quality gate (D) | 7.9662 | 13.3881 | 357.61 |\n| curated, full recipe (F) | **7.9273** | **13.3064** | 322.09 |\n| curated but **encyclopedic register only** (J) | 8.1299 | 13.6209 | 445.88 |\n\nConfirmed, including the sharp prediction: the wiki-only selection is a *cleaner* corpus by\nevery intrinsic quality heuristic, yet its n-gram mass sits essentially as far from the broad\ntarget as random text does (8.1299 vs 8.1343), and its trained perplexity is correspondingly\nclose to random (445.9 vs 485.5) and far worse than the mixture-matched recipe (317–322).\nQuality-without-coverage does not work; the probe sees this before a single training step.\n\n*Second observable:* per-register `H(target_register | train)` should be **unequal**, which is\nwhy the optimal training mixture is not the eval mixture. Measured under the final selection:\n**wiki 8.31, qa 7.94, web 7.71**. The encyclopedic register is the hardest, and partly\n*irreducibly* so — the target renders Wikipedia with WikiText detokenisation artefacts\n(`@-@`, `@,@`, spaced ` , `) that occur in **no** pool document, and selection cannot create\nsurface form it does not have. Budget shifted off wiki toward the registers whose loss is\nactually reducible measured better (317.0 tilted vs 322.1 at the target's own 25/50/25).\n\n## Falsification\n\nThe hypothesis would be falsified by any of:\n\n- **Coverage is not what matters** → the single-register selection (J) would have matched the\n  mixture-matched selection. *Tested: 445.9 vs 317.0. Not falsified.*\n- **Junk removal is cosmetic** → dropping the quality gate would not hurt. *Tested: 357.6 vs\n  333.8 with the gate held otherwise identical — the gate is worth ~24 PPL. Not falsified.*\n- **The probe is not the mechanism** → `H(target | train)` would move independently of PPL.\n  *Tested: the four selections above rank identically on both. Not falsified.*\n- **Still open / weakly supported:** the mixture *tilt* is the least robust claim. Neighbouring\n  mixtures scored 317.0 / 323.1 / 324.3 / 325.9, a non-smooth ~5-point spread, so run-level\n  variance between selections is a few PPL and the tilt's true effect is small. It should be\n  treated as a marginal gain, not an established law. A clean falsification test would be\n  three seeds per mixture; that did not fit the compute budget.\n- One guess **was** falsified: I expected stripping HTML markup from the technical-Q&A\n  positives to help (the markup cannot transfer to plain-text pool documents). It measurably\n  hurt — 329.7 vs 322.1 — so the markup functions as a useful register indicator for the\n  classifier. The recipe keeps it, against my prior.\n\n## Transfer\n\nThe recipe needs no labels, no internet, and no target-specific engineering, so it transfers\nto any (unlabeled pool, small disclosed target sample) pair:\n\n1. Decode/collect a small sample of the target distribution; split it into registers.\n2. Fit one hashed-n-gram logistic regression per register, target-vs-random-pool.\n3. Veto junk with cheap surface heuristics (looser thresholds for code-bearing registers,\n   whose low alpha/stopword ratios are legitimate).\n4. Emit a weighted round-robin over registers so that **every prefix** of the list is\n   mixture-matched — which makes the output robust to *any* budget, since wherever the\n   trainer stops, the realised mixture is still right.\n\nCost is minutes of CPU plus a few seconds of GPU for the whole 200M-token pool; nothing\nscales worse than linearly, so the same script applies to much larger pools and budgets.\nThe parts I would expect to transfer least: the specific tilt weights, and the\ngate thresholds, both of which are tuned against this pool's junk profile. The parts I\nwould expect to transfer most: register coverage dominating single-axis quality, and\n`H(target | train)` as a zero-training proxy for ranking candidate selections — that probe\nis what makes the approach cheap to re-tune on a new corpus.\n\n---\n\n### Ablation ladder (dev PPL, frozen trainer, identical 12M budget)\n\n| # | selection | dev PPL |\n|---|---|---|\n| – | random baseline | 485.46 |\n| J | encyclopedic register only (falsification) | 445.88 |\n| D | 4-register, no quality gate | 357.61 |\n| B | 4-register, global (non-specialised) ranking | 333.83 |\n| E | merged web signal, markup stripped | 329.92 |\n| H | markup stripped from Q&A positives only | 329.73 |\n| C | 4-register (news/prose split), argmax assign | 327.85 |\n| A | 3-register 25/50/25, argmax, gate | 323.22 |\n| F | A + non-English filter | 322.09 |\n| **I** | **F + mixture tilt 0.22 / 0.43 / 0.35** | **317.04** |\n\nReproduce: `python3 submission/curate.py` → `submission/selection.json`;\nevaluate with `bash dev_eval.sh`.\n"
}

{
  "command": "for v in N O P; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done; cat /workspace/.timer/remaining_secs",
  "description": "Check N, O, P results"
}

{
  "command": "sleep 105; for v in N O P; do echo -n \"$v: \"; grep -h HELDOUT work/train$v.log || echo running; done",
  "description": "Collect N, O, P"
}

{
  "command": "setsid nohup python submission/curate.py --out /workspace/work/sel_final.json > work/curate.log 2>&1 < /dev/null & sleep 100; grep -v Warning work/curate.log | tail -8",
  "description": "Run curate.py end-to-end"
}

{
  "command": "python - <<'EOF'\np='/workspace/submission/curate.py'; s=open(p).read()\ns=s.replace('''def _qual_chunk(chunk):\n    return [qual_feats(t) for t in chunk]''','''def _qual_chunk(chunk):\n    return [qual_feats(t) for t in chunk]\n\ndef _tokline_chunk(chunk):\n    \"\"\"GPT-2 token length per doc = how much of the budget that doc costs.\"\"\"\n    from transformers import GPT2TokenizerFast\n    t = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n    o = []\n    for s in range(0, len(chunk), 256):\n        o.extend(len(x) for x in t(chunk[s:s + 256], add_special_tokens=False)[\"input_ids\"])\n    return o''')\ns=s.replace('''    # GPT-2 token length per doc (defines how much of the budget a doc costs)\n    from transformers import GPT2TokenizerFast\n    def _tl(chunk):\n        t = GPT2TokenizerFast.from_pretrained(\"gpt2\")\n        o = []\n        for s in range(0, len(chunk), 256):\n            o.extend(len(x) for x in t(chunk[s:s + 256], add_special_tokens=False)[\"input_ids\"])\n        return o\n    bnd''','''    bnd''')\ns=s.replace('        tl = p.map(_tl, [texts[bnd[i]:bnd[i + 1]] for i in range(NWORK)])',\n            '        tl = p.map(_tokline_chunk, [texts[bnd[i]:bnd[i + 1]] for i in range(NWORK)])')\nopen(p,'w').write(s)\nimport ast; ast.parse(s); print('fixed')\nEOF\nsetsid nohup python submission/curate.py --out /workspace/work/sel_final.json > work/curate.log 2>&1 < /dev/null & sleep 110; grep -v Warning work/curate.log | tail -6",
  "description": "Fix pickling and rerun curate.py"
}

{
  "command": "sleep 115; grep -v Warning work/curate.log | tail -12; ls -la work/sel_final.json 2>/dev/null; cat /workspace/.timer/remaining_secs",
  "description": "Poll curate.py progress"
}

{
  "command": "python - <<'EOF'\nimport json\na=json.load(open('work/sel_final.json')); b=json.load(open('work/selI.json'))\nprint('curate.py output == candidate I :', a==b)\nprint('len',len(a),'unique',len(set(a))==len(a),'min',min(a),'max',max(a))\nEOF\ncp work/sel_final.json /workspace/submission/selection.json\npython work/fast_pack.py /workspace/submission/selection.json /tmp/tr_final.npy\nsetsid nohup python train_nano.py --train_npy /tmp/tr_final.npy --heldout_npy data/multi_dev.npy --out_json /tmp/r_final.json > work/train_final.log 2>&1 < /dev/null &\nsleep 5; echo \"final confirmation run launched\"",
  "description": "Verify, install, and launch final confirmation training"
}

{
  "command": "python - <<'EOF'\nimport json, numpy as np, re\n# verify: is EVERY prefix of the list mixture-matched? (the property that makes it budget-robust)\nz=np.load('work/scores.npz'); ids=z['ids']; lens=np.load('work/pool_lens.npy')\nrow={int(v):i for i,v in enumerate(ids)}\nZ=np.stack([(z[k]-z[k].mean())/z[k].std() for k in ('wiki','web','qa')]); asg=Z.argmax(0)\nsel=json.load(open('/workspace/submission/selection.json'))\ngot=np.zeros(3); tot=0; marks=[]\nfor n,i in enumerate(sel,1):\n    r=row[i]; got[asg[r]]+=lens[r]+1; tot+=lens[r]+1\n    if n in (500,2000,8000,15000,18543) or (tot>=12_000_000 and not marks[-1:] or False):\n        marks.append((n,tot,np.round(got/tot,3)))\nfor n,t,m in marks: print(f\"  prefix {n:6d} docs / {t/1e6:6.3f}M tok -> mix {m}\")\n# where does the 12M budget actually cut?\nc=np.cumsum([lens[row[i]]+1 for i in sel]); k=int(np.searchsorted(c,12_000_000))+1\nprint(f\"12M budget consumes the first {k} of {len(sel)} ids ({c[-1]/1e6:.2f}M tok provided)\")\nEOF\ngrep -h HELDOUT work/train_final.log || echo \"final: running\"",
  "description": "Verify prefix-mixture property and final PPL"
}