claude-opus-5

mix30  pending
mix20  pending
loose  pending
nodedup  pending
T=3987

[stdout]
mix30  pending
mix20  pending
loose  pending
nodedup  pending
T=3987
[metadata] {"noOutputExpected": false}

mix30  pending
mix20  pending
loose  pending
nodedup  pending
T=3806

[stdout]
mix30  pending
mix20  pending
loose  pending
nodedup  pending
T=3806
[metadata] {"noOutputExpected": false}

7
100 %, 38138 MiB
mix30  HELDOUT_PPL 325.809  loss 5.7863  train_s 215  /tmp/tr_mix30.npy
mix20  HELDOUT_PPL 338.298  loss 5.8239  train_s 215  /tmp/tr_mix20.npy
loose  HELDOUT_PPL 324.500  loss 5.7823  train_s 215  /tmp/tr_loose.npy
nodedup  HELDOUT_PPL 345.683  loss 5.8455  train_s 215  /tmp/tr_nodedup.npy
T=3736

[stdout]
7
100 %, 38138 MiB
mix30  HELDOUT_PPL 325.809  loss 5.7863  train_s 215  /tmp/tr_mix30.npy
mix20  HELDOUT_PPL 338.298  loss 5.8239  train_s 215  /tmp/tr_mix20.npy
loose  HELDOUT_PPL 324.500  loss 5.7823  train_s 215  /tmp/tr_loose.npy
nodedup  HELDOUT_PPL 345.683  loss 5.8455  train_s 215  /tmp/tr_nodedup.npy
T=3736
[metadata] {"noOutputExpected": false}

ok
[filter] kept 173452 / 182016
[dedup] dropped 22632, kept 153740
[1]   Done                    ( python curate.py --out /tmp/sel_A.json --loose --quota 0.30,0.45,0.25 > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_B.json --looser --quota 0.30,0.45,0.25 2>&1 | grep -E "filter|dedup" )
[3]+  Done                    ( python curate.py --out /tmp/sel_C.json --loose --quota 0.34,0.40,0.26 > /dev/null 2>&1 )
packed 12000000 tokens from 29648 ids -> /tmp/tr_A.npy
packed 12000000 tokens from 27545 ids -> /tmp/tr_B.npy
packed 12000000 tokens from 29687 ids -> /tmp/tr_C.npy
[1]+  Done                    setsid nohup /tmp/runall.sh A B C > /tmp/runall.log 2>&1 < /dev/null
launched
T=3626

[stdout]
ok
[filter] kept 173452 / 182016
[dedup] dropped 22632, kept 153740
[1]   Done                    ( python curate.py --out /tmp/sel_A.json --loose --quota 0.30,0.45,0.25 > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_B.json --looser --quota 0.30,0.45,0.25 2>&1 | grep -E "filter|dedup" )
[3]+  Done                    ( python curate.py --out /tmp/sel_C.json --loose --quota 0.34,0.40,0.26 > /dev/null 2>&1 )
packed 12000000 tokens from 29648 ids -> /tmp/tr_A.npy
packed 12000000 tokens from 27545 ids -> /tmp/tr_B.npy
packed 12000000 tokens from 29687 ids -> /tmp/tr_C.npy
[1]+  Done                    setsid nohup /tmp/runall.sh A B C > /tmp/runall.log 2>&1 < /dev/null
launched
T=3626
[metadata] {"noOutputExpected": false}

[bg] 12000865 random pool tokens
[lm] target model trained 62s
--- evals ---
A  pending
B  pending
C  pending
T=3526

[stdout]
[bg] 12000865 random pool tokens
[lm] target model trained 62s
--- evals ---
A  pending
B  pending
C  pending
T=3526
[metadata] {"noOutputExpected": false}

[bg] 12000865 random pool tokens
[lm] target model trained 62s
--- evals ---
A  HELDOUT_PPL 317.756  loss 5.7613  train_s 219  /tmp/tr_A.npy
B  HELDOUT_PPL 323.922  loss 5.7805  train_s 219  /tmp/tr_B.npy
C  HELDOUT_PPL 318.199  loss 5.7627  train_s 218  /tmp/tr_C.npy
T=3405

[stdout]
[bg] 12000865 random pool tokens
[lm] target model trained 62s
--- evals ---
A  HELDOUT_PPL 317.756  loss 5.7613  train_s 219  /tmp/tr_A.npy
B  HELDOUT_PPL 323.922  loss 5.7805  train_s 219  /tmp/tr_B.npy
C  HELDOUT_PPL 318.199  loss 5.7627  train_s 218  /tmp/tr_C.npy
T=3405
[metadata] {"noOutputExpected": false}

ids 29648 unique 29648 min 1 max 181981
[bg] 12000865 random pool tokens
[lm] target model trained 62s
[lm] target scoring done 233s  mean 7.580
T=3395

[stdout]
ids 29648 unique 29648 min 1 max 181981
[bg] 12000865 random pool tokens
[lm] target model trained 62s
[lm] target scoring done 233s  mean 7.580
T=3395
[metadata] {"noOutputExpected": false}

[1]-  Done                    ( python curate.py --out /tmp/sel_D.json --loose --quota 0.32,0.42,0.26 > /dev/null 2>&1 )
[2]+  Done                    ( python curate.py --out /tmp/sel_E.json --loose --quota 0.30,0.45,0.25 --overfill 4.0 > /dev/null 2>&1 )
packed 12000000 tokens from 29662 ids -> /tmp/tr_D.npy
packed 12000000 tokens from 53297 ids -> /tmp/tr_E.npy
E len 53297 overlap with A in first 29648: 29648
T=3335

[stdout]
[1]-  Done                    ( python curate.py --out /tmp/sel_D.json --loose --quota 0.32,0.42,0.26 > /dev/null 2>&1 )
[2]+  Done                    ( python curate.py --out /tmp/sel_E.json --loose --quota 0.30,0.45,0.25 --overfill 4.0 > /dev/null 2>&1 )
packed 12000000 tokens from 29662 ids -> /tmp/tr_D.npy
packed 12000000 tokens from 53297 ids -> /tmp/tr_E.npy
E len 53297 overlap with A in first 29648: 29648
T=3335
[metadata] {"noOutputExpected": false}

[bg] 12000865 random pool tokens
[lm] target model trained 62s
[lm] target scoring done 233s  mean 7.580
[lm] pool model trained 249s
ce pending
T=3325

[stdout]
[bg] 12000865 random pool tokens
[lm] target model trained 62s
[lm] target scoring done 233s  mean 7.580
[lm] pool model trained 249s
ce pending
T=3325
[metadata] {"noOutputExpected": false}

[bg] 12000865 random pool tokens
[lm] target model trained 62s
[lm] target scoring done 233s  mean 7.580
[lm] pool model trained 249s
[lm] pool scoring done 315s  mean 6.082
[ce] mean 1.498 std 1.041 -> /workspace/cache/ce_diff.npy
[ce] pct [0.107 0.354 0.712 1.171 2.135 3.449 4.367]
T=3134

[stdout]
[bg] 12000865 random pool tokens
[lm] target model trained 62s
[lm] target scoring done 233s  mean 7.580
[lm] pool model trained 249s
[lm] pool scoring done 315s  mean 6.082
[ce] mean 1.498 std 1.041 -> /workspace/cache/ce_diff.npy
[ce] pct [0.107 0.354 0.712 1.171 2.135 3.449 4.367]
T=3134
[metadata] {"noOutputExpected": false}

[1]   Done                    ( python curate.py --out /tmp/sel_ce$w.json --loose --quota 0.30,0.45,0.25 --ce cache/ce_diff.npy --ce_w $w > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_ce$w.json --loose --quota 0.30,0.45,0.25 --ce cache/ce_diff.npy --ce_w $w > /dev/null 2>&1 )
[3]+  Done                    ( python curate.py --out /tmp/sel_ce$w.json --loose --quota 0.30,0.45,0.25 --ce cache/ce_diff.npy --ce_w $w > /dev/null 2>&1 )
packed 12000000 tokens from 29662 ids -> /tmp/tr_D.npy
packed 12000000 tokens from 30656 ids -> /tmp/tr_ce0.3.npy
packed 12000000 tokens from 31355 ids -> /tmp/tr_ce0.6.npy
packed 12000000 tokens from 31658 ids -> /tmp/tr_ce1.2.npy
launched
T=3054

[stdout]
[1]   Done                    ( python curate.py --out /tmp/sel_ce$w.json --loose --quota 0.30,0.45,0.25 --ce cache/ce_diff.npy --ce_w $w > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_ce$w.json --loose --quota 0.30,0.45,0.25 --ce cache/ce_diff.npy --ce_w $w > /dev/null 2>&1 )
[3]+  Done                    ( python curate.py --out /tmp/sel_ce$w.json --loose --quota 0.30,0.45,0.25 --ce cache/ce_diff.npy --ce_w $w > /dev/null 2>&1 )
packed 12000000 tokens from 29662 ids -> /tmp/tr_D.npy
packed 12000000 tokens from 30656 ids -> /tmp/tr_ce0.3.npy
packed 12000000 tokens from 31355 ids -> /tmp/tr_ce0.6.npy
packed 12000000 tokens from 31658 ids -> /tmp/tr_ce1.2.npy
launched
T=3054
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/claim.md (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/claim.md", "content": "# Curating a raw web pool for a broad high-quality target\n\n## Hypothesis\n\nFor a fixed 12M-token budget drawn from a 200M-token raw web pool, held-out\nperplexity on a broad high-quality target is governed **first by register-mixture\nmatch and only second by document-level \"quality\"**.\n\nConcretely: the target is an equal-parts mixture of encyclopedic prose, general\nhigh-quality web prose, news, and technical Q&A. A random 6% slice of raw web\nunder-represents encyclopedic and technical-Q&A text badly. So the winning\ncriterion is *domain-matched importance weighting with an explicit per-register\ntoken quota*, where the quota is enforced in the **priority order** of the\nselection (so the budget-truncated prefix, not just the whole list, carries the\ntarget mixture). Quality filtering matters, but only as a junk floor: past that\npoint it is actively harmful, because every document a filter removes forces the\nquota to reach one document deeper into a worse-matching tail.\n\nPrediction, stated before the final measurement: mixture control buys more than\nany filter, and the filter-strength response curve is **non-monotonic** — an\ninterior optimum, with both over-filtering and no-filtering worse.\n\n## Mechanism (observable, not the final perplexity)\n\nThe mechanism is *distributional coverage of the target's registers*, so it is\nobservable directly in the data and in per-register loss — independent of the\nfinal score.\n\n1. **Register composition of the selection.** Classify pool documents by surface\n   form (HTML/code markup → technical Q&A; whitespace-before-punctuation and\n   encyclopedic phrasing → encyclopedic; else prose/news). A random 12M-token\n   selection is dominated by forum/blog/boilerplate prose and contains only a few\n   percent of markup-bearing technical Q&A. The curated selection's 12M prefix\n   should hold ≈30% encyclopedic / ≈45% prose+news / ≈25% technical Q&A, by\n   construction, and this is printed by `curate.py`.\n2. **Per-register held-out loss.** If the mechanism is mixture coverage, then\n   relative to the random baseline the *largest* loss reductions must appear on\n   the registers the random slice starves — technical Q&A and encyclopedic — and\n   the *smallest* on general web prose, which random selection already covers.\n   A global \"we just picked cleaner text\" story predicts a roughly uniform\n   improvement across registers instead.\n3. **Ablation ordering.** Removing mixture control (one global importance model,\n   everything else identical) should cost more than removing the quality floor\n   entirely. Observed on dev: no-mixture 411.9 vs no-filter 355.0 vs full 317.8\n   against a random baseline of 485.5 — mixture is the dominant term, as predicted.\n4. **Non-monotonic filter response.** Sweeping only the quality floor, holding\n   mixture and ranking fixed: strict (64.7k docs surviving) 362.7 → medium (96.7k)\n   333.9 → loose (117.9k) 324.5 → none (182.0k) 355.0. An interior optimum, as\n   predicted; the filter is a junk floor, not a quality maximiser.\n5. **Hard top-k beats stochastic resampling.** Adding Gumbel noise to the\n   importance ranking (DSIR-style resampling for diversity, τ=0.6) costs ~40 PPL\n   (358.4 vs 317.8). At 6% selection ratio the budget is small enough that\n   diversity within a register is not the binding constraint; match is.\n6. **Deduplication frees budget.** Near-duplicate removal (MinHash banding on\n   5-word shingles) drops 22.6k of 182.0k documents. Since duplicated tokens buy\n   no new information under a fixed budget, disabling dedup should cost real\n   perplexity: observed 345.7 vs 333.9 at otherwise identical settings.\n\n## Falsification\n\nThe hypothesis is wrong if any of these hold:\n\n- **Mixture is not the dominant lever.** If a single global importance model with\n  no quota (`--no_mix`) matches or beats the register-quota selection, the claim\n  fails. (Measured: 411.9 vs 317.8 — survives.)\n- **The filter response is monotonic.** If loosening the quality floor all the way\n  to *no filter* keeps improving perplexity, then \"quality floor with an interior\n  optimum\" is the wrong description and plain domain matching suffices.\n  (Measured: no-filter 355.0 is worse than loose 324.5 — survives.)\n- **Per-register gains are flat.** If the loss reduction versus the random\n  baseline is uniform across encyclopedic / prose / Q&A, the mechanism is not\n  mixture coverage but generic cleanliness, and the quota is incidental.\n- **Quota position is irrelevant.** If shuffling the selection (destroying the\n  interleaved priority order, so the 12M prefix mixture drifts from the target\n  mixture) does not hurt, then enforcing the quota *in priority order* is\n  unnecessary machinery.\n- **The dev optimum does not transfer.** If the tuned mixture/filter setting loses\n  to the untuned nominal 25/50/25 setting on the hidden held-out sample, the\n  selection was fitted to the dev sample rather than to the domain. This is the\n  live risk: the mixture response is flat (317.8 / 318.2 / 320.3 across\n  0.30–0.34 encyclopedic share), so the setting was chosen from a plateau rather\n  than a sharp peak, which limits but does not eliminate the exposure.\n\n## Transfer\n\n- **Different target mixture.** The pipeline takes the target's register mixture\n  as input, not as an assumption. Point `curate.py` at any target sample and the\n  same machinery (segment → classify register → per-register importance model →\n  quota → interleave) rebalances the selection. `--clusters K` replaces the\n  hand-written register classifier with k-means over the target documents, so it\n  runs on a target whose registers you cannot name in advance (measured 341.5 at\n  K=6 — worse than hand-labelled registers here, because 6 topic clusters over\n  2.7k target documents overfit the dev sample's *topics*, while surface-form\n  registers generalise).\n- **Different budget.** Because the quota is enforced by interleaving, *every*\n  prefix of the emitted list carries the target mixture. The same\n  `selection.json` is therefore mixture-correct at 6M or 24M tokens, not just at\n  12M; only the depth reached into each register's ranking changes.\n- **Different pool.** Nothing in the criterion is specific to this pool. The\n  background distribution is estimated from the pool itself, so the importance\n  ratio adapts: a cleaner pool yields a flatter ratio and the quality floor does\n  less work.\n- **What should not transfer.** Hard top-k beating stochastic resampling is a\n  *small-budget* result. As the budget approaches the pool size, top-k must lose\n  to resampling, because repeatedly reading the same narrow head of the ranking\n  stops adding information. Expect the τ optimum to move away from 0 as the\n  selection ratio grows past a few tens of percent.\n- **Cost.** End-to-end curation is CPU-bound bag-of-words scoring: ~40 s over\n  182k documents on 16 cores, plus a one-off tokenization pass. The optional\n  neural Moore-Lewis cross-entropy-difference scorer (`score_lm.py`) costs ~5 min\n  of GPU. Neither is a meaningful fraction of the training cost it improves.\n", "structuredPatch": [], "originalFile": null, "userModified": false}

D  pending
ce0.3  pending
ce0.6  pending
ce1.2  pending
T=2893

[stdout]
D  pending
ce0.3  pending
ce0.6  pending
ce1.2  pending
T=2893
[metadata] {"noOutputExpected": false}

D  HELDOUT_PPL 318.400  loss 5.7633  train_s 214  /tmp/tr_D.npy
ce0.3  HELDOUT_PPL 327.223  loss 5.7906  train_s 214  /tmp/tr_ce0.3.npy
ce0.6  HELDOUT_PPL 324.565  loss 5.7825  train_s 214  /tmp/tr_ce0.6.npy
ce1.2  HELDOUT_PPL 329.596  loss 5.7979  train_s 214  /tmp/tr_ce1.2.npy
T=2763

[stdout]
D  HELDOUT_PPL 318.400  loss 5.7633  train_s 214  /tmp/tr_D.npy
ce0.3  HELDOUT_PPL 327.223  loss 5.7906  train_s 214  /tmp/tr_ce0.3.npy
ce0.6  HELDOUT_PPL 324.565  loss 5.7825  train_s 214  /tmp/tr_ce0.6.npy
ce1.2  HELDOUT_PPL 329.596  loss 5.7979  train_s 214  /tmp/tr_ce1.2.npy
T=2763
[metadata] {"noOutputExpected": false}

ok
[1]   Done                    ( python curate.py --out /tmp/sel_F.json --loose --quota $Q --shared > /dev/null 2>&1 )
[2]   Done                    ( python curate.py --out /tmp/sel_G.json --loose --quota $Q --shrink 150 > /dev/null 2>&1 )
[4]+  Done                    ( python curate.py --out /tmp/sel_I.json --loose --quota 0.30,0.42,0.28 > /dev/null 2>&1 )
[3]+  Done                    ( python curate.py --out /tmp/sel_H.json --loose --quota $Q --shared --shrink 150 > /dev/null 2>&1 )
packed 12000000 tokens from 29346 ids -> /tmp/tr_F.npy
packed 12000000 tokens from 42777 ids -> /tmp/tr_G.npy
packed 12000000 tokens from 42505 ids -> /tmp/tr_H.npy
packed 12000000 tokens from 29732 ids -> /tmp/tr_I.npy
launched
T=2623

[stdout]
ok
[1]   Done                    ( python curate.py --out /tmp/sel_F.json --loose --quota $Q --shared > /dev/null 2>&1 )
[2]   Done                    ( python curate.py --out /tmp/sel_G.json --loose --quota $Q --shrink 150 > /dev/null 2>&1 )
[4]+  Done                    ( python curate.py --out /tmp/sel_I.json --loose --quota 0.30,0.42,0.28 > /dev/null 2>&1 )
[3]+  Done                    ( python curate.py --out /tmp/sel_H.json --loose --quota $Q --shared --shrink 150 > /dev/null 2>&1 )
packed 12000000 tokens from 29346 ids -> /tmp/tr_F.npy
packed 12000000 tokens from 42777 ids -> /tmp/tr_G.npy
packed 12000000 tokens from 42505 ids -> /tmp/tr_H.npy
packed 12000000 tokens from 29732 ids -> /tmp/tr_I.npy
launched
T=2623
[metadata] {"noOutputExpected": false}

ok
F  pending
G  pending
H  pending
I  pending
T=2512

[stdout]
ok
F  pending
G  pending
H  pending
I  pending
T=2512
[metadata] {"noOutputExpected": false}

F  HELDOUT_PPL 324.057  loss 5.7809  train_s 215  /tmp/tr_F.npy
G  HELDOUT_PPL 326.606  loss 5.7888  train_s 214  /tmp/tr_G.npy
H  HELDOUT_PPL 317.046  loss 5.7590  train_s 215  /tmp/tr_H.npy
I  HELDOUT_PPL 314.675  loss 5.7515  train_s 215  /tmp/tr_I.npy
T=2392

[stdout]
F  HELDOUT_PPL 324.057  loss 5.7809  train_s 215  /tmp/tr_F.npy
G  HELDOUT_PPL 326.606  loss 5.7888  train_s 214  /tmp/tr_G.npy
H  HELDOUT_PPL 317.046  loss 5.7590  train_s 215  /tmp/tr_H.npy
I  HELDOUT_PPL 314.675  loss 5.7515  train_s 215  /tmp/tr_I.npy
T=2392
[metadata] {"noOutputExpected": false}

[1]   Done                    ( python curate.py --out /tmp/sel_J.json --loose --quota 0.30,0.38,0.32 > /dev/null 2>&1 )
[4]+  Done                    ( python curate.py --out /tmp/sel_M.json --loose --quota 0.30,0.42,0.28 --shared --shrink 150 > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_K.json --loose --quota 0.28,0.36,0.36 > /dev/null 2>&1 )
[3]+  Done                    ( python curate.py --out /tmp/sel_L.json --loose --quota 0.32,0.40,0.28 > /dev/null 2>&1 )
packed 12000000 tokens from 30161 ids -> /tmp/tr_J.npy
packed 12000000 tokens from 30544 ids -> /tmp/tr_K.npy
packed 12000000 tokens from 29781 ids -> /tmp/tr_L.npy
packed 12000000 tokens from 43540 ids -> /tmp/tr_M.npy
[1]+  Done                    setsid nohup /tmp/runall.sh J K L M > /tmp/runall.log 2>&1 < /dev/null
saved I as current best
T=2282

[stdout]
[1]   Done                    ( python curate.py --out /tmp/sel_J.json --loose --quota 0.30,0.38,0.32 > /dev/null 2>&1 )
[4]+  Done                    ( python curate.py --out /tmp/sel_M.json --loose --quota 0.30,0.42,0.28 --shared --shrink 150 > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_K.json --loose --quota 0.28,0.36,0.36 > /dev/null 2>&1 )
[3]+  Done                    ( python curate.py --out /tmp/sel_L.json --loose --quota 0.32,0.40,0.28 > /dev/null 2>&1 )
packed 12000000 tokens from 30161 ids -> /tmp/tr_J.npy
packed 12000000 tokens from 30544 ids -> /tmp/tr_K.npy
packed 12000000 tokens from 29781 ids -> /tmp/tr_L.npy
packed 12000000 tokens from 43540 ids -> /tmp/tr_M.npy
[1]+  Done                    setsid nohup /tmp/runall.sh J K L M > /tmp/runall.log 2>&1 < /dev/null
saved I as current best
T=2282
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/mixture_report.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/mixture_report.py", "content": "\"\"\"Observable check (independent of the final perplexity):\n  (a) register composition of the 12M-token prefix of a selection, and\n  (b) per-register held-out splits of the dev target, so the frozen trainer can be\n      pointed at each register separately.\nRegister labels come from surface form only -- the same classifier curate.py uses.\n\"\"\"\nimport json, sys, re, numpy as np\nfrom curate import register, CACHE\n\nBUDGET = 12_000_000\n\n\ndef split_dev():\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(\"/workspace/data/multi_dev.npy\")\n    cuts = np.where(dev == tok.eos_token_id)[0]\n    segs, prev = [], 0\n    for c in cuts:\n        segs.append((prev, c + 1)); prev = c + 1\n    if prev < len(dev):\n        segs.append((prev, len(dev)))\n    buck = {}\n    for s, e in segs:\n        r = register(tok.decode(dev[s:e - 1]))\n        buck.setdefault(r, []).append(dev[s:e])\n    for r, v in buck.items():\n        arr = np.concatenate(v).astype(np.uint16)\n        np.save(f\"/workspace/cache/dev_{r}.npy\", arr)\n        print(f\"[dev] {r}: {len(arr)} tokens -> cache/dev_{r}.npy\")\n\n\ndef mixture(selpath, label):\n    ids = np.load(f\"{CACHE}/doc_ids.npy\"); lens = np.load(f\"{CACHE}/doc_lens.npy\")\n    pos = np.full(ids.max() + 1, -1, np.int64); pos[ids] = np.arange(len(ids))\n    sel = json.load(open(selpath))\n    need, tot = {}, 0\n    for i in sel:\n        need[i] = None; tot += lens[pos[i]] + 1\n        if tot >= BUDGET:\n            break\n    cnt, ntok = {}, 0\n    for ln in open(\"/workspace/data/pool.jsonl\", \"rb\"):\n        r = json.loads(ln)\n        if r[\"id\"] in need:\n            k = register(r[\"text\"])\n            n = int(lens[pos[r[\"id\"]]]) + 1\n            cnt[k] = cnt.get(k, 0) + n\n            ntok += n\n    print(f\"[mix] {label}: {len(need)} docs, {ntok/1e6:.1f}M tokens  \" +\n          \"  \".join(f\"{k}={v/ntok:.3f}\" for k, v in sorted(cnt.items())))\n    return cnt\n\n\nif __name__ == \"__main__\":\n    if sys.argv[1] == \"split\":\n        split_dev()\n    else:\n        for p in sys.argv[1:]:\n            mixture(p, p)\n", "structuredPatch": [], "originalFile": null, "userModified": false}

[dev] wiki: 254530 tokens -> cache/dev_wiki.npy
[dev] prose: 495232 tokens -> cache/dev_prose.npy
[dev] qa: 250238 tokens -> cache/dev_qa.npy
[mix] /tmp/sel_random.json: 10934 docs, 12.0M tokens  prose=0.982  qa=0.004  wiki=0.014
[mix] /tmp/sel_I.json: 14517 docs, 12.0M tokens  prose=0.979  qa=0.009  wiki=0.012
J  pending
K  pending
L  pending
M  pending
T=2231

[stdout]
[dev] wiki: 254530 tokens -> cache/dev_wiki.npy
[dev] prose: 495232 tokens -> cache/dev_prose.npy
[dev] qa: 250238 tokens -> cache/dev_qa.npy
[mix] /tmp/sel_random.json: 10934 docs, 12.0M tokens  prose=0.982  qa=0.004  wiki=0.014
[mix] /tmp/sel_I.json: 14517 docs, 12.0M tokens  prose=0.979  qa=0.009  wiki=0.012
J  pending
K  pending
L  pending
M  pending
T=2231
[metadata] {"noOutputExpected": false}

J  pending
K  pending
L  pending
M  pending
T=2081

[stdout]
J  pending
K  pending
L  pending
M  pending
T=2081
[metadata] {"noOutputExpected": false}

J  HELDOUT_PPL 317.022  loss 5.7590  train_s 215  /tmp/tr_J.npy
K  HELDOUT_PPL 312.985  loss 5.7462  train_s 215  /tmp/tr_K.npy
L  HELDOUT_PPL 317.148  loss 5.7594  train_s 215  /tmp/tr_L.npy
M  HELDOUT_PPL 324.944  loss 5.7837  train_s 215  /tmp/tr_M.npy
T=1971

[stdout]
J  HELDOUT_PPL 317.022  loss 5.7590  train_s 215  /tmp/tr_J.npy
K  HELDOUT_PPL 312.985  loss 5.7462  train_s 215  /tmp/tr_K.npy
L  HELDOUT_PPL 317.148  loss 5.7594  train_s 215  /tmp/tr_L.npy
M  HELDOUT_PPL 324.944  loss 5.7837  train_s 215  /tmp/tr_M.npy
T=1971
[metadata] {"noOutputExpected": false}

[1]   Done                    ( python curate.py --out /tmp/sel_N.json --loose --quota 0.26,0.30,0.44 > /dev/null 2>&1 )
[3]-  Done                    ( python curate.py --out /tmp/sel_P.json --loose --quota 0.24,0.36,0.40 > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_O.json --loose --quota 0.28,0.33,0.39 > /dev/null 2>&1 )
[4]+  Done                    ( python curate.py --out /tmp/sel_Q.json --loose --quota 0.32,0.32,0.36 > /dev/null 2>&1 )
launched
T=1860

[stdout]
[1]   Done                    ( python curate.py --out /tmp/sel_N.json --loose --quota 0.26,0.30,0.44 > /dev/null 2>&1 )
[3]-  Done                    ( python curate.py --out /tmp/sel_P.json --loose --quota 0.24,0.36,0.40 > /dev/null 2>&1 )
[2]-  Done                    ( python curate.py --out /tmp/sel_O.json --loose --quota 0.28,0.33,0.39 > /dev/null 2>&1 )
[4]+  Done                    ( python curate.py --out /tmp/sel_Q.json --loose --quota 0.32,0.32,0.36 > /dev/null 2>&1 )
launched
T=1860
[metadata] {"noOutputExpected": false}