claude-opus-5

A: HELDOUT_PPL 343.731  loss 5.8399  train_s 177  /tmp/tr_A.npy
B: HELDOUT_PPL 362.215  loss 5.8922  train_s 177  /tmp/tr_B.npy
C: HELDOUT_PPL 339.323  loss 5.8270  train_s 177  /tmp/tr_C.npy
2256

[stdout]
A: HELDOUT_PPL 343.731  loss 5.8399  train_s 177  /tmp/tr_A.npy
B: HELDOUT_PPL 362.215  loss 5.8922  train_s 177  /tmp/tr_B.npy
C: HELDOUT_PPL 339.323  loss 5.8270  train_s 177  /tmp/tr_C.npy
2256
[metadata] {"noOutputExpected": false}

ok
[1]   Done                    python curate.py --lo 1 --hi 99 --pad 0.45 --out /tmp/sel_D.json > /tmp/bD.log 2>&1
[2]-  Done                    python curate.py --loose --lenmatch --min_words 40 --out /tmp/sel_E.json > /tmp/bE.log 2>&1
[3]+  Done                    python curate.py --loose --rank random --out /tmp/sel_G.json > /tmp/bG.log 2>&1
acceptance box: pct[1.0,99.0] pad 0.3
wrote 34957 ids -> /tmp/sel_G.json  (~36.0M est tokens; first ~12146 ids fill the 12M budget)
acceptance box: pct[1.0,99.0] pad 0.45
wrote 34023 ids -> /tmp/sel_D.json  (~36.0M est tokens; first ~7815 ids fill the 12M budget)
acceptance box: pct[1.0,99.0] pad 0.3
wrote 60355 ids -> /tmp/sel_E.json  (~36.0M est tokens; first ~24260 ids fill the 12M budget)
2115

[stdout]
ok
[1]   Done                    python curate.py --lo 1 --hi 99 --pad 0.45 --out /tmp/sel_D.json > /tmp/bD.log 2>&1
[2]-  Done                    python curate.py --loose --lenmatch --min_words 40 --out /tmp/sel_E.json > /tmp/bE.log 2>&1
[3]+  Done                    python curate.py --loose --rank random --out /tmp/sel_G.json > /tmp/bG.log 2>&1
acceptance box: pct[1.0,99.0] pad 0.3
wrote 34957 ids -> /tmp/sel_G.json  (~36.0M est tokens; first ~12146 ids fill the 12M budget)
acceptance box: pct[1.0,99.0] pad 0.45
wrote 34023 ids -> /tmp/sel_D.json  (~36.0M est tokens; first ~7815 ids fill the 12M budget)
acceptance box: pct[1.0,99.0] pad 0.3
wrote 60355 ids -> /tmp/sel_E.json  (~36.0M est tokens; first ~24260 ids fill the 12M budget)
2115
[metadata] {"noOutputExpected": false}

launched
target register token shares: [0.016 0.258 0.064 0.533 0.129]
acceptance box: pct[1.0,99.0] pad 0.3
duplicates removed: 32736
  5 register x length cells; unmatched target mass 0.000
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 34455 ids -> /workspace/submission/selection.json  (~36.0M est tokens; first ~8195 ids fill the 12M budget)

[stdout]
launched
target register token shares: [0.016 0.258 0.064 0.533 0.129]
acceptance box: pct[1.0,99.0] pad 0.3
duplicates removed: 32736
  5 register x length cells; unmatched target mass 0.000
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 34455 ids -> /workspace/submission/selection.json  (~36.0M est tokens; first ~8195 ids fill the 12M budget)
[metadata] {"noOutputExpected": false}

identical to evaluated sel_loose: True
n ids 34455 unique 34455 min 1 max 181994 all int True
total 239
drwxr-xr-x 1 root root     60 Aug  4 02:05 .
drwxr-xr-x 1 root root    420 Aug  4 02:05 ..
-rw-r--r-- 1 root root 244286 Aug  4 02:05 selection.json

[stdout]
identical to evaluated sel_loose: True
n ids 34455 unique 34455 min 1 max 181994 all int True
total 239
drwxr-xr-x 1 root root     60 Aug  4 02:05 .
drwxr-xr-x 1 root root    420 Aug  4 02:05 ..
-rw-r--r-- 1 root root 244286 Aug  4 02:05 selection.json
[metadata] {"noOutputExpected": false}

D: (running)
E: (running)
G: (running)
1935

[stdout]
D: (running)
E: (running)
G: (running)
1935
[metadata] {"noOutputExpected": false}

D: HELDOUT_PPL 347.517  loss 5.8508  train_s 175  /tmp/tr_D.npy
E: HELDOUT_PPL 343.053  loss 5.8379  train_s 173  /tmp/tr_E.npy
G: HELDOUT_PPL 378.356  loss 5.9358  train_s 176  /tmp/tr_G.npy
1815

[stdout]
D: HELDOUT_PPL 347.517  loss 5.8508  train_s 175  /tmp/tr_D.npy
E: HELDOUT_PPL 343.053  loss 5.8379  train_s 173  /tmp/tr_E.npy
G: HELDOUT_PPL 378.356  loss 5.9358  train_s 176  /tmp/tr_G.npy
1815
[metadata] {"noOutputExpected": false}

patched
syntax ok
target register token shares: [0.016 0.258 0.064 0.533 0.129]
acceptance box: pct[1.0,99.0] pad 0.3
duplicates removed: 32736
  5 register x length cells; unmatched target mass 0.000
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 34455 ids -> /tmp/sel_final.json  (~36.0M est tokens; first ~8195 ids fill the 12M budget)
default config reproduces the evaluated selection: True

[stdout]
patched
syntax ok
target register token shares: [0.016 0.258 0.064 0.533 0.129]
acceptance box: pct[1.0,99.0] pad 0.3
duplicates removed: 32736
  5 register x length cells; unmatched target mass 0.000
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 34455 ids -> /tmp/sel_final.json  (~36.0M est tokens; first ~8195 ids fill the 12M budget)
default config reproduces the evaluated selection: True
[metadata] {"noOutputExpected": false}

emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 34455 ids -> /workspace/submission/selection.json  (~36.0M est tokens; first ~8195 ids fill the 12M budget)
submission == evaluated selection: True | n ids 34455 | unique 34455
range ok: True
total 270
drwxr-xr-x 1 root root    160 Aug  4 02:11 .
drwxr-xr-x 1 root root    420 Aug  4 02:05 ..
drwxr-xr-x 1 root root     80 Aug  4 02:11 __pycache__
-rw-r--r-- 1 root root  14383 Aug  4 02:11 curate.py
-rw-r--r-- 1 root root   4873 Aug  4 02:11 diag_eval.py
-rw-r--r-- 1 root root   4628 Aug  4 02:11 feat.py
-rw-r--r-- 1 root root   5883 Aug  4 02:11 score.py
-rw-r--r-- 1 root root 244286 Aug  4 02:12 selection.json

[stdout]
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 34455 ids -> /workspace/submission/selection.json  (~36.0M est tokens; first ~8195 ids fill the 12M budget)
submission == evaluated selection: True | n ids 34455 | unique 34455
range ok: True
total 270
drwxr-xr-x 1 root root    160 Aug  4 02:11 .
drwxr-xr-x 1 root root    420 Aug  4 02:05 ..
drwxr-xr-x 1 root root     80 Aug  4 02:11 __pycache__
-rw-r--r-- 1 root root  14383 Aug  4 02:11 curate.py
-rw-r--r-- 1 root root   4873 Aug  4 02:11 diag_eval.py
-rw-r--r-- 1 root root   4628 Aug  4 02:11 feat.py
-rw-r--r-- 1 root root   5883 Aug  4 02:11 score.py
-rw-r--r-- 1 root root 244286 Aug  4 02:12 selection.json
[metadata] {"noOutputExpected": false}

The file /workspace/claim.md has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"type": "update", "filePath": "/workspace/claim.md", "content": "# Curating a raw web pool for a fixed 12M-token budget\n\n**Result: dev perplexity 336.1 vs 469.9 for a random selection of the same pool (−28.5%),\nusing the frozen recipe at 12M tokens, seed 1337.**\n\n## Hypothesis\n\nAt a budget of ~6% of the pool, held-out perplexity on a broad multi-domain target is governed\nless by *generic document quality* than by **distributional match to the target's register\nmixture**, and the binding constraint is diversity rather than purity. Three sub-claims:\n\n1. **The target is its own best supervision.** `data/multi_dev.npy` is 1M GPT-2 tokens *drawn from\n   the target distribution*; decoding it back to text yields ~2.7k in-domain sample documents. A\n   hashed-unigram importance score `s(d) = mean_{w∈d} log p_target(w) − log p_pool(w)`\n   (DSIR, Xie et al. 2023) built from those samples should rank pool documents usefully with no\n   labels, no reference LM and no internet.\n2. **Ranking alone is not enough.** Greedy top-k concentrates the budget on whichever register is\n   easiest to match and starves the others. Imposing **per-register token quotas equal to the\n   target's own register token mass** (registers from spherical k-means on the decoded target)\n   should beat pure ranking.\n3. **Quality filtering must be a junk filter, not a quality bar.** Filtering to the *centre* of the\n   target's surface-statistic distribution should *hurt*, even though every surviving document\n   looks more target-like, because at 6% of the pool diversity is the scarce resource.\n\n## Mechanism — predicted observables other than the final perplexity\n\n**(a) The scorer must generalise across topics, not memorise them.** Predicted *before* any\ntraining run: fit the importance score on a random 70% of the target documents; held-out target\ndocuments should outrank random pool documents with AUC ≥ 0.9. **Measured AUC = 0.9517.**\nThis observable is what rejected the first attempt: a discriminative logistic regression on the\nsame features (2.7k positives, 2^18 hashed dims) put its largest weights on proper nouns —\n`bush`, `grissom`, `wilder`, `boosey`, `shackleton` — and scored only 0.6% of the pool positive.\nThat is topic memorisation, not register, and it would not transfer to the hidden sample.\nRestricting the feature space to the 40k buckets frequent in the pool and switching to a smoothed\nlikelihood ratio fixed it. *Diagnosis and repair happened entirely on this observable.*\n\n**(b) The gain from quota-balancing must be concentrated in the register that greedy ranking\nstarves, not spread evenly.** This is the discriminating prediction — if balancing merely raised\naverage data quality, all registers would improve together. Per-register held-out perplexity\n(`diag_eval.py`; identical frozen recipe, each eval window assigned to the register of its\nmajority document):\n\n| register (k=5) | rank-only | quota-balanced | change |\n|---|---|---|---|\n| 1 · technical Q&A / code | 409.1 | **289.7** | **−29%** |\n| 0 · niche web prose | 850.6 | 783.7 | −8% |\n| 2 · science / geography wiki | 720.0 | 703.0 | −2% |\n| 3 · news (53% of target mass) | 298.0 | 296.2 | −1% |\n| 4 · biography wiki | 762.4 | 766.1 | +0.5% |\n\n**Confirmed:** essentially the entire improvement is register 1, which greedy ranking under-served,\nwhile the dominant news register is untouched. Worst-register perplexity falls 850.6 → 783.7 and\nthe max/min spread narrows 2.85 → 2.71. One sub-prediction **failed**: the *log* spread across\nregisters widened (0.407 → 0.462), because one starved register improved sharply rather than all\nregisters converging. Dispersion is therefore not a valid summary of this mechanism; concentration\nof the gain is.\n\n**(c) A floor exists that no selection can remove.** 863 / 2705 target documents (32%, the\nencyclopedic portion) carry the WikiText detokenisation artifact ` @-@ `; `grep` finds it in\n**0 of 182,016** pool documents. Predicted consequence: the wiki-formatted registers stay the\nworst-served however the pool is curated, and absolute perplexity stays in the hundreds.\nConfirmed — registers 0/2/4 remain at 703–784 while news reaches 296. Any curation effort spent\nchasing the encyclopedic register is capped by pool coverage, not by the selection algorithm.\n\n## Falsification\n\nEvery component was given a way to fail, and two predictions did fail. All numbers are dev\nperplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.\n\n| test | prediction | measured | verdict |\n|---|---|---|---|\n| random pool sample (do-nothing) | — | 469.9 | reference |\n| DSIR ranking, no register quotas | quotas win | 394.3 | ✓ |\n| no acceptance box (length + dedup only) | box helps | 381.2 | ✓ |\n| **random order within the same quotas** | **ranking carries real signal** | **378.4** | **✓ (+42)** |\n| *tighter* box (5–95 pct, pad .05) | *hurts* — diversity is scarce | 437.9 | ✓ (sign predicted) |\n| box loosened past the optimum (0.5–99.5, pad .6) | hurts | 362.2 | ✓ |\n| box pad 0.45 vs 0.30 | — | 347.5 | 0.30 optimal |\n| exact-dedup only (keep 21k near-dupes) | near-dedup helps | 343.7 | ✓ |\n| document-length matching (median 726→283 tok) | helps | 343.1 / 359.2 | ✗ **no effect** |\n| finer registers (K=12 vs K=5) | helps | 361.3 | ✗ no effect |\n| loss-mass-weighted quotas (∝ mass × ppl) | helps | 358.0 | ✗ no effect |\n\nThe hypothesis would have been falsified outright if quota-balancing had not beaten pure ranking,\nif random ordering within quotas had matched the DSIR ordering (it loses 42 perplexity, so the\nranking is load-bearing), or if the per-register breakdown had shown uniform rather than\nconcentrated improvement.\n\nTwo clean negatives worth recording. **Length matching does nothing:** the target's documents\nhave a median of 192 GPT-2 tokens against 726 for a score-ranked selection, and forcing the\nselection to reproduce the target's length histogram (1221 → 497 mean tokens, 9.8k → 24.1k\ndocuments inside the budget) moved perplexity by +0.3 at the tight box and +7 at the loose box.\nWhatever the EOS-boundary statistics of the eval stream cost, selection cannot recover it.\n**Filtering and dedup interact:** near-duplicate removal *helps* under the loose box\n(336.1 vs 343.7) but *hurt* under the tighter one (358.9 vs 348.8) — the two are substitutes,\nboth spending the same diversity budget, so their strengths cannot be tuned independently.\n\n## Transfer\n\nThe method needs only (i) a raw pool and (ii) a sample of the target distribution — no labels, no\nreference LM, no internet. Cost for 182k documents / 200M tokens: **45 s of CPU** for hashed\nfeatures (15 processes) plus a few seconds of GPU for scoring, both linear in pool size, so the\nsame code runs on a pool 1000× larger. It transfers directly to budgeted pretraining and\ndomain-adaptive pretraining wherever a modest in-domain sample exists.\n\nThe part worth carrying over is the register-quota step: it is what converts a *similarity score*\ninto a *mixture*, and it delivers the largest single gain over rank-only selection. The\ntransferable warning is the opposite of the usual instinct — **do not tighten the quality filter**.\nEvery step toward a purer, more target-like selection past the junk-filter threshold cost\nperplexity here (437.9 at the tight box vs 336.1), because the budget is small enough that\ncoverage, not cleanliness, is binding.\n\nTwo failure modes are diagnosable in advance, without training:\n* `unmatched target mass`, printed by `curate.py` (0.000 here), is the fraction of target token\n  mass for which no eligible pool document exists. If it is large the pool cannot express the\n  target and the quotas will silently redistribute — as happened in an early version whose\n  per-register acceptance boxes, estimated from as few as 156 documents, starved three of five\n  registers and drove the emitted mixture to 87% news.\n* The scorer's held-out AUC is a go/no-go check before any GPU time is spent: below ~0.9 the\n  importance score is memorising topics rather than registers and the ranking should not be trusted.\n\n## Files\n\n* `submission/selection.json` — 34,455 pool ids in priority order; the first ~8.2k fill the 12M\n  budget, the remainder is spare so the pipeline can never run short.\n* `submission/curate.py` — reproduces `selection.json` exactly from the stated criterion\n  (`python curate.py`, no flags); `feat.py` / `score.py` are its two cached stages.\n* `submission/diag_eval.py` — the per-register diagnostic used for mechanism observable (b). It\n  duplicates the frozen recipe verbatim and is not part of the official run.\n", "structuredPatch": [{"oldStart": 1, "oldLines": 39, "newStart": 1, "newLines": 44, "lines": [" # Curating a raw web pool for a fixed 12M-token budget", " ", "+**Result: dev perplexity 336.1 vs 469.9 for a random selection of the same pool (−28.5%),", "+using the frozen recipe at 12M tokens, seed 1337.**", "+", " ## Hypothesis", " ", "-At a fixed token budget far below the pool size, held-out perplexity on a broad multi-domain", "-target is governed less by *generic document quality* than by **distributional match to the", "-target's register mixture**. Specifically:", "+At a budget of ~6% of the pool, held-out perplexity on a broad multi-domain target is governed", "+less by *generic document quality* than by **distributional match to the target's register", "+mixture**, and the binding constraint is diversity rather than purity. Three sub-claims:", " ", "-1. The single most useful supervision available is the dev target itself. `data/multi_dev.npy` is", "-   1M GPT-2 tokens *drawn from the target distribution*; decoding it back to text yields ~2.7k", "-   in-domain sample documents. A hashed-unigram importance score", "-   `s(d) = mean_{w in d} log p_target(w) - log p_pool(w)` (DSIR, Xie et al. 2023) built from those", "-   samples ranks pool documents by target-likeness with no labels and no external model.", "-2. Ranking by that score alone is **not** enough: greedy top-k concentrates the budget on", "-   whichever register is easiest to match and starves the others. Imposing **per-register token", "-   quotas equal to the target's own register token mass** (registers found by spherical k-means on", "-   the decoded target) should beat pure ranking.", "-3. Quality filtering should be a **junk filter, not a quality bar**. Filtering to the *centre* of", "-   the target's surface-statistic distribution destroys diversity and should *hurt*, even though", "-   every document kept looks more target-like.", "+1. **The target is its own best supervision.** `data/multi_dev.npy` is 1M GPT-2 tokens *drawn from", "+   the target distribution*; decoding it back to text yields ~2.7k in-domain sample documents. A", "+   hashed-unigram importance score `s(d) = mean_{w∈d} log p_target(w) − log p_pool(w)`", "+   (DSIR, Xie et al. 2023) built from those samples should rank pool documents usefully with no", "+   labels, no reference LM and no internet.", "+2. **Ranking alone is not enough.** Greedy top-k concentrates the budget on whichever register is", "+   easiest to match and starves the others. Imposing **per-register token quotas equal to the", "+   target's own register token mass** (registers from spherical k-means on the decoded target)", "+   should beat pure ranking.", "+3. **Quality filtering must be a junk filter, not a quality bar.** Filtering to the *centre* of the", "+   target's surface-statistic distribution should *hurt*, even though every surviving document", "+   looks more target-like, because at 6% of the pool diversity is the scarce resource.", " ", "-## Mechanism — predicted observables (not the final perplexity)", "+## Mechanism — predicted observables other than the final perplexity", " ", " **(a) The scorer must generalise across topics, not memorise them.** Predicted *before* any", " training run: fit the importance score on a random 70% of the target documents; held-out target", "-documents should outrank random pool documents with AUC ≥ 0.9. **Measured: AUC = 0.9517.**", "-A discriminative logistic-regression alternative was rejected on this observable, not on", "-perplexity: with 2.7k positives and 2^18 hashed features it drove its top weights onto proper", "-nouns (`bush`, `grissom`, `wilder`, `boosey`) and flagged only 0.6% of the pool — topic", "-memorisation, not register. Restricting the feature space to buckets frequent in the pool and", "-scoring by a smoothed likelihood ratio fixes it.", "+documents should outrank random pool documents with AUC ≥ 0.9. **Measured AUC = 0.9517.**", "+This observable is what rejected the first attempt: a discriminative logistic regression on the", "+same features (2.7k positives, 2^18 hashed dims) put its largest weights on proper nouns —", "+`bush`, `grissom`, `wilder`, `boosey`, `shackleton` — and scored only 0.6% of the pool positive.", "+That is topic memorisation, not register, and it would not transfer to the hidden sample.", "+Restricting the feature space to the 40k buckets frequent in the pool and switching to a smoothed", "+likelihood ratio fixed it. *Diagnosis and repair happened entirely on this observable.*", " ", "-**(b) The gain from quota-balancing must be *concentrated in the register that greedy ranking", "-starves*, not spread evenly.** This is the discriminating prediction: if balancing merely", "-improved average data quality, all registers would improve together. Measured per-register", "-held-out perplexity (`diag_eval.py`, identical frozen recipe, register assigned per eval window):", "+**(b) The gain from quota-balancing must be concentrated in the register that greedy ranking", "+starves, not spread evenly.** This is the discriminating prediction — if balancing merely raised", "+average data quality, all registers would improve together. Per-register held-out perplexity", "+(`diag_eval.py`; identical frozen recipe, each eval window assigned to the register of its", "+majority document):", " ", " | register (k=5) | rank-only | quota-balanced | change |", " |---|---|---|---|"]}, {"oldStart": 43, "oldLines": 63, "newStart": 48, "newLines": 82, "lines": [" | 3 · news (53% of target mass) | 298.0 | 296.2 | −1% |", " | 4 · biography wiki | 762.4 | 766.1 | +0.5% |", " ", "-Confirmed: essentially the whole improvement is register 1, which greedy ranking under-served,", "-while the dominant news register is unchanged. Worst-register perplexity falls 850.6 → 783.7 and", "-the max/min spread narrows 2.85 → 2.71. One sub-prediction failed: the *log* spread across", "-registers widened (0.407 → 0.462), because a single starved register improved sharply rather than", "-all registers converging — the gain is concentrated, so dispersion is not a reliable summary of it.", "+**Confirmed:** essentially the entire improvement is register 1, which greedy ranking under-served,", "+while the dominant news register is untouched. Worst-register perplexity falls 850.6 → 783.7 and", "+the max/min spread narrows 2.85 → 2.71. One sub-prediction **failed**: the *log* spread across", "+registers widened (0.407 → 0.462), because one starved register improved sharply rather than all", "+registers converging. Dispersion is therefore not a valid summary of this mechanism; concentration", "+of the gain is.", " ", "-**(c) A hard floor exists that no selection can remove.** 863 / 2705 target documents (32%,", "-the encyclopedic portion) carry the WikiText detokenisation artifact ` @-@ `; `grep` finds it in", "-**0** of 182,016 pool documents. Predicted consequence: the wiki-formatted registers stay the", "-worst-served no matter how the pool is curated, and absolute perplexity stays in the hundreds.", "-Confirmed — registers 0/2/4 remain at 703–784 while news reaches 296.", "+**(c) A floor exists that no selection can remove.** 863 / 2705 target documents (32%, the", "+encyclopedic portion) carry the WikiText detokenisation artifact ` @-@ `; `grep` finds it in", "+**0 of 182,016** pool documents. Predicted consequence: the wiki-formatted registers stay the", "+worst-served however the pool is curated, and absolute perplexity stays in the hundreds.", "+Confirmed — registers 0/2/4 remain at 703–784 while news reaches 296. Any curation effort spent", "+chasing the encyclopedic register is capped by pool coverage, not by the selection algorithm.", " ", " ## Falsification", " ", "-Each claim was given a way to fail, and the third partly did:", "+Every component was given a way to fail, and two predictions did fail. All numbers are dev", "+perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.", " ", "-| test | prediction | result |", "-|---|---|---|", "-| quotas vs pure ranking | quotas win | 358.9 vs 394.3 ✓ |", "-| acceptance box vs none | box wins | 358.9 vs 381.2 ✓ |", "-| **tighter** box (5–95 pct) | *hurts* — diversity, not purity, is scarce | 437.9 vs 358.9 ✓ (predicted sign) |", "-| document-length matching | helps (target median 192 tok vs 726 selected) | 359.2 vs 358.9 — **no effect** ✗ |", "-| finer registers (K=12 vs 5) | helps | 361.3 vs 358.9 — no effect ✗ |", "-| loss-mass-weighted quotas | helps | see table below |", "+| test | prediction | measured | verdict |", "+|---|---|---|---|", "+| random pool sample (do-nothing) | — | 469.9 | reference |", "+| DSIR ranking, no register quotas | quotas win | 394.3 | ✓ |", "+| no acceptance box (length + dedup only) | box helps | 381.2 | ✓ |", "+| **random order within the same quotas** | **ranking carries real signal** | **378.4** | **✓ (+42)** |", "+| *tighter* box (5–95 pct, pad .05) | *hurts* — diversity is scarce | 437.9 | ✓ (sign predicted) |", "+| box loosened past the optimum (0.5–99.5, pad .6) | hurts | 362.2 | ✓ |", "+| box pad 0.45 vs 0.30 | — | 347.5 | 0.30 optimal |", "+| exact-dedup only (keep 21k near-dupes) | near-dedup helps | 343.7 | ✓ |", "+| document-length matching (median 726→283 tok) | helps | 343.1 / 359.2 | ✗ **no effect** |", "+| finer registers (K=12 vs K=5) | helps | 361.3 | ✗ no effect |", "+| loss-mass-weighted quotas (∝ mass × ppl) | helps | 358.0 | ✗ no effect |", " ", " The hypothesis would have been falsified outright if quota-balancing had not beaten pure ranking,", "-or if the per-register breakdown had shown a uniform improvement instead of one concentrated in", "-the starved register. Length-matching is the clearest negative: matching the target's", "-document-length distribution (mean selected length 1221 → 497 tokens, 9.8k → 24.1k documents in", "-the budget) changed perplexity by +0.3, inside single-seed noise. Whatever the EOS-boundary", "-statistics of the eval stream cost, it is not recoverable by selection.", "+if random ordering within quotas had matched the DSIR ordering (it loses 42 perplexity, so the", "+ranking is load-bearing), or if the per-register breakdown had shown uniform rather than", "+concentrated improvement.", " ", "-## Results (frozen recipe, 12M tokens, seed 1337, dev target)", "+Two clean negatives worth recording. **Length matching does nothing:** the target's documents", "+have a median of 192 GPT-2 tokens against 726 for a score-ranked selection, and forcing the", "+selection to reproduce the target's length histogram (1221 → 497 mean tokens, 9.8k → 24.1k", "+documents inside the budget) moved perplexity by +0.3 at the tight box and +7 at the loose box.", "+Whatever the EOS-boundary statistics of the eval stream cost, selection cannot recover it.", "+**Filtering and dedup interact:** near-duplicate removal *helps* under the loose box", "+(336.1 vs 343.7) but *hurt* under the tighter one (358.9 vs 348.8) — the two are substitutes,", "+both spending the same diversity budget, so their strengths cannot be tuned independently.", " ", "-| selection | dev PPL |", "-|---|---|", "-| random pool sample (do-nothing baseline) | 469.9 |", "-| DSIR rank-only, no register quotas | 394.3 |", "-| quotas, no acceptance box (length + dedup only) | 381.2 |", "-| quotas + box, K=12 registers | 361.3 |", "-| quotas + box + document-length matching | 359.2 |", "-| **quotas + box, K=5 (submitted configuration)** | **358.9** |", "-| quotas + *tight* box | 437.9 |", "-", " ## Transfer", " ", " The method needs only (i) a raw pool and (ii) a sample of the target distribution — no labels, no", "-reference LM, no internet. Cost for 182k documents / 200M tokens: 45 s of CPU for hashed features", "-(15 processes) and a few seconds of GPU for scoring; both are linear in pool size, so the same", "-recipe runs on a pool 1000× larger with the same code. It transfers directly to any budgeted", "-pretraining or domain-adaptive-pretraining problem where a modest in-domain sample exists, and the", "-register-quota step is the part worth carrying over: it is what converts a *similarity* score into", "-a *mixture* and it is where 90% of the measured gain over rank-only selection comes from.", "+reference LM, no internet. Cost for 182k documents / 200M tokens: **45 s of CPU** for hashed", "+features (15 processes) plus a few seconds of GPU for scoring, both linear in pool size, so the", "+same code runs on a pool 1000× larger. It transfers directly to budgeted pretraining and", "+domain-adaptive pretraining wherever a modest in-domain sample exists.", " ", "-Two transfer caveats are diagnosable in advance, without training. The `unmatched target mass`", "-figure printed by `curate.py` (0.004 here) reports the fraction of target token mass for which no", "-eligible pool document exists — if it is large, the pool cannot express the target and quotas will", "-silently redistribute. And the scorer's held-out AUC is a pre-training go/no-go check: below ~0.9", "-the importance score is memorising topics rather than registers, and the ranking should not be", "-trusted.", "+The part worth carrying over is the register-quota step: it is what converts a *similarity score*", "+into a *mixture*, and it delivers the largest single gain over rank-only selection. The", "+transferable warning is the opposite of the usual instinct — **do not tighten the quality filter**.", "+Every step toward a purer, more target-like selection past the junk-filter threshold cost", "+perplexity here (437.9 at the tight box vs 336.1), because the budget is small enough that", "+coverage, not cleanliness, is binding.", "+", "+Two failure modes are diagnosable in advance, without training:", "+* `unmatched target mass`, printed by `curate.py` (0.000 here), is the fraction of target token", "+  mass for which no eligible pool document exists. If it is large the pool cannot express the", "+  target and the quotas will silently redistribute — as happened in an early version whose", "+  per-register acceptance boxes, estimated from as few as 156 documents, starved three of five", "+  registers and drove the emitted mixture to 87% news.", "+* The scorer's held-out AUC is a go/no-go check before any GPU time is spent: below ~0.9 the", "+  importance score is memorising topics rather than registers and the ranking should not be trusted.", "+", "+## Files", "+", "+* `submission/selection.json` — 34,455 pool ids in priority order; the first ~8.2k fill the 12M", "+  budget, the remainder is spare so the pipeline can never run short.", "+* `submission/curate.py` — reproduces `selection.json` exactly from the stated criterion", "+  (`python curate.py`, no flags); `feat.py` / `score.py` are its two cached stages.", "+* `submission/diag_eval.py` — the per-register diagnostic used for mechanism observable (b). It", "+  duplicates the frozen recipe verbatim and is not part of the official run."]}], "originalFile": "# Curating a raw web pool for a fixed 12M-token budget\n\n## Hypothesis\n\nAt a fixed token budget far below the pool size, held-out perplexity on a broad multi-domain\ntarget is governed less by *generic document quality* than by **distributional match to the\ntarget's register mixture**. Specifically:\n\n1. The single most useful supervision available is the dev target itself. `data/multi_dev.npy` is\n   1M GPT-2 tokens *drawn from the target distribution*; decoding it back to text yields ~2.7k\n   in-domain sample documents. A hashed-unigram importance score\n   `s(d) = mean_{w in d} log p_target(w) - log p_pool(w)` (DSIR, Xie et al. 2023) built from those\n   samples ranks pool documents by target-likeness with no labels and no external model.\n2. Ranking by that score alone is **not** enough: greedy top-k concentrates the budget on\n   whichever register is easiest to match and starves the others. Imposing **per-register token\n   quotas equal to the target's own register token mass** (registers found by spherical k-means on\n   the decoded target) should beat pure ranking.\n3. Quality filtering should be a **junk filter, not a quality bar**. Filtering to the *centre* of\n   the target's surface-statistic distribution destroys diversity and should *hurt*, even though\n   every document kept looks more target-like.\n\n## Mechanism — predicted observables (not the final perplexity)\n\n**(a) The scorer must generalise across topics, not memorise them.** Predicted *before* any\ntraining run: fit the importance score on a random 70% of the target documents; held-out target\ndocuments should outrank random pool documents with AUC ≥ 0.9. **Measured: AUC = 0.9517.**\nA discriminative logistic-regression alternative was rejected on this observable, not on\nperplexity: with 2.7k positives and 2^18 hashed features it drove its top weights onto proper\nnouns (`bush`, `grissom`, `wilder`, `boosey`) and flagged only 0.6% of the pool — topic\nmemorisation, not register. Restricting the feature space to buckets frequent in the pool and\nscoring by a smoothed likelihood ratio fixes it.\n\n**(b) The gain from quota-balancing must be *concentrated in the register that greedy ranking\nstarves*, not spread evenly.** This is the discriminating prediction: if balancing merely\nimproved average data quality, all registers would improve together. Measured per-register\nheld-out perplexity (`diag_eval.py`, identical frozen recipe, register assigned per eval window):\n\n| register (k=5) | rank-only | quota-balanced | change |\n|---|---|---|---|\n| 1 · technical Q&A / code | 409.1 | **289.7** | **−29%** |\n| 0 · niche web prose | 850.6 | 783.7 | −8% |\n| 2 · science / geography wiki | 720.0 | 703.0 | −2% |\n| 3 · news (53% of target mass) | 298.0 | 296.2 | −1% |\n| 4 · biography wiki | 762.4 | 766.1 | +0.5% |\n\nConfirmed: essentially the whole improvement is register 1, which greedy ranking under-served,\nwhile the dominant news register is unchanged. Worst-register perplexity falls 850.6 → 783.7 and\nthe max/min spread narrows 2.85 → 2.71. One sub-prediction failed: the *log* spread across\nregisters widened (0.407 → 0.462), because a single starved register improved sharply rather than\nall registers converging — the gain is concentrated, so dispersion is not a reliable summary of it.\n\n**(c) A hard floor exists that no selection can remove.** 863 / 2705 target documents (32%,\nthe encyclopedic portion) carry the WikiText detokenisation artifact ` @-@ `; `grep` finds it in\n**0** of 182,016 pool documents. Predicted consequence: the wiki-formatted registers stay the\nworst-served no matter how the pool is curated, and absolute perplexity stays in the hundreds.\nConfirmed — registers 0/2/4 remain at 703–784 while news reaches 296.\n\n## Falsification\n\nEach claim was given a way to fail, and the third partly did:\n\n| test | prediction | result |\n|---|---|---|\n| quotas vs pure ranking | quotas win | 358.9 vs 394.3 ✓ |\n| acceptance box vs none | box wins | 358.9 vs 381.2 ✓ |\n| **tighter** box (5–95 pct) | *hurts* — diversity, not purity, is scarce | 437.9 vs 358.9 ✓ (predicted sign) |\n| document-length matching | helps (target median 192 tok vs 726 selected) | 359.2 vs 358.9 — **no effect** ✗ |\n| finer registers (K=12 vs 5) | helps | 361.3 vs 358.9 — no effect ✗ |\n| loss-mass-weighted quotas | helps | see table below |\n\nThe hypothesis would have been falsified outright if quota-balancing had not beaten pure ranking,\nor if the per-register breakdown had shown a uniform improvement instead of one concentrated in\nthe starved register. Length-matching is the clearest negative: matching the target's\ndocument-length distribution (mean selected length 1221 → 497 tokens, 9.8k → 24.1k documents in\nthe budget) changed perplexity by +0.3, inside single-seed noise. Whatever the EOS-boundary\nstatistics of the eval stream cost, it is not recoverable by selection.\n\n## Results (frozen recipe, 12M tokens, seed 1337, dev target)\n\n| selection | dev PPL |\n|---|---|\n| random pool sample (do-nothing baseline) | 469.9 |\n| DSIR rank-only, no register quotas | 394.3 |\n| quotas, no acceptance box (length + dedup only) | 381.2 |\n| quotas + box, K=12 registers | 361.3 |\n| quotas + box + document-length matching | 359.2 |\n| **quotas + box, K=5 (submitted configuration)** | **358.9** |\n| quotas + *tight* box | 437.9 |\n\n## Transfer\n\nThe method needs only (i) a raw pool and (ii) a sample of the target distribution — no labels, no\nreference LM, no internet. Cost for 182k documents / 200M tokens: 45 s of CPU for hashed features\n(15 processes) and a few seconds of GPU for scoring; both are linear in pool size, so the same\nrecipe runs on a pool 1000× larger with the same code. It transfers directly to any budgeted\npretraining or domain-adaptive-pretraining problem where a modest in-domain sample exists, and the\nregister-quota step is the part worth carrying over: it is what converts a *similarity* score into\na *mixture* and it is where 90% of the measured gain over rank-only selection comes from.\n\nTwo transfer caveats are diagnosable in advance, without training. The `unmatched target mass`\nfigure printed by `curate.py` (0.004 here) reports the fraction of target token mass for which no\neligible pool document exists — if it is large, the pool cannot express the target and quotas will\nsilently redistribute. And the scorer's held-out AUC is a pre-training go/no-go check: below ~0.9\nthe importance score is memorising topics rather than registers, and the ranking should not be\ntrusted.\n", "userModified": false}

=== final verification: pack the submitted selection with the official packer ===
packed 12000000 tokens from 34455 ids -> /tmp/final_check.npy
packed tokens: 12000000 dtype uint16 max id 50256
ids: 34455 unique: 34455 in range: True
-rw-r--r-- 1 root root 8626 Aug  4 02:12 /workspace/claim.md

/workspace/submission/:
total 270
drwxr-xr-x 1 root root    140 Aug  4 02:12 .
drwxr-xr-x 1 root root    440 Aug  4 02:12 ..
-rw-r--r-- 1 root root  14383 Aug  4 02:11 curate.py
-rw-r--r-- 1 root root   4873 Aug  4 02:11 diag_eval.py
-rw-r--r-- 1 root root   4628 Aug  4 02:11 feat.py
-rw-r--r-- 1 root root   5883 Aug  4 02:11 score.py
-rw-r--r-- 1 root root 244286 Aug  4 02:12 selection.json
1585

[stdout]
=== final verification: pack the submitted selection with the official packer ===
packed 12000000 tokens from 34455 ids -> /tmp/final_check.npy
packed tokens: 12000000 dtype uint16 max id 50256
ids: 34455 unique: 34455 in range: True
-rw-r--r-- 1 root root 8626 Aug  4 02:12 /workspace/claim.md

/workspace/submission/:
total 270
drwxr-xr-x 1 root root    140 Aug  4 02:12 .
drwxr-xr-x 1 root root    440 Aug  4 02:12 ..
-rw-r--r-- 1 root root  14383 Aug  4 02:11 curate.py
-rw-r--r-- 1 root root   4873 Aug  4 02:11 diag_eval.py
-rw-r--r-- 1 root root   4628 Aug  4 02:11 feat.py
-rw-r--r-- 1 root root   5883 Aug  4 02:11 score.py
-rw-r--r-- 1 root root 244286 Aug  4 02:12 selection.json
1585
[metadata] {"noOutputExpected": false}

ok
[1]-  Done                    python curate.py --min_words 40 --out /tmp/sel_H.json > /tmp/bH.log 2>&1
[2]+  Done                    python curate.py --sketch 3 --out /tmp/sel_I.json > /tmp/bI.log 2>&1
duplicates removed: 32736
wrote 37147 ids -> /tmp/sel_H.json  (~36.0M est tokens; first ~9030 ids fill the 12M budget)
duplicates removed: 61441
wrote 36856 ids -> /tmp/sel_I.json  (~36.0M est tokens; first ~9693 ids fill the 12M budget)
1474

[stdout]
ok
[1]-  Done                    python curate.py --min_words 40 --out /tmp/sel_H.json > /tmp/bH.log 2>&1
[2]+  Done                    python curate.py --sketch 3 --out /tmp/sel_I.json > /tmp/bI.log 2>&1
duplicates removed: 32736
wrote 37147 ids -> /tmp/sel_H.json  (~36.0M est tokens; first ~9030 ids fill the 12M budget)
duplicates removed: 61441
wrote 36856 ids -> /tmp/sel_I.json  (~36.0M est tokens; first ~9693 ids fill the 12M budget)
1474
[metadata] {"noOutputExpected": false}

launched
H: (running)
I: (running)
1354

[stdout]
launched
H: (running)
I: (running)
1354
[metadata] {"noOutputExpected": false}

H: HELDOUT_PPL 333.125  loss 5.8085  train_s 118  /tmp/tr_H.npy
I: HELDOUT_PPL 346.374  loss 5.8475  train_s 118  /tmp/tr_I.npy
1234

[stdout]
H: HELDOUT_PPL 333.125  loss 5.8085  train_s 118  /tmp/tr_H.npy
I: HELDOUT_PPL 346.374  loss 5.8475  train_s 118  /tmp/tr_I.npy
1234
[metadata] {"noOutputExpected": false}

ok
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 37147 ids -> /workspace/submission/selection.json  (~36.0M est tokens; first ~9030 ids fill the 12M budget)
submission == evaluated H (333.1): True | ids 37147 unique 37147 range ok True
packed 12000000 tokens from 37147 ids -> /tmp/fc2.npy
1114

[stdout]
ok
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 37147 ids -> /workspace/submission/selection.json  (~36.0M est tokens; first ~9030 ids fill the 12M budget)
submission == evaluated H (333.1): True | ids 37147 unique 37147 range ok True
packed 12000000 tokens from 37147 ids -> /tmp/fc2.npy
1114
[metadata] {"noOutputExpected": false}

updated
3:**Result: dev perplexity 333.1 vs 469.9 for a random selection of the same pool (−29.1%),
68:perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.
81:| length floor 80 words instead of 40 | — | 336.1 | 40 better by 3.0 |
95:Lowering the *length floor* from 80 to 40 words does help slightly (336.1 → 333.1, replicated
100:(333.1 vs 339.3) but *hurt* under the tighter one (358.9 vs 348.8) — the two are substitutes,
115:perplexity here (437.9 at the tight box vs 333.1), because the budget is small enough that
129:* `submission/selection.json` — 37,147 pool ids in priority order; the first ~9.0k fill the 12M

[stdout]
updated
3:**Result: dev perplexity 333.1 vs 469.9 for a random selection of the same pool (−29.1%),
68:perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.
81:| length floor 80 words instead of 40 | — | 336.1 | 40 better by 3.0 |
95:Lowering the *length floor* from 80 to 40 words does help slightly (336.1 → 333.1, replicated
100:(333.1 vs 339.3) but *hurt* under the tighter one (358.9 vs 348.8) — the two are substitutes,
115:perplexity here (437.9 at the tight box vs 333.1), because the budget is small enough that
129:* `submission/selection.json` — 37,147 pool ids in priority order; the first ~9.0k fill the 12M
[metadata] {"noOutputExpected": false}

The file /workspace/claim.md has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/claim.md", "oldString": "perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.", "newString": "perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 333.1.", "originalFile": "# Curating a raw web pool for a fixed 12M-token budget\n\n**Result: dev perplexity 333.1 vs 469.9 for a random selection of the same pool (−29.1%),\nusing the frozen recipe at 12M tokens, seed 1337.**\n\n## Hypothesis\n\nAt a budget of ~6% of the pool, held-out perplexity on a broad multi-domain target is governed\nless by *generic document quality* than by **distributional match to the target's register\nmixture**, and the binding constraint is diversity rather than purity. Three sub-claims:\n\n1. **The target is its own best supervision.** `data/multi_dev.npy` is 1M GPT-2 tokens *drawn from\n   the target distribution*; decoding it back to text yields ~2.7k in-domain sample documents. A\n   hashed-unigram importance score `s(d) = mean_{w∈d} log p_target(w) − log p_pool(w)`\n   (DSIR, Xie et al. 2023) built from those samples should rank pool documents usefully with no\n   labels, no reference LM and no internet.\n2. **Ranking alone is not enough.** Greedy top-k concentrates the budget on whichever register is\n   easiest to match and starves the others. Imposing **per-register token quotas equal to the\n   target's own register token mass** (registers from spherical k-means on the decoded target)\n   should beat pure ranking.\n3. **Quality filtering must be a junk filter, not a quality bar.** Filtering to the *centre* of the\n   target's surface-statistic distribution should *hurt*, even though every surviving document\n   looks more target-like, because at 6% of the pool diversity is the scarce resource.\n\n## Mechanism — predicted observables other than the final perplexity\n\n**(a) The scorer must generalise across topics, not memorise them.** Predicted *before* any\ntraining run: fit the importance score on a random 70% of the target documents; held-out target\ndocuments should outrank random pool documents with AUC ≥ 0.9. **Measured AUC = 0.9517.**\nThis observable is what rejected the first attempt: a discriminative logistic regression on the\nsame features (2.7k positives, 2^18 hashed dims) put its largest weights on proper nouns —\n`bush`, `grissom`, `wilder`, `boosey`, `shackleton` — and scored only 0.6% of the pool positive.\nThat is topic memorisation, not register, and it would not transfer to the hidden sample.\nRestricting the feature space to the 40k buckets frequent in the pool and switching to a smoothed\nlikelihood ratio fixed it. *Diagnosis and repair happened entirely on this observable.*\n\n**(b) The gain from quota-balancing must be concentrated in the register that greedy ranking\nstarves, not spread evenly.** This is the discriminating prediction — if balancing merely raised\naverage data quality, all registers would improve together. Per-register held-out perplexity\n(`diag_eval.py`; identical frozen recipe, each eval window assigned to the register of its\nmajority document):\n\n| register (k=5) | rank-only | quota-balanced | change |\n|---|---|---|---|\n| 1 · technical Q&A / code | 409.1 | **289.7** | **−29%** |\n| 0 · niche web prose | 850.6 | 783.7 | −8% |\n| 2 · science / geography wiki | 720.0 | 703.0 | −2% |\n| 3 · news (53% of target mass) | 298.0 | 296.2 | −1% |\n| 4 · biography wiki | 762.4 | 766.1 | +0.5% |\n\n**Confirmed:** essentially the entire improvement is register 1, which greedy ranking under-served,\nwhile the dominant news register is untouched. Worst-register perplexity falls 850.6 → 783.7 and\nthe max/min spread narrows 2.85 → 2.71. One sub-prediction **failed**: the *log* spread across\nregisters widened (0.407 → 0.462), because one starved register improved sharply rather than all\nregisters converging. Dispersion is therefore not a valid summary of this mechanism; concentration\nof the gain is.\n\n**(c) A floor exists that no selection can remove.** 863 / 2705 target documents (32%, the\nencyclopedic portion) carry the WikiText detokenisation artifact ` @-@ `; `grep` finds it in\n**0 of 182,016** pool documents. Predicted consequence: the wiki-formatted registers stay the\nworst-served however the pool is curated, and absolute perplexity stays in the hundreds.\nConfirmed — registers 0/2/4 remain at 703–784 while news reaches 296. Any curation effort spent\nchasing the encyclopedic register is capped by pool coverage, not by the selection algorithm.\n\n## Falsification\n\nEvery component was given a way to fail, and two predictions did fail. All numbers are dev\nperplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.\n\n| test | prediction | measured | verdict |\n|---|---|---|---|\n| random pool sample (do-nothing) | — | 469.9 | reference |\n| DSIR ranking, no register quotas | quotas win | 394.3 | ✓ |\n| no acceptance box (length + dedup only) | box helps | 381.2 | ✓ |\n| **random order within the same quotas** | **ranking carries real signal** | **378.4** | **✓ (+42)** |\n| *tighter* box (5–95 pct, pad .05) | *hurts* — diversity is scarce | 437.9 | ✓ (sign predicted) |\n| box loosened past the optimum (0.5–99.5, pad .6) | hurts | 362.2 | ✓ |\n| box pad 0.45 vs 0.30 | — | 347.5 | 0.30 optimal |\n| exact-dedup only (keep 21k near-dupes) | near-dedup helps | 343.7 | ✓ |\n| *stronger* near-dedup (3-word sketch, 61k removed) | — | 346.4 | 4-word sketch optimal |\n| length floor 80 words instead of 40 | — | 336.1 | 40 better by 3.0 |\n| document-length matching (median 726→283 tok) | helps | 343.1 / 359.2 | ✗ **no effect** |\n| finer registers (K=12 vs K=5) | helps | 361.3 | ✗ no effect |\n| loss-mass-weighted quotas (∝ mass × ppl) | helps | 358.0 | ✗ no effect |\n\nThe hypothesis would have been falsified outright if quota-balancing had not beaten pure ranking,\nif random ordering within quotas had matched the DSIR ordering (it loses 42 perplexity, so the\nranking is load-bearing), or if the per-register breakdown had shown uniform rather than\nconcentrated improvement.\n\nTwo clean negatives worth recording. **Length matching does nothing:** the target's documents\nhave a median of 192 GPT-2 tokens against 726 for a score-ranked selection, and forcing the\nselection to reproduce the target's length histogram (1221 → 497 mean tokens, 9.8k → 24.1k\ndocuments inside the budget) moved perplexity by +0.3 at the tight box and +10 at the loose box.\nLowering the *length floor* from 80 to 40 words does help slightly (336.1 → 333.1, replicated\nindependently under exact-dedup at 343.7 → 339.3), but that is a diversity effect — more\ndocuments per budget — not a distribution-matching one: matching the full histogram still loses.\nWhatever the EOS-boundary statistics of the eval stream cost, selection cannot recover it.\n**Filtering and dedup interact:** near-duplicate removal *helps* under the loose box\n(333.1 vs 339.3) but *hurt* under the tighter one (358.9 vs 348.8) — the two are substitutes,\nboth spending the same diversity budget, so their strengths cannot be tuned independently.\n\n## Transfer\n\nThe method needs only (i) a raw pool and (ii) a sample of the target distribution — no labels, no\nreference LM, no internet. Cost for 182k documents / 200M tokens: **45 s of CPU** for hashed\nfeatures (15 processes) plus a few seconds of GPU for scoring, both linear in pool size, so the\nsame code runs on a pool 1000× larger. It transfers directly to budgeted pretraining and\ndomain-adaptive pretraining wherever a modest in-domain sample exists.\n\nThe part worth carrying over is the register-quota step: it is what converts a *similarity score*\ninto a *mixture*, and it delivers the largest single gain over rank-only selection. The\ntransferable warning is the opposite of the usual instinct — **do not tighten the quality filter**.\nEvery step toward a purer, more target-like selection past the junk-filter threshold cost\nperplexity here (437.9 at the tight box vs 333.1), because the budget is small enough that\ncoverage, not cleanliness, is binding.\n\nTwo failure modes are diagnosable in advance, without training:\n* `unmatched target mass`, printed by `curate.py` (0.000 here), is the fraction of target token\n  mass for which no eligible pool document exists. If it is large the pool cannot express the\n  target and the quotas will silently redistribute — as happened in an early version whose\n  per-register acceptance boxes, estimated from as few as 156 documents, starved three of five\n  registers and drove the emitted mixture to 87% news.\n* The scorer's held-out AUC is a go/no-go check before any GPU time is spent: below ~0.9 the\n  importance score is memorising topics rather than registers and the ranking should not be trusted.\n\n## Files\n\n* `submission/selection.json` — 37,147 pool ids in priority order; the first ~9.0k fill the 12M\n  budget, the remainder is spare so the pipeline can never run short.\n* `submission/curate.py` — reproduces `selection.json` exactly from the stated criterion\n  (`python curate.py`, no flags); `feat.py` / `score.py` are its two cached stages.\n* `submission/diag_eval.py` — the per-register diagnostic used for mechanism observable (b). It\n  duplicates the frozen recipe verbatim and is not part of the official run.\n", "structuredPatch": [{"oldStart": 65, "oldLines": 7, "newStart": 65, "newLines": 7, "lines": [" ## Falsification", " ", " Every component was given a way to fail, and two predictions did fail. All numbers are dev", "-perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 336.1.", "+perplexity under the frozen recipe, 12M tokens, seed 1337; the submitted configuration is 333.1.", " ", " | test | prediction | measured | verdict |", " |---|---|---|---|"]}], "userModified": false, "replaceAll": false}

=== deliverables ===
-rw-r--r-- 1 root root 9081 Aug  4 02:21 claim.md

submission/:
total 288
drwxr-xr-x 1 root root    140 Aug  4 02:21 .
drwxr-xr-x 1 root root    460 Aug  4 02:21 ..
-rw-r--r-- 1 root root  14568 Aug  4 02:19 curate.py
-rw-r--r-- 1 root root   4873 Aug  4 02:11 diag_eval.py
-rw-r--r-- 1 root root   4628 Aug  4 02:11 feat.py
-rw-r--r-- 1 root root   5883 Aug  4 02:11 score.py
-rw-r--r-- 1 root root 263449 Aug  4 02:20 selection.json

=== reproducibility: submission/curate.py with no flags ===
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 37147 ids -> /tmp/repro.json  (~36.0M est tokens; first ~9030 ids fill the 12M budget)
selection.json reproduced exactly: True
ids 37147 | unique 37147 | in range True

=== all runs (dev PPL, frozen recipe, 12M tokens, seed 1337) ===
H           333.1
loose       336.1
C           339.3
E           343.1
A           343.7
I           346.4
D           347.5
nodd        348.8
balanced    355.6
lossw       358.0
bal2        358.9
lm          359.2
bal12       361.3
B           362.2
G           378.4
nogate      381.2
global      394.3
lmt         436.7
tight       437.9
random      469.9
1044

[stdout]
=== deliverables ===
-rw-r--r-- 1 root root 9081 Aug  4 02:21 claim.md

submission/:
total 288
drwxr-xr-x 1 root root    140 Aug  4 02:21 .
drwxr-xr-x 1 root root    460 Aug  4 02:21 ..
-rw-r--r-- 1 root root  14568 Aug  4 02:19 curate.py
-rw-r--r-- 1 root root   4873 Aug  4 02:11 diag_eval.py
-rw-r--r-- 1 root root   4628 Aug  4 02:11 feat.py
-rw-r--r-- 1 root root   5883 Aug  4 02:11 score.py
-rw-r--r-- 1 root root 263449 Aug  4 02:20 selection.json

=== reproducibility: submission/curate.py with no flags ===
emitted register mix: [0.016 0.258 0.064 0.533 0.129] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 37147 ids -> /tmp/repro.json  (~36.0M est tokens; first ~9030 ids fill the 12M budget)
selection.json reproduced exactly: True
ids 37147 | unique 37147 | in range True

=== all runs (dev PPL, frozen recipe, 12M tokens, seed 1337) ===
H           333.1
loose       336.1
C           339.3
E           343.1
A           343.7
I           346.4
D           347.5
nodd        348.8
balanced    355.6
lossw       358.0
bal2        358.9
lm          359.2
bal12       361.3
B           362.2
G           378.4
nogate      381.2
global      394.3
lmt         436.7
tight       437.9
random      469.9
1044
[metadata] {"noOutputExpected": false}