claude-opus-4-8

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "K = 1 << 20            # hashed feature buckets", "newString": "K = 1 << 22            # hashed feature buckets (large -> few collisions, low noise)", "originalFile": "\"\"\"Curate the raw web pool toward a broad multi-domain high-quality English target.\n\nCriterion (DSIR-style importance selection, Xie et al. 2023):\n  Rank each pool document by how much its hashed word n-gram distribution looks\n  like the disclosed high-quality TARGET (Wikipedia + general web prose + news +\n  technical Q&A) relative to the raw pool BACKGROUND. Concretely, for hashed\n  unigram+bigram features we estimate a target distribution p_t and a background\n  distribution p_b, and score a document by the mean per-feature log-likelihood\n  ratio  mean_ngram log(p_t / p_b).  High score == reads like the target domain.\n\nThe target distribution is estimated from the provided dev sample of the target\ndomain (data/multi_dev.npy, GPT-2 tokens) which we decode back to text. We select\ndocuments in descending score order (a min-length gate removes noise), emitting\nenough ids to comfortably exceed the 12M-token training budget.\n\nNothing here is hand-picked: the output is a pure function of the stated score.\n\"\"\"\nimport json, re, math, sys, zlib\nimport numpy as np\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nK = 1 << 20            # hashed feature buckets\nMIN_WORDS = 50         # ignore very short docs (noisy scores, little value)\nN_EMIT = 60000         # ids to emit (>> enough to fill 12M tokens)\n\n_split = re.compile(r\"[^a-z0-9]+\")\n\n# Common English function words: fluent prose is ~30-50% of these; junk\n# (number/name lists, code dumps, tag spam) is near 0%.\nSTOP = set((\"the of and to a in that is was he for it with as his on be at by i this \"\n            \"had not are but from or have an they which one you were her all she there \"\n            \"would their we him been has when who will more no if out so said what up its \"\n            \"about into than them can only other new some could time these two may then do \"\n            \"first any my now such like our over man me even most made after also did many \"\n            \"before must through back years where much your way well down should because \"\n            \"each just those people how too little state good very make world still see own \"\n            \"work men day get here between both under never same another know while last\").split())\n\ndef words(text):\n    return [w for w in _split.split(text.lower()) if w]\n\ndef is_prose(ws):\n    \"\"\"Hard quality gate: keep only well-formed English prose.\"\"\"\n    n = len(ws)\n    if n < MIN_WORDS:\n        return False\n    stop = sum(1 for w in ws if w in STOP) / n\n    digit = sum(1 for w in ws if w.isdigit()) / n\n    uniq = len(set(ws)) / n\n    mwl = sum(len(w) for w in ws) / n\n    return (stop >= 0.20 and digit <= 0.15 and uniq >= 0.35 and 3.0 <= mwl <= 9.0)\n\ndef _h(s):\n    # deterministic hash (crc32) so re-runs are bit-identical, unlike salted hash()\n    return zlib.crc32(s.encode(\"utf-8\")) & (K - 1)\n\ndef ngram_buckets(ws):\n    # unigram + bigram hashed features\n    b = [_h(w) for w in ws]\n    b.extend(_h(ws[i] + \" \" + ws[i + 1]) for i in range(len(ws) - 1))\n    return b\n\ndef clean_target(t):\n    # de-tokenize WikiText artifacts so features reflect content, not formatting\n    t = t.replace(\" @-@ \", \"-\").replace(\" @,@ \", \",\").replace(\" @.@ \", \".\")\n    t = t.replace(\"<|endoftext|>\", \" \")\n    return t\n\ndef main():\n    # ---- target counts (from decoded dev sample) ----\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64).tolist()\n    tgt_text = clean_target(tok.decode(dev))\n    tgt = np.zeros(K, dtype=np.float64)\n    for b in ngram_buckets(words(tgt_text)):\n        tgt[b] += 1.0\n    print(f\"target features: {int(tgt.sum())}\", file=sys.stderr)\n\n    # ---- background counts (full pool, pass 1) + cache words for pass 2 ----\n    bg = np.zeros(K, dtype=np.float64)\n    ids, docs = [], []\n    for line in open(POOL):\n        r = json.loads(line)\n        ws = words(r[\"text\"])\n        ids.append(r[\"id\"]); docs.append(ws)\n        for b in ngram_buckets(ws):\n            bg[b] += 1.0\n    print(f\"pool docs: {len(ids)}  bg features: {int(bg.sum())}\", file=sys.stderr)\n\n    # ---- log-likelihood ratio per feature ----\n    a = 1.0\n    pt = (tgt + a) / (tgt.sum() + a * K)\n    pb = (bg + a) / (bg.sum() + a * K)\n    logratio = np.log(pt) - np.log(pb)\n\n    # ---- score each doc: mean per-feature log-ratio (pass 2) ----\n    scores = np.full(len(ids), -1e9, dtype=np.float64)\n    kept = 0\n    for j, ws in enumerate(docs):\n        if not is_prose(ws):\n            continue\n        kept += 1\n        bs = ngram_buckets(ws)\n        scores[j] = logratio[np.asarray(bs)].mean()\n    print(f\"prose docs kept: {kept}\", file=sys.stderr)\n\n    # descending score, skipping near-duplicates (same normalized 120-char prefix)\n    order = np.argsort(-scores)\n    sel, seen = [], set()\n    for j in order:\n        if scores[j] <= -1e8:\n            break\n        sig = \"\".join(docs[j])[:120]\n        if sig in seen:\n            continue\n        seen.add(sig)\n        sel.append(int(ids[j]))\n        if len(sel) >= N_EMIT:\n            break\n    json.dump(sel, open(OUT, \"w\"))\n    print(f\"wrote {len(sel)} ids -> {OUT}\", file=sys.stderr)\n    print(f\"score range: {scores[order[0]]:.4f} .. {scores[order[min(len(sel),N_EMIT)-1]]:.4f}\", file=sys.stderr)\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 22, "oldLines": 7, "newStart": 22, "newLines": 7, "lines": [" DEV  = \"/workspace/data/multi_dev.npy\"", " OUT  = \"/workspace/submission/selection.json\"", " ", "-K = 1 << 20            # hashed feature buckets", "+K = 1 << 22            # hashed feature buckets (large -> few collisions, low noise)", " MIN_WORDS = 50         # ignore very short docs (noisy scores, little value)", " N_EMIT = 60000         # ids to emit (>> enough to fill 12M tokens)", " "]}], "userModified": false, "replaceAll": false}

Command running in background with ID: bnpyiskvn. Output is being written to: /tmp/claude-0/-workspace/53cd7a88-b4ee-48d8-b093-e6ea28b06723/tasks/bnpyiskvn.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bnpyiskvn", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

(Bash completed with no output)

[metadata] {"noOutputExpected": false}

pool docs: 182016  bg features: 256545972
prose docs kept: 137633
wrote 60000 ids -> /workspace/submission/selection.json
score range: 0.2356 .. -0.8914

[stdout]
pool docs: 182016  bg features: 256545972
prose docs kept: 137633
wrote 60000 ids -> /workspace/submission/selection.json
score range: 0.2356 .. -0.8914
[metadata] {"noOutputExpected": false}

Command running in background with ID: b0ijwbr0q. Output is being written to: /tmp/claude-0/-workspace/53cd7a88-b4ee-48d8-b093-e6ea28b06723/tasks/b0ijwbr0q.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "b0ijwbr0q", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

HELDOUT_PPL 344.925  loss 5.8433  train_s 54  /tmp/dev_train.npy
time:
3057

[stdout]
HELDOUT_PPL 344.925  loss 5.8433  train_s 54  /tmp/dev_train.npy
time:
3057
[metadata] {"noOutputExpected": false}

The file /workspace/claim.md has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/claim.md", "oldString": "- The gated+deduped DSIR selection does **not** beat a random selection on\n  held-out dev perplexity. (Observed: **349 vs. 458** dev PPL — a 24% reduction.)", "newString": "- The gated+deduped DSIR selection does **not** beat a random selection on\n  held-out dev perplexity. (Observed: **345 vs. 458** dev PPL — a 25% reduction.)", "originalFile": "# Curation claim\n\n## Hypothesis\nUnder a fixed 12M-token budget (~6% of the pool), held-out perplexity on the\nbroad multi-domain HQ-English target is minimized not by training on random web\ntext but by selecting the pool documents whose word n-gram distribution most\nresembles the target domain **and** that are well-formed English prose. A\nDSIR-style importance score (mean per-feature log-likelihood ratio of hashed\nunigram+bigram features, target vs. pool background) combined with a hard prose\nquality gate and near-duplicate removal selects such documents. Prediction:\nthis selection trains a model with substantially lower held-out perplexity than\na random selection of the same budget.\n\n## Mechanism (observable other than final perplexity)\nThe pool is ~24% non-prose (number tables, name/tag lists, boilerplate) that a\nmean-log-ratio-only score actually *ranks at the very top*, because a degenerate\ndocument that repeats a handful of moderately-target-like tokens attains a near-\nperfect mean. Two directly checkable observables follow, both confirmed here:\n\n1. **Gate content.** Applying the prose gate (function-word ratio ≥ 0.20, digit\n   ratio ≤ 0.15, type-token ratio ≥ 0.35, sane mean word length) removes\n   **23.9%** of a random pool sample — exactly the degenerate documents that\n   otherwise top the ranking (observed: prime-number lists, plant-name\n   galleries, hashtag spam were the top-scored docs *before* gating).\n2. **Distributional shift toward the target.** The selected training stream has a\n   mean English function-word ratio of **0.408**, versus **0.356** for a random\n   pool sample — i.e. the curation demonstrably moves the training distribution\n   toward fluent HQ prose (the target register) *before* any model is trained.\n\nIf the mechanism is right, these shifts are the cause of the perplexity drop; the\nperplexity number is downstream of them.\n\n## Falsification\nThe claim is false if any of:\n- The gated+deduped DSIR selection does **not** beat a random selection on\n  held-out dev perplexity. (Observed: **349 vs. 458** dev PPL — a 24% reduction.)\n- Removing the prose gate (mean-log-ratio ranking alone) does **not** degrade the\n  selection into non-prose junk at the top. (Observed: it does — top docs become\n  prime-number/name lists.)\n- Removing near-duplicates does **not** help. (Observed: dedup improved 360→349.)\n- Favoring length instead of per-token target-likeness helps. (Observed: a\n  length-blended score was *worse*, 353 vs. 349 — quality-per-token, not raw\n  length, is what matters.)\n\n## Transfer\nThe method is target-agnostic and label-free: it only needs (a) a sample of the\ndesired target domain to estimate the target n-gram distribution and (b) the raw\npool for the background. Swap the target sample and the same `curate.py` selects\nfor a different domain (e.g. purely encyclopedic, or code) with no retraining of\nany scorer. It scales to larger pools/budgets (two linear passes, hashed\nfeatures, O(1) memory per feature) and composes with stronger downstream quality\nsignals (a learned classifier, KenLM perplexity) that could replace the linear\nlog-ratio score while keeping the prose gate and dedup unchanged.\n```\nReference numbers (frozen 30M GPT, 12M-token budget, dev target multi_dev.npy):\n  random baseline ......................... 458 PPL\n  DSIR mean + prose gate .................. 360 PPL\n  + near-duplicate removal (final) ........ 349 PPL   <- submitted selection\n  length-blended variant .................. 353 PPL   (rejected)\n```\n", "structuredPatch": [{"oldStart": 33, "oldLines": 7, "newStart": 33, "newLines": 7, "lines": [" ## Falsification", " The claim is false if any of:", " - The gated+deduped DSIR selection does **not** beat a random selection on", "-  held-out dev perplexity. (Observed: **349 vs. 458** dev PPL — a 24% reduction.)", "+  held-out dev perplexity. (Observed: **345 vs. 458** dev PPL — a 25% reduction.)", " - Removing the prose gate (mean-log-ratio ranking alone) does **not** degrade the", "   selection into non-prose junk at the top. (Observed: it does — top docs become", "   prime-number/name lists.)"]}], "userModified": false, "replaceAll": false}

The file /workspace/claim.md has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/claim.md", "oldString": "Reference numbers (frozen 30M GPT, 12M-token budget, dev target multi_dev.npy):\n  random baseline ......................... 458 PPL\n  DSIR mean + prose gate .................. 360 PPL\n  + near-duplicate removal (final) ........ 349 PPL   <- submitted selection\n  length-blended variant .................. 353 PPL   (rejected)", "newString": "Reference numbers (frozen 30M GPT, 12M-token budget, dev target multi_dev.npy):\n  random baseline ............................ 458 PPL\n  DSIR mean + prose gate ..................... 360 PPL\n  + near-duplicate removal ................... 349 PPL\n  + deterministic crc32 hash, K=2^22 (final) . 345 PPL   <- submitted selection\n  length-blended variant ..................... 353 PPL   (rejected)\nReproducibility: curate.py uses crc32 feature hashing (not salted hash()), so a\nfresh run reproduces submission/selection.json bit-for-bit.", "originalFile": "# Curation claim\n\n## Hypothesis\nUnder a fixed 12M-token budget (~6% of the pool), held-out perplexity on the\nbroad multi-domain HQ-English target is minimized not by training on random web\ntext but by selecting the pool documents whose word n-gram distribution most\nresembles the target domain **and** that are well-formed English prose. A\nDSIR-style importance score (mean per-feature log-likelihood ratio of hashed\nunigram+bigram features, target vs. pool background) combined with a hard prose\nquality gate and near-duplicate removal selects such documents. Prediction:\nthis selection trains a model with substantially lower held-out perplexity than\na random selection of the same budget.\n\n## Mechanism (observable other than final perplexity)\nThe pool is ~24% non-prose (number tables, name/tag lists, boilerplate) that a\nmean-log-ratio-only score actually *ranks at the very top*, because a degenerate\ndocument that repeats a handful of moderately-target-like tokens attains a near-\nperfect mean. Two directly checkable observables follow, both confirmed here:\n\n1. **Gate content.** Applying the prose gate (function-word ratio ≥ 0.20, digit\n   ratio ≤ 0.15, type-token ratio ≥ 0.35, sane mean word length) removes\n   **23.9%** of a random pool sample — exactly the degenerate documents that\n   otherwise top the ranking (observed: prime-number lists, plant-name\n   galleries, hashtag spam were the top-scored docs *before* gating).\n2. **Distributional shift toward the target.** The selected training stream has a\n   mean English function-word ratio of **0.408**, versus **0.356** for a random\n   pool sample — i.e. the curation demonstrably moves the training distribution\n   toward fluent HQ prose (the target register) *before* any model is trained.\n\nIf the mechanism is right, these shifts are the cause of the perplexity drop; the\nperplexity number is downstream of them.\n\n## Falsification\nThe claim is false if any of:\n- The gated+deduped DSIR selection does **not** beat a random selection on\n  held-out dev perplexity. (Observed: **345 vs. 458** dev PPL — a 25% reduction.)\n- Removing the prose gate (mean-log-ratio ranking alone) does **not** degrade the\n  selection into non-prose junk at the top. (Observed: it does — top docs become\n  prime-number/name lists.)\n- Removing near-duplicates does **not** help. (Observed: dedup improved 360→349.)\n- Favoring length instead of per-token target-likeness helps. (Observed: a\n  length-blended score was *worse*, 353 vs. 349 — quality-per-token, not raw\n  length, is what matters.)\n\n## Transfer\nThe method is target-agnostic and label-free: it only needs (a) a sample of the\ndesired target domain to estimate the target n-gram distribution and (b) the raw\npool for the background. Swap the target sample and the same `curate.py` selects\nfor a different domain (e.g. purely encyclopedic, or code) with no retraining of\nany scorer. It scales to larger pools/budgets (two linear passes, hashed\nfeatures, O(1) memory per feature) and composes with stronger downstream quality\nsignals (a learned classifier, KenLM perplexity) that could replace the linear\nlog-ratio score while keeping the prose gate and dedup unchanged.\n```\nReference numbers (frozen 30M GPT, 12M-token budget, dev target multi_dev.npy):\n  random baseline ......................... 458 PPL\n  DSIR mean + prose gate .................. 360 PPL\n  + near-duplicate removal (final) ........ 349 PPL   <- submitted selection\n  length-blended variant .................. 353 PPL   (rejected)\n```\n", "structuredPatch": [{"oldStart": 53, "oldLines": 8, "newStart": 53, "newLines": 11, "lines": [" log-ratio score while keeping the prose gate and dedup unchanged.", " ```", " Reference numbers (frozen 30M GPT, 12M-token budget, dev target multi_dev.npy):", "-  random baseline ......................... 458 PPL", "-  DSIR mean + prose gate .................. 360 PPL", "-  + near-duplicate removal (final) ........ 349 PPL   <- submitted selection", "-  length-blended variant .................. 353 PPL   (rejected)", "+  random baseline ............................ 458 PPL", "+  DSIR mean + prose gate ..................... 360 PPL", "+  + near-duplicate removal ................... 349 PPL", "+  + deterministic crc32 hash, K=2^22 (final) . 345 PPL   <- submitted selection", "+  length-blended variant ..................... 353 PPL   (rejected)", "+Reproducibility: curate.py uses crc32 feature hashing (not salted hash()), so a", "+fresh run reproduces submission/selection.json bit-for-bit.", " ```"]}], "userModified": false, "replaceAll": false}

selection.json OK: 60000 unique ids, all in-range
matches K22 eval selection: True
--- files ---
-rw-r--r-- 1 root root 3735 Aug  4 04:02 claim.md

submission/:
total 423
drwxr-xr-x 1 root root    240 Aug  4 03:56 .
drwxr-xr-x 1 root root    220 Aug  4 04:02 ..
drwxr-xr-x 1 root root     80 Aug  4 03:43 __pycache__
-rw-r--r-- 1 root root   5391 Aug  4 03:56 curate.py
-rw-r--r-- 1 root root 427451 Aug  4 03:59 selection.json
--- curate head ---
25:K = 1 << 22            # hashed feature buckets (large -> few collisions, low noise)
26:MIN_WORDS = 50         # ignore very short docs (noisy scores, little value)
45:def is_prose(ws):
48:    if n < MIN_WORDS:
57:    # deterministic hash (crc32) so re-runs are bit-identical, unlike salted hash()
58:    return zlib.crc32(s.encode("utf-8")) & (K - 1)
104:        if not is_prose(ws):
108:        scores[j] = logratio[np.asarray(bs)].mean()
111:    # descending score, skipping near-duplicates (same normalized 120-char prefix)

[stdout]
selection.json OK: 60000 unique ids, all in-range
matches K22 eval selection: True
--- files ---
-rw-r--r-- 1 root root 3735 Aug  4 04:02 claim.md

submission/:
total 423
drwxr-xr-x 1 root root    240 Aug  4 03:56 .
drwxr-xr-x 1 root root    220 Aug  4 04:02 ..
drwxr-xr-x 1 root root     80 Aug  4 03:43 __pycache__
-rw-r--r-- 1 root root   5391 Aug  4 03:56 curate.py
-rw-r--r-- 1 root root 427451 Aug  4 03:59 selection.json
--- curate head ---
25:K = 1 << 22            # hashed feature buckets (large -> few collisions, low noise)
26:MIN_WORDS = 50         # ignore very short docs (noisy scores, little value)
45:def is_prose(ws):
48:    if n < MIN_WORDS:
57:    # deterministic hash (crc32) so re-runs are bit-identical, unlike salted hash()
58:    return zlib.crc32(s.encode("utf-8")) & (K - 1)
104:        if not is_prose(ws):
108:        scores[j] = logratio[np.asarray(bs)].mean()
111:    # descending score, skipping near-duplicates (same normalized 120-char prefix)
[metadata] {"noOutputExpected": false}