claude-opus-4-8

File created successfully at: /tmp/build_sel.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/tmp/build_sel.py", "content": "\"\"\"Build a selection variant from the cached per-doc features (no re-scoring).\nUsage: build_sel.py STRATEGY TEMP OUT\n  STRATEGY: tilt   -> code lane + Gumbel-top-k over gated docs with temperature TEMP\n                      (TEMP->0 = strict top-k score; TEMP large = uniform/diverse)\n\"\"\"\nimport json, sys, numpy as np\nc=np.load(\"/tmp/pool_feats.npz\")\nids,sc,ntok,gate,tag=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"]\nstrat=sys.argv[1]; T=float(sys.argv[2]); OUT=sys.argv[3]\nCODE_LANE = len(sys.argv)<5 or sys.argv[4]!=\"nocode\"\nBUDGET=12_000_000; OVER=2.5\nrng=np.random.default_rng(1337)\n\ncode_ids=[]\nif CODE_LANE:\n    m=(tag==1)&(sc>-50)\n    co=np.argsort(-np.where(m,sc,-1e9))\n    code_ids=[int(ids[j]) for j in co if m[j]]\n\ng=np.where(gate==1)[0]\nif strat==\"tilt\":\n    gum=rng.gumbel(size=len(g))\n    key=sc[g]/max(T,1e-6)+gum\n    gorder=g[np.argsort(-key)]\nelse:\n    raise SystemExit(\"unknown\")\n\nseen=set(code_ids); row={int(ids[k]):k for k in range(len(ids))}\nsel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)\nfor j in gorder:\n    i=int(ids[j])\n    if i in seen: continue\n    seen.add(i); sel.append(i); tot+=int(ntok[j])\n    if tot>=OVER*BUDGET: break\njson.dump(sel,open(OUT,\"w\"))\nprint(f\"{strat} T={T} code={len(code_ids)}: {len(sel)} ids ~{tot/1e6:.1f}M est tok -> {OUT}\")\n", "structuredPatch": [], "originalFile": null, "userModified": false}

tilt T=1000000000.0 code=192: 33441 ids ~30.0M est tok -> /tmp/sel_uniform.json
launched gated-uniform eval pid 2333
remaining: 3898

[stdout]
tilt T=1000000000.0 code=192: 33441 ids ~30.0M est tok -> /tmp/sel_uniform.json
launched gated-uniform eval pid 2333
remaining: 3898
[metadata] {"noOutputExpected": false}

tilt T=0.3 code=192: 32920 ids ~30.0M est tok -> /tmp/sel_t030.json
tilt T=0.1 code=192: 32503 ids ~30.0M est tok -> /tmp/sel_t010.json
remaining: 3888

[stdout]
tilt T=0.3 code=192: 32920 ids ~30.0M est tok -> /tmp/sel_t030.json
tilt T=0.1 code=192: 32503 ids ~30.0M est tok -> /tmp/sel_t010.json
remaining: 3888
[metadata] {"noOutputExpected": false}

=== GATED-UNIFORM result ===
HELDOUT_PPL 366.504  loss 5.9040  train_s 56  /tmp/dev_train.npy
(random=457.2, v2 top-k=538.6)
remaining: 3768

[stdout]
=== GATED-UNIFORM result ===
HELDOUT_PPL 366.504  loss 5.9040  train_s 56  /tmp/dev_train.npy
(random=457.2, v2 top-k=538.6)
remaining: 3768
[metadata] {"noOutputExpected": false}

The file /tmp/build_sel.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/tmp/build_sel.py", "oldString": "c=np.load(\"/tmp/pool_feats.npz\")\nids,sc,ntok,gate,tag=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"]\nstrat=sys.argv[1]; T=float(sys.argv[2]); OUT=sys.argv[3]", "newString": "c=np.load(\"/tmp/pool_feats.npz\")\nids,sc,ntok,gate,tag,lpar,lrar=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"],c[\"lpar\"],c[\"lrar\"]\nstrat=sys.argv[1]; T=float(sys.argv[2]); OUT=sys.argv[3]\n# strat: \"tilt\"=combined sc | \"lr\"=target-vs-pool log-ratio (DSIR) | \"lp\"=target-LM logprob\nSIGNAL={\"tilt\":sc,\"lr\":lrar,\"lp\":lpar}.get(strat,sc)", "originalFile": "\"\"\"Build a selection variant from the cached per-doc features (no re-scoring).\nUsage: build_sel.py STRATEGY TEMP OUT\n  STRATEGY: tilt   -> code lane + Gumbel-top-k over gated docs with temperature TEMP\n                      (TEMP->0 = strict top-k score; TEMP large = uniform/diverse)\n\"\"\"\nimport json, sys, numpy as np\nc=np.load(\"/tmp/pool_feats.npz\")\nids,sc,ntok,gate,tag=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"]\nstrat=sys.argv[1]; T=float(sys.argv[2]); OUT=sys.argv[3]\nCODE_LANE = len(sys.argv)<5 or sys.argv[4]!=\"nocode\"\nBUDGET=12_000_000; OVER=2.5\nrng=np.random.default_rng(1337)\n\ncode_ids=[]\nif CODE_LANE:\n    m=(tag==1)&(sc>-50)\n    co=np.argsort(-np.where(m,sc,-1e9))\n    code_ids=[int(ids[j]) for j in co if m[j]]\n\ng=np.where(gate==1)[0]\nif strat==\"tilt\":\n    gum=rng.gumbel(size=len(g))\n    key=sc[g]/max(T,1e-6)+gum\n    gorder=g[np.argsort(-key)]\nelse:\n    raise SystemExit(\"unknown\")\n\nseen=set(code_ids); row={int(ids[k]):k for k in range(len(ids))}\nsel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)\nfor j in gorder:\n    i=int(ids[j])\n    if i in seen: continue\n    seen.add(i); sel.append(i); tot+=int(ntok[j])\n    if tot>=OVER*BUDGET: break\njson.dump(sel,open(OUT,\"w\"))\nprint(f\"{strat} T={T} code={len(code_ids)}: {len(sel)} ids ~{tot/1e6:.1f}M est tok -> {OUT}\")\n", "structuredPatch": [{"oldStart": 5, "oldLines": 8, "newStart": 5, "newLines": 10, "lines": [" \"\"\"", " import json, sys, numpy as np", " c=np.load(\"/tmp/pool_feats.npz\")", "-ids,sc,ntok,gate,tag=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"]", "+ids,sc,ntok,gate,tag,lpar,lrar=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"],c[\"lpar\"],c[\"lrar\"]", " strat=sys.argv[1]; T=float(sys.argv[2]); OUT=sys.argv[3]", "+# strat: \"tilt\"=combined sc | \"lr\"=target-vs-pool log-ratio (DSIR) | \"lp\"=target-LM logprob", "+SIGNAL={\"tilt\":sc,\"lr\":lrar,\"lp\":lpar}.get(strat,sc)", " CODE_LANE = len(sys.argv)<5 or sys.argv[4]!=\"nocode\"", " BUDGET=12_000_000; OVER=2.5", " rng=np.random.default_rng(1337)"]}], "userModified": false, "replaceAll": false}

The file /tmp/build_sel.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/tmp/build_sel.py", "oldString": "g=np.where(gate==1)[0]\nif strat==\"tilt\":\n    gum=rng.gumbel(size=len(g))\n    key=sc[g]/max(T,1e-6)+gum\n    gorder=g[np.argsort(-key)]\nelse:\n    raise SystemExit(\"unknown\")", "newString": "g=np.where(gate==1)[0]\ngum=rng.gumbel(size=len(g))\nkey=SIGNAL[g]/max(T,1e-6)+gum      # Gumbel-top-k => sample without replacement ∝ exp(SIGNAL/T)\ngorder=g[np.argsort(-key)]", "originalFile": "\"\"\"Build a selection variant from the cached per-doc features (no re-scoring).\nUsage: build_sel.py STRATEGY TEMP OUT\n  STRATEGY: tilt   -> code lane + Gumbel-top-k over gated docs with temperature TEMP\n                      (TEMP->0 = strict top-k score; TEMP large = uniform/diverse)\n\"\"\"\nimport json, sys, numpy as np\nc=np.load(\"/tmp/pool_feats.npz\")\nids,sc,ntok,gate,tag,lpar,lrar=c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"],c[\"lpar\"],c[\"lrar\"]\nstrat=sys.argv[1]; T=float(sys.argv[2]); OUT=sys.argv[3]\n# strat: \"tilt\"=combined sc | \"lr\"=target-vs-pool log-ratio (DSIR) | \"lp\"=target-LM logprob\nSIGNAL={\"tilt\":sc,\"lr\":lrar,\"lp\":lpar}.get(strat,sc)\nCODE_LANE = len(sys.argv)<5 or sys.argv[4]!=\"nocode\"\nBUDGET=12_000_000; OVER=2.5\nrng=np.random.default_rng(1337)\n\ncode_ids=[]\nif CODE_LANE:\n    m=(tag==1)&(sc>-50)\n    co=np.argsort(-np.where(m,sc,-1e9))\n    code_ids=[int(ids[j]) for j in co if m[j]]\n\ng=np.where(gate==1)[0]\nif strat==\"tilt\":\n    gum=rng.gumbel(size=len(g))\n    key=sc[g]/max(T,1e-6)+gum\n    gorder=g[np.argsort(-key)]\nelse:\n    raise SystemExit(\"unknown\")\n\nseen=set(code_ids); row={int(ids[k]):k for k in range(len(ids))}\nsel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)\nfor j in gorder:\n    i=int(ids[j])\n    if i in seen: continue\n    seen.add(i); sel.append(i); tot+=int(ntok[j])\n    if tot>=OVER*BUDGET: break\njson.dump(sel,open(OUT,\"w\"))\nprint(f\"{strat} T={T} code={len(code_ids)}: {len(sel)} ids ~{tot/1e6:.1f}M est tok -> {OUT}\")\n", "structuredPatch": [{"oldStart": 20, "oldLines": 12, "newStart": 20, "newLines": 9, "lines": ["     code_ids=[int(ids[j]) for j in co if m[j]]", " ", " g=np.where(gate==1)[0]", "-if strat==\"tilt\":", "-    gum=rng.gumbel(size=len(g))", "-    key=sc[g]/max(T,1e-6)+gum", "-    gorder=g[np.argsort(-key)]", "-else:", "-    raise SystemExit(\"unknown\")", "+gum=rng.gumbel(size=len(g))", "+key=SIGNAL[g]/max(T,1e-6)+gum      # Gumbel-top-k => sample without replacement ∝ exp(SIGNAL/T)", "+gorder=g[np.argsort(-key)]", " ", " seen=set(code_ids); row={int(ids[k]):k for k in range(len(ids))}", " sel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)"]}], "userModified": false, "replaceAll": false}

lrar (gated): mean -0.160 std 0.171  p10 -0.364 p50 -0.166 p90 0.045
sc  (gated): mean -8.165 std 0.876
lr T=0.15 code=192: 30042 ids ~30.0M est tok -> /tmp/sel_lr015.json
lr T=0.08 code=192: 28862 ids ~30.0M est tok -> /tmp/sel_lr008.json
remaining: 3708

[stdout]
lrar (gated): mean -0.160 std 0.171  p10 -0.364 p50 -0.166 p90 0.045
sc  (gated): mean -8.165 std 0.876
lr T=0.15 code=192: 30042 ids ~30.0M est tok -> /tmp/sel_lr015.json
lr T=0.08 code=192: 28862 ids ~30.0M est tok -> /tmp/sel_lr008.json
remaining: 3708
[metadata] {"noOutputExpected": false}

File created successfully at: /tmp/myeval.sh (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/tmp/myeval.sh", "content": "#!/bin/bash\n# myeval.sh SEL TAG  -> pack to unique file, train, print tagged PPL. Safe to run concurrently.\nset -euo pipefail\nexport TOKENIZERS_PARALLELISM=false\nSEL=\"$1\"; TAG=\"$2\"; cd /workspace\npython3 pack_selection.py \"$SEL\" /tmp/tr_${TAG}.npy >/tmp/pack_${TAG}.log 2>&1\npython3 train_nano.py --train_npy /tmp/tr_${TAG}.npy --heldout_npy /workspace/data/multi_dev.npy \\\n    --out_json /tmp/out_${TAG}.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL | sed \"s/^/[$TAG] /\"\n", "structuredPatch": [], "originalFile": null, "userModified": false}

launched lr015 + lr008 concurrently
remaining: 3677

[stdout]
launched lr015 + lr008 concurrently
remaining: 3677
[metadata] {"noOutputExpected": false}

Updated task #3 status

[metadata] {"success": true, "taskId": "3", "updatedFields": ["status"], "statusChange": {"from": "in_progress", "to": "completed"}}

Updated task #4 status

[metadata] {"success": true, "taskId": "4", "updatedFields": ["status"], "statusChange": {"from": "pending", "to": "in_progress"}}

=== lr015 ===
=== lr008 ===
--- raw tails ---
(uniform=366.5)
remaining: 3537

[stdout]
=== lr015 ===
=== lr008 ===
--- raw tails ---
(uniform=366.5)
remaining: 3537
[metadata] {"noOutputExpected": false}

=== pack logs ===
packed 12000000 tokens from 30042 ids -> /tmp/tr_lr015.npy
packed 12000000 tokens from 28862 ids -> /tmp/tr_lr008.npy
=== out jsons (done if present) ===
=== running procs ===
5
=== GPU ===
19071 MiB, 100 %
remaining: 3527

[stdout]
=== pack logs ===
packed 12000000 tokens from 30042 ids -> /tmp/tr_lr015.npy
packed 12000000 tokens from 28862 ids -> /tmp/tr_lr008.npy
=== out jsons (done if present) ===
=== running procs ===
5
=== GPU ===
19071 MiB, 100 %
remaining: 3527
[metadata] {"noOutputExpected": false}

=== lr015 ===
[lr015] HELDOUT_PPL 321.630  loss 5.7734  train_s 119  /tmp/tr_lr015.npy
=== lr008 ===
[lr008] HELDOUT_PPL 314.637  loss 5.7514  train_s 118  /tmp/tr_lr008.npy
(uniform=366.5, random=457)
remaining: 3447

[stdout]
=== lr015 ===
[lr015] HELDOUT_PPL 321.630  loss 5.7734  train_s 119  /tmp/tr_lr015.npy
=== lr008 ===
[lr008] HELDOUT_PPL 314.637  loss 5.7514  train_s 118  /tmp/tr_lr008.npy
(uniform=366.5, random=457)
remaining: 3447
[metadata] {"noOutputExpected": false}

lr T=0.05 code=192: 28362 ids ~30.0M est tok -> /tmp/sel_lr005.json
lr T=0.03 code=192: 28429 ids ~30.0M est tok -> /tmp/sel_lr003.json
launched lr005 + lr003
remaining: 3427

[stdout]
lr T=0.05 code=192: 28362 ids ~30.0M est tok -> /tmp/sel_lr005.json
lr T=0.03 code=192: 28429 ids ~30.0M est tok -> /tmp/sel_lr003.json
launched lr005 + lr003
remaining: 3427
[metadata] {"noOutputExpected": false}

=== lr005 ===
[lr005] HELDOUT_PPL 304.468  loss 5.7186  train_s 118  /tmp/tr_lr005.npy
=== lr003 ===
[lr003] HELDOUT_PPL 310.757  loss 5.7390  train_s 118  /tmp/tr_lr003.npy
(T0.15=321.6, T0.08=314.6, uniform=366)
remaining: 3227

[stdout]
=== lr005 ===
[lr005] HELDOUT_PPL 304.468  loss 5.7186  train_s 118  /tmp/tr_lr005.npy
=== lr003 ===
[lr003] HELDOUT_PPL 310.757  loss 5.7390  train_s 118  /tmp/tr_lr003.npy
(T0.15=321.6, T0.08=314.6, uniform=366)
remaining: 3227
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "# ---------------- 4. emit: code lane first (guaranteed), then clean backbone --------\n# Code lane: every tag-code doc with a minimal amount of English prose, best-first.\ncode_ok = (tag==1) & (sc > -50)                  # has words + some target signal\ncode_order = np.argsort(-np.where(code_ok, sc, -1e9))\ncode_ids = [int(ids[j]) for j in code_order if code_ok[j]]\n# Backbone: gated clean-English docs, best-first (excluding ones already in code lane).\nfinal = np.where(gate==1, sc, -1e9)\nback_order = np.argsort(-final)\nseen = set(code_ids)\nsel=[]; tot=0\nfor i in code_ids:                               # code lane at the front\n    sel.append(i); tot += int(ntok[np.searchsorted(ids, i)]) if False else 0\n# recompute code-lane token totals cleanly via id->row map\nrow = {int(ids[k]):k for k in range(len(ids))}\nsel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)\nfor j in back_order:                             # then clean backbone\n    if final[j] <= -1e8: break\n    i=int(ids[j])\n    if i in seen: continue\n    sel.append(i); tot += int(ntok[j])\n    if tot >= OVER*BUDGET: break\njson.dump(sel, open(OUT,\"w\"))\nprint(f\"selection: {len(sel)} ids ({len(code_ids)} code-lane), ~{tot/1e6:.1f}M est tokens -> {OUT}\", flush=True)", "newString": "# ---------------- 4. emit: code lane first, then DIVERSITY-PRESERVING match ----------\n# Key empirical finding: the target is BROAD (4 registers), so strict top-k on any\n# quality axis over-concentrates and HURTS (worse than random). Instead we (a) drop\n# junk with the gate, then (b) IMPORTANCE-RESAMPLE the survivors toward the target\n# distribution with a Gumbel-top-k on the target-vs-pool log-ratio (DSIR).  A mild\n# temperature keeps diversity while tilting the register mix toward the target.\nFINAL_T = 0.05          # resampling temperature (swept on dev: min ppl near 0.05)\nrng = np.random.default_rng(1337)\n# code lane: the pool's only supply for the target's ~25%-of-tokens code register\ncode_ok = (tag==1) & (sc > -50)\ncode_ids = [int(ids[j]) for j in np.argsort(-np.where(code_ok, sc, -1e9)) if code_ok[j]]\nseen = set(code_ids); row = {int(ids[k]):k for k in range(len(ids))}\nsel = list(code_ids); tot = sum(int(ntok[row[i]]) for i in sel)\n# resample gated docs without replacement ∝ exp(logratio / T)\ng = np.where(gate==1)[0]\nkey = lrar[g]/FINAL_T + rng.gumbel(size=len(g))\nfor j in g[np.argsort(-key)]:\n    i = int(ids[j])\n    if i in seen: continue\n    seen.add(i); sel.append(i); tot += int(ntok[j])\n    if tot >= OVER*BUDGET: break\njson.dump(sel, open(OUT,\"w\"))\nprint(f\"selection: {len(sel)} ids ({len(code_ids)} code-lane), ~{tot/1e6:.1f}M est tokens -> {OUT}\", flush=True)", "originalFile": "\"\"\"Curate a 12M-token pretraining selection from a raw web pool by domain-matching\nto the disclosed broad high-quality target (Wikipedia / web prose / news / tech Q&A).\n\nCriterion (stated, reproducible), CCNet-style:\n  1. GATE — drop non-English / boilerplate / word-salad / degenerate docs with cheap\n     structural filters (English stop-word ratio, letter ratio, unique-word ratio,\n     word length, length).  The stop-word-ratio gate is the workhorse: fluent English\n     prose (all four target registers) is rich in {the,of,and,to,a,in,is,...} whereas\n     CSS/JS dumps, foreign text and SEO word-salad are not.\n  2. SCORE — rank survivors by a TARGET bigram language model's mean per-word\n     log-probability (interpolated bigram+unigram+uniform).  Documents whose wording\n     is *typical of the target* score high; this rewards clean encyclopedic / news /\n     web-prose / Q&A English and demotes odd-but-English filler.\n  3. A small target-vs-pool log-ratio bonus adds discrimination toward\n     target-distinctive content over generic web filler.\n  Emit ids best-first until ~2x the 12M-token budget so packing never underfills.\n\nThe dev target only DEFINES the target word distribution (which generalises to the\nhidden official sample); no per-id labels are used.  Per-doc features are cached to\n/tmp so the ranking/threshold can be re-derived without re-reading the pool.\n\"\"\"\nimport json, re, math, sys, os, numpy as np\nfrom collections import defaultdict\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\nCACHE = \"/tmp/pool_feats.npz\"\n\nMAX_WORDS = 600          # words scanned per doc for scoring (bounds cost)\nBG_SAMPLE = 40000        # pool docs for background unigram model (log-ratio bonus)\nBUDGET    = 12_000_000\nOVER      = 2.0          # over-provide ids to this multiple of the budget\nCHARS_PER_TOK = 4.435\nEXPLORE = \"--explore\" in sys.argv\nRATIO_W = 0.35           # weight of target-vs-pool log-ratio bonus\n\nSTOP = set(\"the of and to a in is that it for on with as was are be by this at from \"\n           \"or an not but have has had he she they we you i his her their our your its \"\n           \"which who will would can could there been were said more one all if them \"\n           \"when so what about into than then some other time up out only over also \"\n           \"no do does did how new may these two his\".split())\nword_re = re.compile(r\"[a-z][a-z']+\")     # alphabetic words only (for gates + LM)\n\ndef words(text):\n    return word_re.findall(text.lower())\n\n# ---------------- 1. target bigram LM + unigram, from decoded dev ----------------\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained(\"gpt2\")\ndev = np.load(DEV).tolist(); EOS = 50256\ndocs, cur = [], []\nfor t in dev:\n    if t == EOS:\n        if cur: docs.append(cur); cur = []\n    else: cur.append(t)\nif cur: docs.append(cur)\ndef clean(s): return s.replace(\" @-@ \", \"-\").replace(\" @,@ \", \",\").replace(\" @.@ \", \".\")\ntgt_uni = defaultdict(int); tgt_bi = defaultdict(int); tgt_N = 0\nfor d in docs:\n    ws = words(clean(tok.decode(d)))\n    for j,wd in enumerate(ws):\n        tgt_uni[wd] += 1; tgt_N += 1\n        if j: tgt_bi[(ws[j-1], wd)] += 1\nV = len(tgt_uni)\nprint(f\"target: {len(docs)} docs, {tgt_N} words, vocab {V}\", flush=True)\n\n# ---------------- 2. pool background unigram (for log-ratio bonus) ----------------\nbg_uni = defaultdict(int); bg_N = 0; nbg = 0\nwith open(POOL) as fh:\n    for line in fh:\n        if nbg >= BG_SAMPLE: break\n        for wd in words(json.loads(line)[\"text\"])[:MAX_WORDS]:\n            bg_uni[wd] += 1; bg_N += 1\n        nbg += 1\nprint(f\"background: {nbg} docs, {bg_N} words\", flush=True)\n\n# precompute unigram log-probs and log-ratio per word\nL2, L1, L0 = 0.6, 0.399, 0.001    # bigram / unigram / uniform interpolation\ndef uni_logp(wd):\n    return math.log(L1 * tgt_uni.get(wd,0)/tgt_N + L0/1.0 * 1.0/ (V+1))\n# log ratio target/pool for a word (smoothed), clipped\ndef logratio(wd):\n    pt = (tgt_uni.get(wd,0)+0.5)/(tgt_N+0.5*V)\n    pp = (bg_uni.get(wd,0)+0.5)/(bg_N+0.5*V)\n    return max(-3.0, min(3.0, math.log(pt/pp)))\n\ndef score_doc(ws):\n    \"\"\"mean per-word target-LM logprob (+ mean log-ratio bonus).\"\"\"\n    n = len(ws)\n    if n == 0: return -99.0, 0.0\n    lp = 0.0; lr = 0.0; prev = None\n    for wd in ws:\n        pu = tgt_uni.get(wd,0)/tgt_N\n        if prev is not None:\n            cb = tgt_bi.get((prev,wd),0)\n            pbi = cb/tgt_uni[prev] if tgt_uni.get(prev,0) else 0.0\n        else:\n            pbi = 0.0\n        p = L2*pbi + L1*pu + L0*(1.0/(V+1))\n        lp += math.log(p)\n        lr += logratio(wd)\n        prev = wd\n    return lp/n, lr/n\n\n# ---------------- 3. single pass over pool: gates + score, cache ----------------\ndef load_and_score():\n    ids=[]; sc=[]; lpar=[]; lrar=[]; ntok=[]; gate=[]; tag=[]\n    with open(POOL) as fh:\n        for line in fh:\n            r = json.loads(line); text = r[\"text\"]\n            # technical/code Q&A lane: HTML/code tags are the only reliable signal for\n            # the target's ~25%-of-tokens code register, which is scarce in the pool.\n            is_tag = (\"<code>\" in text or \"<pre>\" in text or \"</p>\" in text)\n            ws = words(text)[:MAX_WORDS]\n            nw = len(ws)\n            ids.append(r[\"id\"]); ntok.append(int(len(text)/CHARS_PER_TOK)+1)\n            tag.append(1 if is_tag else 0)\n            if nw < 60:\n                sc.append(-99.0); lpar.append(-99.0); lrar.append(0.0); gate.append(0); continue\n            head = text[:6000]\n            alpha_frac = sum(c.isalpha() for c in head)/max(1,len(head))\n            stop_ratio = sum(w in STOP for w in ws)/nw\n            uniq_ratio = len(set(ws))/nw\n            mwl = sum(len(w) for w in ws)/nw\n            ok = (alpha_frac>0.60 and stop_ratio>0.20 and stop_ratio<0.65\n                  and uniq_ratio>0.34 and 3.0<=mwl<=9.0)\n            lp, lr = score_doc(ws)\n            sc.append(lp + RATIO_W*lr); lpar.append(lp); lrar.append(lr)\n            gate.append(1 if ok else 0)\n    return (np.array(ids), np.array(sc,dtype=np.float32), np.array(lpar,dtype=np.float32),\n            np.array(lrar,dtype=np.float32), np.array(ntok), np.array(gate,dtype=np.int8),\n            np.array(tag,dtype=np.int8))\n\nids, sc, lpar, lrar, ntok, gate, tag = load_and_score()\nnp.savez(CACHE, ids=ids, sc=sc, lpar=lpar, lrar=lrar, ntok=ntok, gate=gate, tag=tag)\nprint(f\"scored {len(ids)} docs; gate-pass {int(gate.sum())}; tag-code {int(tag.sum())}\", flush=True)\n\n# ---------------- 4. emit: code lane first (guaranteed), then clean backbone --------\n# Code lane: every tag-code doc with a minimal amount of English prose, best-first.\ncode_ok = (tag==1) & (sc > -50)                  # has words + some target signal\ncode_order = np.argsort(-np.where(code_ok, sc, -1e9))\ncode_ids = [int(ids[j]) for j in code_order if code_ok[j]]\n# Backbone: gated clean-English docs, best-first (excluding ones already in code lane).\nfinal = np.where(gate==1, sc, -1e9)\nback_order = np.argsort(-final)\nseen = set(code_ids)\nsel=[]; tot=0\nfor i in code_ids:                               # code lane at the front\n    sel.append(i); tot += int(ntok[np.searchsorted(ids, i)]) if False else 0\n# recompute code-lane token totals cleanly via id->row map\nrow = {int(ids[k]):k for k in range(len(ids))}\nsel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)\nfor j in back_order:                             # then clean backbone\n    if final[j] <= -1e8: break\n    i=int(ids[j])\n    if i in seen: continue\n    sel.append(i); tot += int(ntok[j])\n    if tot >= OVER*BUDGET: break\njson.dump(sel, open(OUT,\"w\"))\nprint(f\"selection: {len(sel)} ids ({len(code_ids)} code-lane), ~{tot/1e6:.1f}M est tokens -> {OUT}\", flush=True)\n\nif EXPLORE:\n    id2text={}\n    with open(POOL) as fh:\n        for line in fh:\n            r=json.loads(line); id2text[r[\"id\"]]=r[\"text\"]\n    def show(rank_idxs, tag):\n        for j in rank_idxs:\n            i=int(ids[j])\n            print(f\"[{tag} sc {sc[j]:.3f} lp {lpar[j]:.3f} lr {lrar[j]:.3f} id {i}] {id2text[i][:200].strip()[:200]!r}\")\n    top=order[:20000]\n    codef=np.mean([(\"<code>\" in id2text[int(ids[j])] or \"<pre>\" in id2text[int(ids[j])]) for j in top])\n    print(f\"\\ntop-20k code/pre-frac={codef:.3f}\")\n    print(\"\\n== TOP 8 ==\"); show(order[:8],\"top\")\n    print(\"\\n== rank ~8000 ==\"); show(order[8000:8003],\"mid\")\n    print(\"\\n== rank ~16000 ==\"); show(order[16000:16003],\"lo\")\n    g=sc[gate==1]\n    print(f\"\\ngated scores: p50 {np.percentile(g,50):.3f} p90 {np.percentile(g,90):.3f} p99 {np.percentile(g,99):.3f}\")\n", "structuredPatch": [{"oldStart": 136, "oldLines": 26, "newStart": 136, "newLines": 26, "lines": [" np.savez(CACHE, ids=ids, sc=sc, lpar=lpar, lrar=lrar, ntok=ntok, gate=gate, tag=tag)", " print(f\"scored {len(ids)} docs; gate-pass {int(gate.sum())}; tag-code {int(tag.sum())}\", flush=True)", " ", "-# ---------------- 4. emit: code lane first (guaranteed), then clean backbone --------", "-# Code lane: every tag-code doc with a minimal amount of English prose, best-first.", "-code_ok = (tag==1) & (sc > -50)                  # has words + some target signal", "-code_order = np.argsort(-np.where(code_ok, sc, -1e9))", "-code_ids = [int(ids[j]) for j in code_order if code_ok[j]]", "-# Backbone: gated clean-English docs, best-first (excluding ones already in code lane).", "-final = np.where(gate==1, sc, -1e9)", "-back_order = np.argsort(-final)", "-seen = set(code_ids)", "-sel=[]; tot=0", "-for i in code_ids:                               # code lane at the front", "-    sel.append(i); tot += int(ntok[np.searchsorted(ids, i)]) if False else 0", "-# recompute code-lane token totals cleanly via id->row map", "-row = {int(ids[k]):k for k in range(len(ids))}", "-sel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)", "-for j in back_order:                             # then clean backbone", "-    if final[j] <= -1e8: break", "-    i=int(ids[j])", "+# ---------------- 4. emit: code lane first, then DIVERSITY-PRESERVING match ----------", "+# Key empirical finding: the target is BROAD (4 registers), so strict top-k on any", "+# quality axis over-concentrates and HURTS (worse than random). Instead we (a) drop", "+# junk with the gate, then (b) IMPORTANCE-RESAMPLE the survivors toward the target", "+# distribution with a Gumbel-top-k on the target-vs-pool log-ratio (DSIR).  A mild", "+# temperature keeps diversity while tilting the register mix toward the target.", "+FINAL_T = 0.05          # resampling temperature (swept on dev: min ppl near 0.05)", "+rng = np.random.default_rng(1337)", "+# code lane: the pool's only supply for the target's ~25%-of-tokens code register", "+code_ok = (tag==1) & (sc > -50)", "+code_ids = [int(ids[j]) for j in np.argsort(-np.where(code_ok, sc, -1e9)) if code_ok[j]]", "+seen = set(code_ids); row = {int(ids[k]):k for k in range(len(ids))}", "+sel = list(code_ids); tot = sum(int(ntok[row[i]]) for i in sel)", "+# resample gated docs without replacement ∝ exp(logratio / T)", "+g = np.where(gate==1)[0]", "+key = lrar[g]/FINAL_T + rng.gumbel(size=len(g))", "+for j in g[np.argsort(-key)]:", "+    i = int(ids[j])", "     if i in seen: continue", "-    sel.append(i); tot += int(ntok[j])", "+    seen.add(i); sel.append(i); tot += int(ntok[j])", "     if tot >= OVER*BUDGET: break", " json.dump(sel, open(OUT,\"w\"))", " print(f\"selection: {len(sel)} ids ({len(code_ids)} code-lane), ~{tot/1e6:.1f}M est tokens -> {OUT}\", flush=True)"]}], "userModified": false, "replaceAll": false}

ids: 28362 unique: 28362 min: 5 max: 182015 in range: True
locked submission = lr005 (dev ppl 304.5). remaining: 3117

[stdout]
ids: 28362 unique: 28362 min: 5 max: 182015 in range: True
locked submission = lr005 (dev ppl 304.5). remaining: 3117
[metadata] {"noOutputExpected": false}

File created successfully at: /tmp/rescore.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/tmp/rescore.py", "content": "\"\"\"Enrich the cache with a bigram target-vs-pool log-ratio (sharper DSIR signal)\nand an exact-duplicate flag. Reuses gate/ntok/tag/lrar/sc from the existing cache.\"\"\"\nimport json, re, math, numpy as np\nfrom collections import defaultdict\nfrom transformers import AutoTokenizer\n\nPOOL=\"/workspace/data/pool.jsonl\"; DEV=\"/workspace/data/multi_dev.npy\"\nc=np.load(\"/tmp/pool_feats.npz\")\nids,sc,lpar,lrar,ntok,gate,tag=c[\"ids\"],c[\"sc\"],c[\"lpar\"],c[\"lrar\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"]\nword_re=re.compile(r\"[a-z][a-z']+\")\ndef words(t): return word_re.findall(t.lower())\nMAXW=400; BGDOCS=30000; BICAP=4_000_000\n\ntok=AutoTokenizer.from_pretrained(\"gpt2\")\ndev=np.load(DEV).tolist(); EOS=50256\ndocs,cur=[],[]\nfor t in dev:\n    if t==EOS:\n        if cur: docs.append(cur); cur=[]\n    else: cur.append(t)\ndef clean(s): return s.replace(\" @-@ \",\"-\").replace(\" @,@ \",\",\").replace(\" @.@ \",\".\")\ntbi=defaultdict(int); tN=0\nfor d in docs:\n    ws=words(clean(tok.decode(d)))\n    for j in range(1,len(ws)):\n        tbi[(ws[j-1],ws[j])]+=1; tN+=1\nprint(\"target bigrams\",len(tbi),tN,flush=True)\n\nbbi=defaultdict(int); bN=0; nb=0\nwith open(POOL) as fh:\n    for line in fh:\n        if nb>=BGDOCS: break\n        ws=words(json.loads(line)[\"text\"])[:MAXW]\n        for j in range(1,len(ws)):\n            k=(ws[j-1],ws[j])\n            if k in bbi or len(bbi)<BICAP: bbi[k]+=1\n            bN+=1\n        nb+=1\nprint(\"bg bigrams\",len(bbi),bN,flush=True)\n\nBv=len(tbi)+len(bbi); a=0.1\nlc_t=math.log(tN+a*Bv); lc_b=math.log(bN+a*Bv)\ndef bilr(k):\n    return max(-3.0,min(3.0,(math.log(tbi.get(k,0)+a)-lc_t)-(math.log(bbi.get(k,0)+a)-lc_b)))\n\nlr2=np.zeros(len(ids),dtype=np.float32); dup=np.zeros(len(ids),dtype=np.int8)\nnorm=re.compile(r\"[^a-z0-9]+\"); seen={}\nidx=0\nwith open(POOL) as fh:\n    for line in fh:\n        r=json.loads(line); t=r[\"text\"]\n        ws=words(t)[:MAXW]\n        if len(ws)>1:\n            s=sum(bilr((ws[j-1],ws[j])) for j in range(1,len(ws)))/(len(ws)-1)\n        else: s=0.0\n        lr2[idx]=s\n        fp=hash(norm.sub(\"\",t.lower())[:400])\n        if fp in seen: dup[idx]=1\n        else: seen[fp]=idx\n        idx+=1\nnp.savez(\"/tmp/pool_feats2.npz\",ids=ids,sc=sc,lpar=lpar,lrar=lrar,ntok=ntok,gate=gate,tag=tag,lr2=lr2,dup=dup)\nprint(f\"rescored {idx}; dups {int(dup.sum())}; lr2 mean {lr2[gate==1].mean():.3f} std {lr2[gate==1].std():.3f}\",flush=True)\n", "structuredPatch": [], "originalFile": null, "userModified": false}

rescore launched pid 3629
remaining: 3087

[stdout]
rescore launched pid 3629
remaining: 3087
[metadata] {"noOutputExpected": false}

File created successfully at: /tmp/build2.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/tmp/build2.py", "content": "\"\"\"Build selection from enriched cache. Usage: build2.py BW T OUT [dedup]\nSignal = lrar(uni) + BW*lr2(bigram). Gumbel-top-k over gated docs at temperature T.\"\"\"\nimport json, sys, numpy as np\nc=np.load(\"/tmp/pool_feats2.npz\")\nids,sc,ntok,gate,tag,lrar,lr2,dup=(c[\"ids\"],c[\"sc\"],c[\"ntok\"],c[\"gate\"],c[\"tag\"],\n                                   c[\"lrar\"],c[\"lr2\"],c[\"dup\"])\nBW=float(sys.argv[1]); T=float(sys.argv[2]); OUT=sys.argv[3]\nDEDUP = len(sys.argv)>4 and sys.argv[4]==\"dedup\"\nBUDGET=12_000_000; OVER=2.5; rng=np.random.default_rng(1337)\nSIG = lrar + BW*lr2\n\nkeep = (gate==1)\nif DEDUP: keep &= (dup==0)\ncode_ok=(tag==1)&(sc>-50)\nif DEDUP: code_ok &= (dup==0)\ncode_ids=[int(ids[j]) for j in np.argsort(-np.where(code_ok,sc,-1e9)) if code_ok[j]]\nseen=set(code_ids); row={int(ids[k]):k for k in range(len(ids))}\nsel=list(code_ids); tot=sum(int(ntok[row[i]]) for i in sel)\ng=np.where(keep)[0]\nkey=SIG[g]/max(T,1e-6)+rng.gumbel(size=len(g))\nfor j in g[np.argsort(-key)]:\n    i=int(ids[j])\n    if i in seen: continue\n    seen.add(i); sel.append(i); tot+=int(ntok[j])\n    if tot>=OVER*BUDGET: break\njson.dump(sel,open(OUT,\"w\"))\nprint(f\"BW={BW} T={T} dedup={DEDUP} code={len(code_ids)}: {len(sel)} ids ~{tot/1e6:.1f}M -> {OUT}\")\n", "structuredPatch": [], "originalFile": null, "userModified": false}

=== rescore log ===
target bigrams 313669 634316
bg bigrams 2634434 7980686
remaining: 2987

[stdout]
=== rescore log ===
target bigrams 313669 634316
bg bigrams 2634434 7980686
remaining: 2987
[metadata] {"noOutputExpected": false}

=== rescore final ===
target bigrams 313669 634316
bg bigrams 2634434 7980686
rescored 182016; dups 11873; lr2 mean -0.073 std 0.469
-rw-r--r-- 1 root root 6372730 Aug  4 04:03 /tmp/pool_feats2.npz
remaining: 2886

[stdout]
=== rescore final ===
target bigrams 313669 634316
bg bigrams 2634434 7980686
rescored 182016; dups 11873; lr2 mean -0.073 std 0.469
-rw-r--r-- 1 root root 6372730 Aug  4 04:03 /tmp/pool_feats2.npz
remaining: 2886
[metadata] {"noOutputExpected": false}

BW=0.35 T=0.07 dedup=True code=169: 27572 ids ~30.0M -> /tmp/sel_A.json
BW=0.0 T=0.05 dedup=True code=169: 29514 ids ~30.0M -> /tmp/sel_B.json
launched A(uni+bi+dedup) B(uni+dedup)
remaining: 2856

[stdout]
BW=0.35 T=0.07 dedup=True code=169: 27572 ids ~30.0M -> /tmp/sel_A.json
BW=0.0 T=0.05 dedup=True code=169: 29514 ids ~30.0M -> /tmp/sel_B.json
launched A(uni+bi+dedup) B(uni+dedup)
remaining: 2856
[metadata] {"noOutputExpected": false}