claude-opus-5

ok
balanced: 52,029 ids, 47,641,974 tokens -> /tmp/sel_bal2.json
balanced: 44,989 ids, 47,520,698 tokens -> /tmp/sel_strict.json
balanced: 47,995 ids, 47,724,909 tokens -> /tmp/sel_nofilt.json
balanced: 51,999 ids, 47,649,982 tokens -> /tmp/sel_bw1.json
gumbel: 51,609 ids, 47,540,065 tokens -> /tmp/sel_gum.json

[stdout]
ok
balanced: 52,029 ids, 47,641,974 tokens -> /tmp/sel_bal2.json
balanced: 44,989 ids, 47,520,698 tokens -> /tmp/sel_strict.json
balanced: 47,995 ids, 47,724,909 tokens -> /tmp/sel_nofilt.json
balanced: 51,999 ids, 47,649,982 tokens -> /tmp/sel_bw1.json
gumbel: 51,609 ids, 47,540,065 tokens -> /tmp/sel_gum.json
[metadata] {"noOutputExpected": false}

launched 5
4599

[stdout]
launched 5
4599
[metadata] {"noOutputExpected": false}

24:NQ = int(os.environ.get("NQ", 4))  # target registers = 4 equal contiguous quarters of the dev stream
175:    with MPPool(int(os.environ.get("NPROC", 14)), initializer=init, initargs=(ui, bi, LRU, LRB)) as p:
179:    np.savez(f"{CACHE}/scores{os.environ.get('TAG','')}.npz",
bg_started

[stdout]
24:NQ = int(os.environ.get("NQ", 4))  # target registers = 4 equal contiguous quarters of the dev stream
175:    with MPPool(int(os.environ.get("NPROC", 14)), initializer=init, initargs=(ui, bi, LRU, LRB)) as p:
179:    np.savez(f"{CACHE}/scores{os.environ.get('TAG','')}.npz",
bg_started
[metadata] {"noOutputExpected": false}

8
2
4549

[stdout]
8
2
4549
[metadata] {"noOutputExpected": false}

2049       0 /bin/bash -c source /logs/agent/sessions/shell-snapshots/snapshot-bash-1785805867628-5fbs02.sh 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && { \builtin unalias -- 'unsetenv'; \builtin unset -f -- 'unsetenv'; } >/dev/null 2>&1 || true && eval 'cd /workspace; ps -eo pid,etimes,args | grep "[s]core_pool" | head -4; echo "--- log:"; tail -3 /tmp/score8.log; echo "--- grep raw:"; grep -n "0.\] \* " score_pool.py' < /dev/null && pwd -P >| /tmp/claude-e586-cwd
 2052       0 /bin/bash -c source /logs/agent/sessions/shell-snapshots/snapshot-bash-1785805867628-5fbs02.sh 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && { \builtin unalias -- 'unsetenv'; \builtin unset -f -- 'unsetenv'; } >/dev/null 2>&1 || true && eval 'cd /workspace; ps -eo pid,etimes,args | grep "[s]core_pool" | head -4; echo "--- log:"; tail -3 /tmp/score8.log; echo "--- grep raw:"; grep -n "0.\] \* " score_pool.py' < /dev/null && pwd -P >| /tmp/claude-e586-cwd
--- log:
target models...
target words/register: [111503, 111229, 111414, 111098, 111659, 111620, 105803, 104378]
background model...
--- grep raw:
115:            rows.append((r["id"], len(text), n, [0.] * 5, [0.] * 5,
116:                         [0.] * 7, 0, 0))

[stdout]
 2049       0 /bin/bash -c source /logs/agent/sessions/shell-snapshots/snapshot-bash-1785805867628-5fbs02.sh 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && { \builtin unalias -- 'unsetenv'; \builtin unset -f -- 'unsetenv'; } >/dev/null 2>&1 || true && eval 'cd /workspace; ps -eo pid,etimes,args | grep "[s]core_pool" | head -4; echo "--- log:"; tail -3 /tmp/score8.log; echo "--- grep raw:"; grep -n "0.\] \* " score_pool.py' < /dev/null && pwd -P >| /tmp/claude-e586-cwd
 2052       0 /bin/bash -c source /logs/agent/sessions/shell-snapshots/snapshot-bash-1785805867628-5fbs02.sh 2>/dev/null || true && shopt -u extglob 2>/dev/null || true && { \builtin unalias -- 'unsetenv'; \builtin unset -f -- 'unsetenv'; } >/dev/null 2>&1 || true && eval 'cd /workspace; ps -eo pid,etimes,args | grep "[s]core_pool" | head -4; echo "--- log:"; tail -3 /tmp/score8.log; echo "--- grep raw:"; grep -n "0.\] \* " score_pool.py' < /dev/null && pwd -P >| /tmp/claude-e586-cwd
--- log:
target models...
target words/register: [111503, 111229, 111414, 111098, 111659, 111620, 105803, 104378]
background model...
--- grep raw:
115:            rows.append((r["id"], len(text), n, [0.] * 5, [0.] * 5,
116:                         [0.] * 7, 0, 0))
[metadata] {"noOutputExpected": false}

The file /workspace/score_pool.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/score_pool.py", "oldString": "            rows.append((r[\"id\"], len(text), n, [0.] * 5, [0.] * 5,\n                         [0.] * 7, 0, 0))", "newString": "            rows.append((r[\"id\"], len(text), n, [0.] * (NQ + 1), [0.] * (NQ + 1),\n                         [0.] * 7, 0, 0))", "originalFile": "\"\"\"Score every pool document against the disclosed multi-domain target.\n\nCriterion (per document d, per target register q):\n    s_q(d) = (1/|d|) * sum_{f in ngrams(d)} log( lam * p_q(f)/p_bg(f) + (1-lam) )\ni.e. a length-normalised log importance ratio between a target-register n-gram\nmodel and the raw-web background model, with the target model interpolated onto\nthe background (Jelinek-Mercer). The interpolation is what keeps the score\nhonest: an n-gram the target never uses contributes exactly log(1-lam) < 0\ninstead of a spuriously positive add-alpha artefact, so boilerplate/spam made of\nrare junk n-grams can no longer outrank prose.\n\nAlso emits per-document quality statistics and dedup signatures.\n\nWrites /workspace/cache/scores.npz\n\"\"\"\nimport json, math, os, re, zlib\nfrom collections import Counter\nimport numpy as np\nfrom multiprocessing import Pool as MPPool\n\nPOOL = \"/workspace/data/pool.jsonl\"\nCACHE = \"/workspace/cache\"\nos.makedirs(CACHE, exist_ok=True)\nNQ = int(os.environ.get(\"NQ\", 4))  # target registers = 4 equal contiguous quarters of the dev stream\nLAM = 0.9           # Jelinek-Mercer: weight on the target model vs background\nINTERP = 0.6        # weight on own-register counts, rest on pooled-target counts\nBG_SAMPLE = 16000   # pool docs used to estimate the background (raw-web) model\n\n# every alnum run is a token; every other non-space char is its own token, so that\n# markup/punctuation style (HTML tags, wikitext \" @-@ \", spacing) is observable.\nTOKRE = re.compile(r\"[a-z0-9']+|[^\\sa-z0-9']\")\nSTOP = set(\"the of and to in a is that for it as was with on be by are this from or an \"\n           \"at not have has but they you we he she his her their our\".split())\n\n\ndef toks(text):\n    return TOKRE.findall(text.lower())\n\n\ndef accum(text, uni, big):\n    t = toks(text)\n    uni.update(t)\n    big.update(map(\" \".join, zip(t, t[1:])))\n    return len(t)\n\n\n# ---------------------------------------------------------------- models\ndef build_target():\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    arr = np.load(\"/workspace/data/multi_dev.npy\")\n    q = len(arr) // NQ\n    uni, big, n = [], [], []\n    for i in range(NQ):\n        txt = tok.decode(arr[i * q:(i + 1) * q].tolist()).replace(\"<|endoftext|>\", \"\\n\")\n        u, b = Counter(), Counter()\n        n.append(accum(txt, u, b))\n        uni.append(u); big.append(b)\n    return uni, big, n\n\n\ndef build_bg():\n    uni, big, n = Counter(), Counter(), 0\n    step = max(1, 182016 // BG_SAMPLE)\n    with open(POOL) as f:\n        for i, line in enumerate(f):\n            if i % step == 0:\n                n += accum(json.loads(line)[\"text\"], uni, big)\n    return uni, big, n\n\n\ndef lr_table(tq, tg, nq, ng, bg, nbg, vocab, interp):\n    \"\"\"log( LAM * p_target(f)/p_bg(f) + (1-LAM) ) for each f in vocab.\n    p_bg is add-0.5 smoothed so target-only n-grams stay finite.\"\"\"\n    dbg = nbg + 0.5 * (len(vocab) + 1)\n    out = np.empty(len(vocab) + 1, dtype=np.float32)\n    for i, f in enumerate(vocab):\n        pt = interp * tq.get(f, 0) / nq + (1 - interp) * tg.get(f, 0) / ng\n        pb = (bg.get(f, 0) + 0.5) / dbg\n        out[i] = math.log(LAM * pt / pb + (1 - LAM))\n    out[-1] = math.log(1 - LAM)   # OOV / unseen-in-target\n    return out\n\n\n# ---------------------------------------------------------------- worker\nG = {}\n\n\ndef init(ui, bi, LRU, LRB):\n    G.update(ui=ui, bi=bi, lru=LRU, lrb=LRB)\n\n\ndef sig(t):\n    \"\"\"cheap 2-hash MinHash over 8-word shingles -> near-duplicate band key\"\"\"\n    if len(t) < 12:\n        return 0\n    sh = [zlib.crc32(\" \".join(t[i:i + 8]).encode()) for i in range(0, len(t) - 8, 3)]\n    if not sh:\n        return 0\n    a = min(sh)\n    b = min(x ^ 0x5bf03635 for x in sh)\n    return (a << 32) | b\n\n\ndef score_chunk(lines):\n    ui, bi, lru, lrb = G[\"ui\"], G[\"bi\"], G[\"lru\"], G[\"lrb\"]\n    uu, ub = len(lru) - 1, len(lrb) - 1\n    rows = []\n    for line in lines:\n        r = json.loads(line)\n        text = r[\"text\"]\n        t = toks(text)\n        n = len(t)\n        if n < 12:\n            rows.append((r[\"id\"], len(text), n, [0.] * 5, [0.] * 5,\n                         [0.] * 7, 0, 0))\n            continue\n        cu = Counter(t)\n        cb = Counter(map(\" \".join, zip(t, t[1:])))\n        iu = np.fromiter((ui.get(w, uu) for w in cu), np.int32, len(cu))\n        wu = np.fromiter(cu.values(), np.float32, len(cu))\n        ib = np.fromiter((bi.get(w, ub) for w in cb), np.int32, len(cb))\n        wb = np.fromiter(cb.values(), np.float32, len(cb))\n        su = (lru[iu] * wu[:, None]).sum(0) / n\n        sb = (lrb[ib] * wb[:, None]).sum(0) / (n - 1)\n\n        # ---- quality statistics\n        words = [w for w in t if w[:1].isalpha()]\n        nw = max(1, len(words))\n        alpha_frac = len(words) / n\n        stop_frac = sum(1 for w in words if w in STOP) / nw\n        ls = text.split(\"\\n\")\n        uniq_line = len(set(ls)) / len(ls)\n        lw = [len(x.split()) for x in ls]\n        mean_lw = sum(lw) / len(lw)\n        short_lines = sum(1 for x in lw if x < 5) / len(lw)\n        nonascii = sum(1 for c in text if ord(c) > 127) / max(1, len(text))\n        top1 = cu.most_common(1)[0][1] / n\n        q = [alpha_frac, stop_frac, uniq_line, mean_lw, short_lines, nonascii, top1]\n        norm = \" \".join(words)\n        rows.append((r[\"id\"], len(text), n, su.tolist(), sb.tolist(), q,\n                     zlib.crc32(norm.encode()), sig(t)))\n    return rows\n\n\ndef main():\n    print(\"target models...\", flush=True)\n    tuni, tbig, tn = build_target()\n    guni, gbig = Counter(), Counter()\n    for u, b in zip(tuni, tbig):\n        guni.update(u); gbig.update(b)\n    gn = sum(tn)\n    print(\"target words/register:\", tn, flush=True)\n\n    print(\"background model...\", flush=True)\n    buni, bbig, bn = build_bg()\n    print(f\"bg words {bn:,}\", flush=True)\n\n    vu = sorted(set(guni) | {w for w, c in buni.items() if c >= 5})\n    vb = sorted(set(gbig) | {w for w, c in bbig.items() if c >= 10})\n    print(f\"vocab uni {len(vu):,} big {len(vb):,}\", flush=True)\n    ui = {w: i for i, w in enumerate(vu)}\n    bi = {w: i for i, w in enumerate(vb)}\n\n    LRU = np.stack([lr_table(tuni[q], guni, tn[q], gn, buni, bn, vu, INTERP)\n                    for q in range(NQ)] +\n                   [lr_table(guni, guni, gn, gn, buni, bn, vu, 1.0)], axis=1)\n    LRB = np.stack([lr_table(tbig[q], gbig, tn[q], gn, bbig, bn, vb, INTERP)\n                    for q in range(NQ)] +\n                   [lr_table(gbig, gbig, gn, gn, bbig, bn, vb, 1.0)], axis=1)\n\n    lines = open(POOL).readlines()\n    chunks = [lines[i:i + 400] for i in range(0, len(lines), 400)]\n    res = []\n    with MPPool(int(os.environ.get(\"NPROC\", 14)), initializer=init, initargs=(ui, bi, LRU, LRB)) as p:\n        for k, rows in enumerate(p.imap(score_chunk, chunks, chunksize=1)):\n            res.extend(rows)\n    res.sort(key=lambda r: r[0])\n    np.savez(f\"{CACHE}/scores{os.environ.get('TAG','')}.npz\",\n             ids=np.array([r[0] for r in res], np.int32),\n             nchars=np.array([r[1] for r in res], np.int32),\n             nwords=np.array([r[2] for r in res], np.int32),\n             su=np.array([r[3] for r in res], np.float32),\n             sb=np.array([r[4] for r in res], np.float32),\n             q=np.array([r[5] for r in res], np.float32),\n             exact=np.array([r[6] for r in res], np.int64),\n             near=np.array([r[7] for r in res], np.uint64))\n    print(\"saved\", len(res))\n\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 112, "oldLines": 7, "newStart": 112, "newLines": 7, "lines": ["         t = toks(text)", "         n = len(t)", "         if n < 12:", "-            rows.append((r[\"id\"], len(text), n, [0.] * 5, [0.] * 5,", "+            rows.append((r[\"id\"], len(text), n, [0.] * (NQ + 1), [0.] * (NQ + 1),", "                          [0.] * 7, 0, 0))", "             continue", "         cu = Counter(t)"]}], "userModified": false, "replaceAll": false}

--- sc8:
bg words 15,006,294
vocab uni 87,994 big 415,049

[stdout]
--- sc8:
bg words 15,006,294
vocab uni 87,994 big 415,049
[metadata] {"noOutputExpected": false}

--- sc8:
saved 182016
4349

[stdout]
--- sc8:
saved 182016
4349
[metadata] {"noOutputExpected": false}

154 1061 -c
156 1061 -c
157 1061 --verbose
1837 260 train_nano.py
1838 260 train_nano.py
1839 260 train_nano.py
1840 260 train_nano.py
1841 260 train_nano.py
--- logs:
bal2: strict: nofilt: bw1: gum:

[stdout]
154 1061 -c
156 1061 -c
157 1061 --verbose
1837 260 train_nano.py
1838 260 train_nano.py
1839 260 train_nano.py
1840 260 train_nano.py
1841 260 train_nano.py
--- logs:
bal2: strict: nofilt: bw1: gum:
[metadata] {"noOutputExpected": false}

bal2: HELDOUT_PPL 380.272  loss 5.9409  train_s 294  /tmp/tr_bal2.npy
strict: HELDOUT_PPL 440.091  loss 6.0870  train_s 294  /tmp/tr_strict.npy
nofilt: HELDOUT_PPL 335.897  loss 5.8168  train_s 294  /tmp/tr_nofilt.npy
bw1: HELDOUT_PPL 391.233  loss 5.9693  train_s 294  /tmp/tr_bw1.npy
gum: HELDOUT_PPL 394.142  loss 5.9767  train_s 294  /tmp/tr_gum.npy
4228

[stdout]
bal2: HELDOUT_PPL 380.272  loss 5.9409  train_s 294  /tmp/tr_bal2.npy
strict: HELDOUT_PPL 440.091  loss 6.0870  train_s 294  /tmp/tr_strict.npy
nofilt: HELDOUT_PPL 335.897  loss 5.8168  train_s 294  /tmp/tr_nofilt.npy
bw1: HELDOUT_PPL 391.233  loss 5.9693  train_s 294  /tmp/tr_bw1.npy
gum: HELDOUT_PPL 394.142  loss 5.9767  train_s 294  /tmp/tr_gum.npy
4228
[metadata] {"noOutputExpected": false}

The file /workspace/build_sel.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"type": "update", "filePath": "/workspace/build_sel.py", "content": "\"\"\"Build a candidate selection from cached importance scores.\n\nUsage: python3 build_sel.py <mode> <out.json> [bigram_w] [gumbel_T]\nmodes: random | global | balanced\nenv:   SCORES=cache/scores.npz  NREG=4  NODEDUP=0  MINW=12\n\"\"\"\nimport json, os, sys\nimport numpy as np\n\nCACHE = \"/workspace/cache\"\nBUDGET = 12_000_000\nNEED = 4 * BUDGET     # emit ~4x the budget in ids so truncation never starves it\nNREG = int(os.environ.get(\"NREG\", 4))\nMINW = int(os.environ.get(\"MINW\", 12))\nNODEDUP = int(os.environ.get(\"NODEDUP\", 0))\n\n\ndef load():\n    d = np.load(os.environ.get(\"SCORES\", f\"{CACHE}/scores.npz\"))\n    tl = np.load(f\"{CACHE}/tok_len.npy\")\n    return d, tl\n\n\ndef dedup_order(order, d):\n    \"\"\"keep the first (= highest-priority) doc of each exact/near-duplicate class\"\"\"\n    if NODEDUP:\n        return list(order)\n    ex, nr = d[\"exact\"], d[\"near\"]\n    seen_e, seen_n, out = set(), set(), []\n    for i in order:\n        e, n = int(ex[i]), int(nr[i])\n        if e in seen_e or (n and n in seen_n):\n            continue\n        seen_e.add(e)\n        if n:\n            seen_n.add(n)\n        out.append(i)\n    return out\n\n\ndef cut(order, tl, ids, need=NEED):\n    tot, out = 0, []\n    for i in order:\n        out.append(int(ids[i]))\n        tot += int(tl[i]) + 1\n        if tot >= need:\n            break\n    return out, tot\n\n\ndef rr(ranks, tl, ids, d):\n    \"\"\"Round-robin across the per-register rankings, interleaving by TOKENS, so that\n    every prefix of the priority list carries an equal token share per register.\"\"\"\n    used, order, ptr, tokq = set(), [], [0] * len(ranks), [0.0] * len(ranks)\n    while True:\n        q = int(np.argmin(tokq))\n        while ptr[q] < len(ranks[q]) and ranks[q][ptr[q]] in used:\n            ptr[q] += 1\n        if ptr[q] >= len(ranks[q]):\n            tokq[q] = float(\"inf\")\n            if all(np.isinf(t) for t in tokq):\n                break\n            continue\n        i = ranks[q][ptr[q]]; ptr[q] += 1\n        used.add(i); order.append(i); tokq[q] += int(tl[i]) + 1\n        if sum(t for t in tokq if not np.isinf(t)) >= NEED:\n            break\n    return cut(dedup_order(order, d), tl, ids)\n\n\ndef main():\n    mode, out = sys.argv[1], sys.argv[2]\n    bw = float(sys.argv[3]) if len(sys.argv) > 3 else 0.5\n    T = float(sys.argv[4]) if len(sys.argv) > 4 else 0.0\n    d, tl = load()\n    ids = d[\"ids\"]\n\n    if mode == \"random\":\n        order = np.random.default_rng(0).permutation(len(ids))\n        sel, tot = cut(order, tl, ids)\n    else:\n        cand = np.flatnonzero(d[\"nwords\"] >= MINW)\n        S = d[\"su\"] + bw * d[\"sb\"]\n        if T > 0:\n            rng = np.random.default_rng(0)\n            S = S / T + rng.gumbel(size=S.shape)\n        if mode == \"global\":\n            sel, tot = cut(dedup_order(cand[np.argsort(-S[cand, NREG])], d), tl, ids)\n        elif mode == \"balanced\":\n            ranks = [dedup_order(cand[np.argsort(-S[cand, q])], d) for q in range(NREG)]\n            sel, tot = rr(ranks, tl, ids, d)\n        else:\n            raise SystemExit(\"bad mode\")\n\n    json.dump(sel, open(out, \"w\"))\n    print(f\"{mode} bw={bw} T={T} NREG={NREG} nodedup={NODEDUP}: \"\n          f\"{len(sel):,} ids, {tot:,} tokens -> {out}\")\n\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 1, "oldLines": 36, "newStart": 1, "newLines": 30, "lines": ["-\"\"\"Build a candidate selection from cached scores. Usage:", "-   python3 build_sel.py <mode> <out.json> [bigram_w] [strict]", "+\"\"\"Build a candidate selection from cached importance scores.", "+", "+Usage: python3 build_sel.py <mode> <out.json> [bigram_w] [gumbel_T]", " modes: random | global | balanced", "+env:   SCORES=cache/scores.npz  NREG=4  NODEDUP=0  MINW=12", " \"\"\"", "-import json, sys", "+import json, os, sys", " import numpy as np", " ", " CACHE = \"/workspace/cache\"", " BUDGET = 12_000_000", "-NEED = 4 * BUDGET     # emit ~4x budget of tokens worth of ids", "+NEED = 4 * BUDGET     # emit ~4x the budget in ids so truncation never starves it", "+NREG = int(os.environ.get(\"NREG\", 4))", "+MINW = int(os.environ.get(\"MINW\", 12))", "+NODEDUP = int(os.environ.get(\"NODEDUP\", 0))", " ", " ", " def load():", "-    d = np.load(f\"{CACHE}/scores.npz\")", "+    d = np.load(os.environ.get(\"SCORES\", f\"{CACHE}/scores.npz\"))", "     tl = np.load(f\"{CACHE}/tok_len.npy\")", "     return d, tl", " ", " ", "-def qfilter(d, strict=1.0):", "-    q = d[\"q\"]", "-    alpha, stop, uniq, mean_lw, shortl, nonascii, top1 = [q[:, i] for i in range(7)]", "-    nw = d[\"nwords\"]", "-    keep = (nw >= 100) & (alpha >= 0.68) & (stop >= 0.18) & (uniq >= 0.55) \\", "-        & (shortl <= 0.60) & (nonascii <= 0.08) & (top1 <= 0.12) & (mean_lw >= 4.0)", "-    if strict <= 0.0:", "-        keep = np.ones(len(nw), bool) & (nw >= 12)", "-    if strict > 1.0:", "-        keep &= (stop >= 0.22) & (alpha >= 0.74) & (nw >= 200) & (top1 <= 0.09)", "-    return keep", "-", "-", " def dedup_order(order, d):", "-    \"\"\"keep the first occurrence of each exact / near-duplicate signature\"\"\"", "+    \"\"\"keep the first (= highest-priority) doc of each exact/near-duplicate class\"\"\"", "+    if NODEDUP:", "+        return list(order)", "     ex, nr = d[\"exact\"], d[\"near\"]", "     seen_e, seen_n, out = set(), set(), []", "     for i in order:"]}, {"oldStart": 55, "oldLines": 9, "newStart": 49, "newLines": 9, "lines": [" ", " ", " def rr(ranks, tl, ids, d):", "-    \"\"\"round-robin across the 4 per-register rankings, interleaving by TOKENS so", "-    that any prefix of the priority list is register-balanced.\"\"\"", "-    used, order, ptr, tokq = set(), [], [0] * 4, [0.0] * 4", "+    \"\"\"Round-robin across the per-register rankings, interleaving by TOKENS, so that", "+    every prefix of the priority list carries an equal token share per register.\"\"\"", "+    used, order, ptr, tokq = set(), [], [0] * len(ranks), [0.0] * len(ranks)", "     while True:", "         q = int(np.argmin(tokq))", "         while ptr[q] < len(ranks[q]) and ranks[q][ptr[q]] in used:"]}, {"oldStart": 75, "oldLines": 95, "newStart": 69, "newLines": 32, "lines": [" ", " ", " def main():", "-    mode = sys.argv[1]", "-    out = sys.argv[2]", "+    mode, out = sys.argv[1], sys.argv[2]", "     bw = float(sys.argv[3]) if len(sys.argv) > 3 else 0.5", "-    strict = float(sys.argv[4]) if len(sys.argv) > 4 else 1.0", "+    T = float(sys.argv[4]) if len(sys.argv) > 4 else 0.0", "     d, tl = load()", "     ids = d[\"ids\"]", "-    n = len(ids)", " ", "     if mode == \"random\":", "-        rng = np.random.default_rng(0)", "-        order = rng.permutation(n)", "+        order = np.random.default_rng(0).permutation(len(ids))", "         sel, tot = cut(order, tl, ids)", "     else:", "-        keep = qfilter(d, strict)", "-        cand = np.flatnonzero(keep)", "+        cand = np.flatnonzero(d[\"nwords\"] >= MINW)", "         S = d[\"su\"] + bw * d[\"sb\"]", "-        print(f\"kept {len(cand):,}/{n:,} after quality filter\", flush=True)", "-        if mode == \"gumbel\":", "+        if T > 0:", "             rng = np.random.default_rng(0)", "-            T = float(sys.argv[5]) if len(sys.argv) > 5 else 0.1", "-            ranks = []", "-            for q in range(4):", "-                k = S[cand, q] / T + rng.gumbel(size=len(cand))", "-                ranks.append(dedup_order(cand[np.argsort(-k)], d))", "-            sel, tot = rr(ranks, tl, ids, d)", "-        elif mode == \"global\":", "-            order = cand[np.argsort(-S[cand, 4])]", "-            order = dedup_order(order, d)", "-            sel, tot = cut(order, tl, ids)", "+            S = S / T + rng.gumbel(size=S.shape)", "+        if mode == \"global\":", "+            sel, tot = cut(dedup_order(cand[np.argsort(-S[cand, NREG])], d), tl, ids)", "         elif mode == \"balanced\":", "-            ranks = [dedup_order(cand[np.argsort(-S[cand, q])], d) for q in range(4)]", "+            ranks = [dedup_order(cand[np.argsort(-S[cand, q])], d) for q in range(NREG)]", "             sel, tot = rr(ranks, tl, ids, d)", "-        elif mode == \"balanced_doc\":", "-            # rank candidates separately per target register, then interleave", "-            # round-robin so that ANY prefix of the list is register-balanced.", "-            ranks = [cand[np.argsort(-S[cand, q])] for q in range(4)]", "-            ranks = [dedup_order(r, d) for r in ranks]", "-            used, order, ptr = set(), [], [0, 0, 0, 0]", "-            while True:", "-                progress = False", "-                for q in range(4):", "-                    while ptr[q] < len(ranks[q]) and ranks[q][ptr[q]] in used:", "-                        ptr[q] += 1", "-                    if ptr[q] < len(ranks[q]):", "-                        i = ranks[q][ptr[q]]", "-                        ptr[q] += 1", "-                        used.add(i)", "-                        order.append(i)", "-                        progress = True", "-                if not progress:", "-                    break", "-                if len(order) > 200000:", "-                    break", "-            order = dedup_order(order, d)", "-            sel, tot = cut(order, tl, ids)", "-        elif mode == \"quota\":", "-            # partition candidates by their best-matching register (on z-scored", "-            # per-register scores), then fill an equal TOKEN quota per register so", "-            # every prefix of the list matches the target's 4-way composition.", "-            Z = np.zeros((len(cand), 4), np.float32)", "-            for q in range(4):", "-                v = S[cand, q]", "-                Z[:, q] = (v - v.mean()) / (v.std() + 1e-9)", "-            best = Z.argmax(1)", "-            groups = []", "-            for q in range(4):", "-                g = cand[best == q]", "-                g = g[np.argsort(-Z[best == q, q])]", "-                groups.append(dedup_order(g, d))", "-                print(f\"  register {q}: {len(g):,} docs\")", "-            ptr, tokq, order = [0] * 4, [0] * 4, []", "-            while True:", "-                q = int(np.argmin(tokq))", "-                if ptr[q] >= len(groups[q]):", "-                    tokq[q] = float(\"inf\")", "-                    if all(np.isinf(tokq)):", "-                        break", "-                    continue", "-                i = groups[q][ptr[q]]; ptr[q] += 1", "-                order.append(i); tokq[q] += int(tl[i]) + 1", "-                if sum(t for t in tokq if not np.isinf(t)) >= NEED:", "-                    break", "-            order = dedup_order(order, d)", "-            sel, tot = cut(order, tl, ids)", "         else:", "             raise SystemExit(\"bad mode\")", " ", "     json.dump(sel, open(out, \"w\"))", "-    print(f\"{mode}: {len(sel):,} ids, {tot:,} tokens -> {out}\")", "+    print(f\"{mode} bw={bw} T={T} NREG={NREG} nodedup={NODEDUP}: \"", "+          f\"{len(sel):,} ids, {tot:,} tokens -> {out}\")", " ", " ", " if __name__ == \"__main__\":"]}], "originalFile": "\"\"\"Build a candidate selection from cached scores. Usage:\n   python3 build_sel.py <mode> <out.json> [bigram_w] [strict]\nmodes: random | global | balanced\n\"\"\"\nimport json, sys\nimport numpy as np\n\nCACHE = \"/workspace/cache\"\nBUDGET = 12_000_000\nNEED = 4 * BUDGET     # emit ~4x budget of tokens worth of ids\n\n\ndef load():\n    d = np.load(f\"{CACHE}/scores.npz\")\n    tl = np.load(f\"{CACHE}/tok_len.npy\")\n    return d, tl\n\n\ndef qfilter(d, strict=1.0):\n    q = d[\"q\"]\n    alpha, stop, uniq, mean_lw, shortl, nonascii, top1 = [q[:, i] for i in range(7)]\n    nw = d[\"nwords\"]\n    keep = (nw >= 100) & (alpha >= 0.68) & (stop >= 0.18) & (uniq >= 0.55) \\\n        & (shortl <= 0.60) & (nonascii <= 0.08) & (top1 <= 0.12) & (mean_lw >= 4.0)\n    if strict <= 0.0:\n        keep = np.ones(len(nw), bool) & (nw >= 12)\n    if strict > 1.0:\n        keep &= (stop >= 0.22) & (alpha >= 0.74) & (nw >= 200) & (top1 <= 0.09)\n    return keep\n\n\ndef dedup_order(order, d):\n    \"\"\"keep the first occurrence of each exact / near-duplicate signature\"\"\"\n    ex, nr = d[\"exact\"], d[\"near\"]\n    seen_e, seen_n, out = set(), set(), []\n    for i in order:\n        e, n = int(ex[i]), int(nr[i])\n        if e in seen_e or (n and n in seen_n):\n            continue\n        seen_e.add(e)\n        if n:\n            seen_n.add(n)\n        out.append(i)\n    return out\n\n\ndef cut(order, tl, ids, need=NEED):\n    tot, out = 0, []\n    for i in order:\n        out.append(int(ids[i]))\n        tot += int(tl[i]) + 1\n        if tot >= need:\n            break\n    return out, tot\n\n\ndef rr(ranks, tl, ids, d):\n    \"\"\"round-robin across the 4 per-register rankings, interleaving by TOKENS so\n    that any prefix of the priority list is register-balanced.\"\"\"\n    used, order, ptr, tokq = set(), [], [0] * 4, [0.0] * 4\n    while True:\n        q = int(np.argmin(tokq))\n        while ptr[q] < len(ranks[q]) and ranks[q][ptr[q]] in used:\n            ptr[q] += 1\n        if ptr[q] >= len(ranks[q]):\n            tokq[q] = float(\"inf\")\n            if all(np.isinf(t) for t in tokq):\n                break\n            continue\n        i = ranks[q][ptr[q]]; ptr[q] += 1\n        used.add(i); order.append(i); tokq[q] += int(tl[i]) + 1\n        if sum(t for t in tokq if not np.isinf(t)) >= NEED:\n            break\n    return cut(dedup_order(order, d), tl, ids)\n\n\ndef main():\n    mode = sys.argv[1]\n    out = sys.argv[2]\n    bw = float(sys.argv[3]) if len(sys.argv) > 3 else 0.5\n    strict = float(sys.argv[4]) if len(sys.argv) > 4 else 1.0\n    d, tl = load()\n    ids = d[\"ids\"]\n    n = len(ids)\n\n    if mode == \"random\":\n        rng = np.random.default_rng(0)\n        order = rng.permutation(n)\n        sel, tot = cut(order, tl, ids)\n    else:\n        keep = qfilter(d, strict)\n        cand = np.flatnonzero(keep)\n        S = d[\"su\"] + bw * d[\"sb\"]\n        print(f\"kept {len(cand):,}/{n:,} after quality filter\", flush=True)\n        if mode == \"gumbel\":\n            rng = np.random.default_rng(0)\n            T = float(sys.argv[5]) if len(sys.argv) > 5 else 0.1\n            ranks = []\n            for q in range(4):\n                k = S[cand, q] / T + rng.gumbel(size=len(cand))\n                ranks.append(dedup_order(cand[np.argsort(-k)], d))\n            sel, tot = rr(ranks, tl, ids, d)\n        elif mode == \"global\":\n            order = cand[np.argsort(-S[cand, 4])]\n            order = dedup_order(order, d)\n            sel, tot = cut(order, tl, ids)\n        elif mode == \"balanced\":\n            ranks = [dedup_order(cand[np.argsort(-S[cand, q])], d) for q in range(4)]\n            sel, tot = rr(ranks, tl, ids, d)\n        elif mode == \"balanced_doc\":\n            # rank candidates separately per target register, then interleave\n            # round-robin so that ANY prefix of the list is register-balanced.\n            ranks = [cand[np.argsort(-S[cand, q])] for q in range(4)]\n            ranks = [dedup_order(r, d) for r in ranks]\n            used, order, ptr = set(), [], [0, 0, 0, 0]\n            while True:\n                progress = False\n                for q in range(4):\n                    while ptr[q] < len(ranks[q]) and ranks[q][ptr[q]] in used:\n                        ptr[q] += 1\n                    if ptr[q] < len(ranks[q]):\n                        i = ranks[q][ptr[q]]\n                        ptr[q] += 1\n                        used.add(i)\n                        order.append(i)\n                        progress = True\n                if not progress:\n                    break\n                if len(order) > 200000:\n                    break\n            order = dedup_order(order, d)\n            sel, tot = cut(order, tl, ids)\n        elif mode == \"quota\":\n            # partition candidates by their best-matching register (on z-scored\n            # per-register scores), then fill an equal TOKEN quota per register so\n            # every prefix of the list matches the target's 4-way composition.\n            Z = np.zeros((len(cand), 4), np.float32)\n            for q in range(4):\n                v = S[cand, q]\n                Z[:, q] = (v - v.mean()) / (v.std() + 1e-9)\n            best = Z.argmax(1)\n            groups = []\n            for q in range(4):\n                g = cand[best == q]\n                g = g[np.argsort(-Z[best == q, q])]\n                groups.append(dedup_order(g, d))\n                print(f\"  register {q}: {len(g):,} docs\")\n            ptr, tokq, order = [0] * 4, [0] * 4, []\n            while True:\n                q = int(np.argmin(tokq))\n                if ptr[q] >= len(groups[q]):\n                    tokq[q] = float(\"inf\")\n                    if all(np.isinf(tokq)):\n                        break\n                    continue\n                i = groups[q][ptr[q]]; ptr[q] += 1\n                order.append(i); tokq[q] += int(tl[i]) + 1\n                if sum(t for t in tokq if not np.isinf(t)) >= NEED:\n                    break\n            order = dedup_order(order, d)\n            sel, tot = cut(order, tl, ids)\n        else:\n            raise SystemExit(\"bad mode\")\n\n    json.dump(sel, open(out, \"w\"))\n    print(f\"{mode}: {len(sel):,} ids, {tot:,} tokens -> {out}\")\n\n\nif __name__ == \"__main__\":\n    main()\n", "userModified": false}

balanced bw=0.25 T=0.0 NREG=4 nodedup=0: 47,782 ids, 47,734,185 tokens -> /tmp/sel_b025.json
balanced bw=0.0 T=0.0 NREG=4 nodedup=0: 47,701 ids, 47,751,270 tokens -> /tmp/sel_b000.json
balanced bw=0.5 T=0.0 NREG=8 nodedup=0: 47,383 ids, 46,753,928 tokens -> /tmp/sel_r8.json
balanced bw=0.5 T=0.0 NREG=4 nodedup=1: 44,456 ids, 48,000,962 tokens -> /tmp/sel_nodd.json
global bw=0.5 T=0.0 NREG=4 nodedup=0: 51,582 ids, 48,001,087 tokens -> /tmp/sel_g0.json
launched
4178

[stdout]
balanced bw=0.25 T=0.0 NREG=4 nodedup=0: 47,782 ids, 47,734,185 tokens -> /tmp/sel_b025.json
balanced bw=0.0 T=0.0 NREG=4 nodedup=0: 47,701 ids, 47,751,270 tokens -> /tmp/sel_b000.json
balanced bw=0.5 T=0.0 NREG=8 nodedup=0: 47,383 ids, 46,753,928 tokens -> /tmp/sel_r8.json
balanced bw=0.5 T=0.0 NREG=4 nodedup=1: 44,456 ids, 48,000,962 tokens -> /tmp/sel_nodd.json
global bw=0.5 T=0.0 NREG=4 nodedup=0: 51,582 ids, 48,001,087 tokens -> /tmp/sel_g0.json
launched
4178
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/submission/curate.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/submission/curate.py", "content": "\"\"\"Curate /workspace/data/pool.jsonl for a 12M-token pretraining budget.\n\nCRITERION  (one stated rule, no hand-picked ids)\n-----------------------------------------------\nThe disclosed evaluation target is a broad, equal-parts blend of four registers\n(encyclopedic / high-quality web prose / news / technical Q&A). A sample of it is\ngiven to us as a token stream (data/multi_dev.npy). We therefore define document\nquality *relative to that target* and select for it in two steps:\n\n1. RELEVANCE.  Split the target stream into NREG equal contiguous parts, one per\n   register, and fit a word+bigram unigram model p_q to each. Fit a background\n   model p_bg to a uniform sample of the raw pool. Score every pool document\n   against every register with the length-normalised log importance ratio\n\n       s_q(d) = (1/|d|) * sum_{f in ngrams(d)} log( LAM * p_q(f)/p_bg(f) + 1-LAM )\n\n   The target model is interpolated onto the background (Jelinek-Mercer), so an\n   n-gram the target never uses contributes exactly log(1-LAM) < 0. Without that\n   interpolation, add-alpha smoothing gives *unseen* n-grams a positive ratio\n   (the target's count denominator is ~60x smaller than the pool's) and the\n   ranking fills with boilerplate and link spam.\n\n   Tokens are alnum runs plus every individual non-alphanumeric character, so\n   markup and punctuation style (HTML tags, wikitext \" @-@ \", spacing) is part of\n   the signal rather than invisible to it.\n\n2. COMPOSITION.  Rank documents separately per register, then merge the NREG\n   rankings round-robin *by token count*, skipping documents already taken and\n   exact/near-duplicates (2-hash MinHash over 8-word shingles). Round-robin by\n   tokens means every prefix of the emitted list — including the exact prefix the\n   12M-token budget happens to cut at — carries an equal token share from each\n   register, matching the equally-weighted evaluation blend.\n\nDeliberately NOT done: no hand-written prose/quality filter. Gopher/C4-style\nfilters (min stopword fraction, alphabetic fraction, line-length, symbol ratios)\nwere measured and made held-out perplexity *worse* (336 -> 380 dev PPL, and a\nstricter version -> 440), because this target includes registers such as\nmarkup-bearing technical Q&A that prose heuristics discard. The importance ratio\nalready encodes what \"good\" means for this target; the heuristics only fight it.\n\nOutput: /workspace/submission/selection.json — pool ids in priority order.\nRun:    python3 submission/curate.py        (~6 min on 16 cores, CPU only)\n\"\"\"\nimport json, math, os, re, sys, zlib\nfrom collections import Counter\nfrom multiprocessing import Pool as MPPool\n\nimport numpy as np\n\nPOOL = \"/workspace/data/pool.jsonl\"\nTARGET = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\nCACHE = \"/workspace/cache\"\n\nNREG = 4            # target registers = equal contiguous parts of the target stream\nLAM = 0.9           # Jelinek-Mercer weight on the target model vs the background\nINTERP = 0.6        # own-register counts vs pooled-target counts within p_q\nBIGRAM_W = 0.25     # weight of the bigram score relative to the unigram score\nBG_SAMPLE = 16000   # pool docs used to estimate the background model\nMINW = 12           # only guard: skip degenerate near-empty documents\nBUDGET = 12_000_000\nEMIT = 4 * BUDGET   # emit 4x the budget in tokens so the cut is never starved\nNPROC = 14\n\nTOKRE = re.compile(r\"[a-z0-9']+|[^\\sa-z0-9']\")\n\n\ndef toks(text):\n    return TOKRE.findall(text.lower())\n\n\ndef accum(text, uni, big):\n    t = toks(text)\n    uni.update(t)\n    big.update(map(\" \".join, zip(t, t[1:])))\n    return len(t)\n\n\n# ---------------------------------------------------------------- 1. models\ndef build_target():\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    arr = np.load(TARGET)\n    span = len(arr) // NREG\n    uni, big, n = [], [], []\n    for i in range(NREG):\n        txt = tok.decode(arr[i * span:(i + 1) * span].tolist())\n        u, b = Counter(), Counter()\n        n.append(accum(txt.replace(\"<|endoftext|>\", \"\\n\"), u, b))\n        uni.append(u); big.append(b)\n    return uni, big, n\n\n\ndef build_bg(nlines):\n    uni, big, n = Counter(), Counter(), 0\n    step = max(1, nlines // BG_SAMPLE)\n    with open(POOL) as f:\n        for i, line in enumerate(f):\n            if i % step == 0:\n                n += accum(json.loads(line)[\"text\"], uni, big)\n    return uni, big, n\n\n\ndef lr_table(tq, tg, nq, ng, bg, nbg, vocab, interp):\n    \"\"\"log( LAM * p_target(f)/p_bg(f) + (1-LAM) ) for f in vocab, then the OOV value.\"\"\"\n    dbg = nbg + 0.5 * (len(vocab) + 1)\n    out = np.empty(len(vocab) + 1, dtype=np.float32)\n    for i, f in enumerate(vocab):\n        pt = interp * tq.get(f, 0) / nq + (1 - interp) * tg.get(f, 0) / ng\n        pb = (bg.get(f, 0) + 0.5) / dbg\n        out[i] = math.log(LAM * pt / pb + (1 - LAM))\n    out[-1] = math.log(1 - LAM)\n    return out\n\n\n# ---------------------------------------------------------------- 2. scoring\nG = {}\n\n\ndef init(ui, bi, LRU, LRB):\n    G.update(ui=ui, bi=bi, lru=LRU, lrb=LRB)\n\n\ndef sig(t):\n    \"\"\"2-hash MinHash over 8-word shingles -> near-duplicate band key\"\"\"\n    sh = [zlib.crc32(\" \".join(t[i:i + 8]).encode()) for i in range(0, len(t) - 8, 3)]\n    if not sh:\n        return 0\n    return (min(sh) << 32) | min(x ^ 0x5bf03635 for x in sh)\n\n\ndef score_chunk(lines):\n    ui, bi, lru, lrb = G[\"ui\"], G[\"bi\"], G[\"lru\"], G[\"lrb\"]\n    uu, ub = len(lru) - 1, len(lrb) - 1\n    rows = []\n    for line in lines:\n        r = json.loads(line)\n        t = toks(r[\"text\"])\n        n = len(t)\n        if n < MINW:\n            rows.append((r[\"id\"], n, [-9.0] * (NREG + 1), 0, 0))\n            continue\n        cu = Counter(t)\n        cb = Counter(map(\" \".join, zip(t, t[1:])))\n        iu = np.fromiter((ui.get(w, uu) for w in cu), np.int32, len(cu))\n        wu = np.fromiter(cu.values(), np.float32, len(cu))\n        ib = np.fromiter((bi.get(w, ub) for w in cb), np.int32, len(cb))\n        wb = np.fromiter(cb.values(), np.float32, len(cb))\n        s = (lru[iu] * wu[:, None]).sum(0) / n \\\n            + BIGRAM_W * (lrb[ib] * wb[:, None]).sum(0) / (n - 1)\n        words = [w for w in t if w[:1].isalpha()]\n        rows.append((r[\"id\"], n, s.tolist(),\n                     zlib.crc32(\" \".join(words).encode()), sig(t)))\n    return rows\n\n\n# ---------------------------------------------------------------- 3. token lengths\ndef token_lengths(nlines):\n    \"\"\"exact GPT-2 token count per document (the budget is counted in these)\"\"\"\n    cached = f\"{CACHE}/tok_len.npy\"\n    if os.path.exists(cached):\n        return np.load(cached)\n    os.makedirs(CACHE, exist_ok=True)\n\n    def w(lines):\n        from transformers import AutoTokenizer\n        if \"t\" not in G:\n            G[\"t\"] = AutoTokenizer.from_pretrained(\"gpt2\")\n        recs = [json.loads(l) for l in lines]\n        enc = G[\"t\"]([r[\"text\"] for r in recs], add_special_tokens=False).input_ids\n        return [(r[\"id\"], len(e)) for r, e in zip(recs, enc)]\n\n    os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n    lines = open(POOL).readlines()\n    lens = np.zeros(nlines, np.int32)\n    with MPPool(NPROC) as p:\n        for out in p.imap(w, [lines[i:i + 256] for i in range(0, len(lines), 256)]):\n            for i, L in out:\n                lens[i] = L\n    np.save(cached, lens)\n    return lens\n\n\n# ---------------------------------------------------------------- 4. selection\ndef main():\n    lines = open(POOL).readlines()\n    N = len(lines)\n    print(f\"pool: {N:,} docs\", flush=True)\n\n    tuni, tbig, tn = build_target()\n    guni, gbig = Counter(), Counter()\n    for u, b in zip(tuni, tbig):\n        guni.update(u); gbig.update(b)\n    gn = sum(tn)\n    print(f\"target words per register: {tn}\", flush=True)\n\n    buni, bbig, bn = build_bg(N)\n    print(f\"background words: {bn:,}\", flush=True)\n\n    vu = sorted(set(guni) | {w for w, c in buni.items() if c >= 5})\n    vb = sorted(set(gbig) | {w for w, c in bbig.items() if c >= 10})\n    ui = {w: i for i, w in enumerate(vu)}\n    bi = {w: i for i, w in enumerate(vb)}\n    print(f\"vocab: {len(vu):,} unigrams, {len(vb):,} bigrams\", flush=True)\n\n    LRU = np.stack([lr_table(tuni[q], guni, tn[q], gn, buni, bn, vu, INTERP)\n                    for q in range(NREG)]\n                   + [lr_table(guni, guni, gn, gn, buni, bn, vu, 1.0)], axis=1)\n    LRB = np.stack([lr_table(tbig[q], gbig, tn[q], gn, bbig, bn, vb, INTERP)\n                    for q in range(NREG)]\n                   + [lr_table(gbig, gbig, gn, gn, bbig, bn, vb, 1.0)], axis=1)\n\n    rows = []\n    chunks = [lines[i:i + 400] for i in range(0, N, 400)]\n    with MPPool(NPROC, initializer=init, initargs=(ui, bi, LRU, LRB)) as p:\n        for out in p.imap(score_chunk, chunks):\n            rows.extend(out)\n    rows.sort(key=lambda r: r[0])\n    ids = np.array([r[0] for r in rows], np.int32)\n    nw = np.array([r[1] for r in rows], np.int32)\n    S = np.array([r[2] for r in rows], np.float32)\n    exact = np.array([r[3] for r in rows], np.int64)\n    near = np.array([r[4] for r in rows], np.uint64)\n    print(\"scored\", len(ids), flush=True)\n\n    tl = token_lengths(N)\n    cand = np.flatnonzero(nw >= MINW)\n\n    def dedup(order):\n        se, sn, out = set(), set(), []\n        for i in order:\n            e, n = int(exact[i]), int(near[i])\n            if e in se or (n and n in sn):\n                continue\n            se.add(e)\n            if n:\n                sn.add(n)\n            out.append(i)\n        return out\n\n    ranks = [dedup(cand[np.argsort(-S[cand, q])]) for q in range(NREG)]\n\n    # round-robin by tokens: always extend the register with the fewest tokens so far\n    used, order, ptr, tokq = set(), [], [0] * NREG, [0.0] * NREG\n    while True:\n        q = int(np.argmin(tokq))\n        while ptr[q] < len(ranks[q]) and ranks[q][ptr[q]] in used:\n            ptr[q] += 1\n        if ptr[q] >= len(ranks[q]):\n            tokq[q] = float(\"inf\")\n            if all(np.isinf(x) for x in tokq):\n                break\n            continue\n        i = ranks[q][ptr[q]]; ptr[q] += 1\n        used.add(i); order.append(i); tokq[q] += int(tl[i]) + 1\n        if sum(x for x in tokq if not np.isinf(x)) >= EMIT:\n            break\n\n    sel, tot = [], 0\n    for i in dedup(order):\n        sel.append(int(ids[i]))\n        tot += int(tl[i]) + 1\n        if tot >= EMIT:\n            break\n    assert len(sel) == len(set(sel)) and min(sel) >= 0 and max(sel) < N\n    os.makedirs(os.path.dirname(OUT), exist_ok=True)\n    json.dump(sel, open(OUT, \"w\"))\n    print(f\"wrote {len(sel):,} ids covering {tot:,} tokens \"\n          f\"({tot / BUDGET:.1f}x the {BUDGET:,}-token budget) -> {OUT}\")\n\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [], "originalFile": null, "userModified": false}

b025: b000: r8: nodd: g0: 4098

[stdout]
b025: b000: r8: nodd: g0: 4098
[metadata] {"noOutputExpected": false}

b025: b000: r8: nodd: g0:

[stdout]
b025: b000: r8: nodd: g0:
[metadata] {"noOutputExpected": false}

b025: HELDOUT_PPL 341.279  loss 5.8327  train_s 293  /tmp/tr_b025.npy
b000: HELDOUT_PPL 344.813  loss 5.8430  train_s 293  /tmp/tr_b000.npy
r8: HELDOUT_PPL 336.815  loss 5.8195  train_s 293  /tmp/tr_r8.npy
nodd: HELDOUT_PPL 344.649  loss 5.8425  train_s 293  /tmp/tr_nodd.npy
g0: HELDOUT_PPL 357.703  loss 5.8797  train_s 293  /tmp/tr_g0.npy
3868

[stdout]
b025: HELDOUT_PPL 341.279  loss 5.8327  train_s 293  /tmp/tr_b025.npy
b000: HELDOUT_PPL 344.813  loss 5.8430  train_s 293  /tmp/tr_b000.npy
r8: HELDOUT_PPL 336.815  loss 5.8195  train_s 293  /tmp/tr_r8.npy
nodd: HELDOUT_PPL 344.649  loss 5.8425  train_s 293  /tmp/tr_nodd.npy
g0: HELDOUT_PPL 357.703  loss 5.8797  train_s 293  /tmp/tr_g0.npy
3868
[metadata] {"noOutputExpected": false}

selection: docs in budget 12168 mean tok/doc 985 median 499
target: mean tok/doc 369
25:LAM = float(os.environ.get("LAM", 0.9))           # Jelinek-Mercer: weight on the target model vs background
26:INTERP = float(os.environ.get("INTERP", 0.6))        # weight on own-register counts, rest on pooled-target counts
launched rescores

[stdout]
selection: docs in budget 12168 mean tok/doc 985 median 499
target: mean tok/doc 369
25:LAM = float(os.environ.get("LAM", 0.9))           # Jelinek-Mercer: weight on the target model vs background
26:INTERP = float(os.environ.get("INTERP", 0.6))        # weight on own-register counts, rest on pooled-target counts
launched rescores
[metadata] {"noOutputExpected": false}

saved 182016
saved 182016
-rw-r--r-- 1 root root 17475472 Aug  4 01:18 cache/scores.npz
-rw-r--r-- 1 root root 23299984 Aug  4 01:26 cache/scores8.npz
-rw-r--r-- 1 root root 17475472 Aug  4 01:37 cache/scores_i85.npz
-rw-r--r-- 1 root root 17475472 Aug  4 01:37 cache/scores_l95.npz

[stdout]
saved 182016
saved 182016
-rw-r--r-- 1 root root 17475472 Aug  4 01:18 cache/scores.npz
-rw-r--r-- 1 root root 23299984 Aug  4 01:26 cache/scores8.npz
-rw-r--r-- 1 root root 17475472 Aug  4 01:37 cache/scores_i85.npz
-rw-r--r-- 1 root root 17475472 Aug  4 01:37 cache/scores_l95.npz
[metadata] {"noOutputExpected": false}

balanced bw=0.75 T=0.0 NREG=4 nodedup=0: 48,002 ids, 47,705,115 tokens -> /tmp/sel_bw075.json
balanced bw=0.5 T=0.0 NREG=4 nodedup=0: 48,028 ids, 47,746,312 tokens -> /tmp/sel_l95.json
balanced bw=0.5 T=0.0 NREG=4 nodedup=0: 47,946 ids, 47,826,981 tokens -> /tmp/sel_i85.json
launched 3
3708

[stdout]
balanced bw=0.75 T=0.0 NREG=4 nodedup=0: 48,002 ids, 47,705,115 tokens -> /tmp/sel_bw075.json
balanced bw=0.5 T=0.0 NREG=4 nodedup=0: 48,028 ids, 47,746,312 tokens -> /tmp/sel_l95.json
balanced bw=0.5 T=0.0 NREG=4 nodedup=0: 47,946 ids, 47,826,981 tokens -> /tmp/sel_i85.json
launched 3
3708
[metadata] {"noOutputExpected": false}

0 250000
1 250000
2 250000
3 250000
bw075: l95: i85:

[stdout]
0 250000
1 250000
2 250000
3 250000
bw075: l95: i85:
[metadata] {"noOutputExpected": false}

bw075: l95: i85: 3548

[stdout]
bw075: l95: i85: 3548
[metadata] {"noOutputExpected": false}

bw075: HELDOUT_PPL 337.681  loss 5.8221  train_s 176  /tmp/tr_bw075.npy
l95: HELDOUT_PPL 343.905  loss 5.8404  train_s 176  /tmp/tr_l95.npy
i85: HELDOUT_PPL 353.695  loss 5.8684  train_s 176  /tmp/tr_i85.npy

[stdout]
bw075: HELDOUT_PPL 337.681  loss 5.8221  train_s 176  /tmp/tr_bw075.npy
l95: HELDOUT_PPL 343.905  loss 5.8404  train_s 176  /tmp/tr_l95.npy
i85: HELDOUT_PPL 353.695  loss 5.8684  train_s 176  /tmp/tr_i85.npy
[metadata] {"noOutputExpected": false}