claude-opus-5

bal2: (running)
bal12: (running)
nogate: (running)
4308

[stdout]
bal2: (running)
bal12: (running)
nogate: (running)
4308
[metadata] {"noOutputExpected": false}

random: HELDOUT_PPL 469.932  loss 6.1526  train_s 177  /tmp/tr_random.npy
global: HELDOUT_PPL 394.337  loss 5.9772  train_s 178  /tmp/tr_global.npy
balanced: HELDOUT_PPL 355.623  loss 5.8739  train_s 178  /tmp/tr_balanced.npy
bal2: HELDOUT_PPL 358.890  loss 5.8830  train_s 177  /tmp/tr_bal2.npy
bal12: HELDOUT_PPL 361.275  loss 5.8896  train_s 176  /tmp/tr_bal12.npy
nogate: HELDOUT_PPL 381.166  loss 5.9432  train_s 177  /tmp/tr_nogate.npy
4088

[stdout]
random: HELDOUT_PPL 469.932  loss 6.1526  train_s 177  /tmp/tr_random.npy
global: HELDOUT_PPL 394.337  loss 5.9772  train_s 178  /tmp/tr_global.npy
balanced: HELDOUT_PPL 355.623  loss 5.8739  train_s 178  /tmp/tr_balanced.npy
bal2: HELDOUT_PPL 358.890  loss 5.8830  train_s 177  /tmp/tr_bal2.npy
bal12: HELDOUT_PPL 361.275  loss 5.8896  train_s 176  /tmp/tr_bal12.npy
nogate: HELDOUT_PPL 381.166  loss 5.9432  train_s 177  /tmp/tr_nogate.npy
4088
[metadata] {"noOutputExpected": false}

--- pool docs containing wikitext ' @-@ ' artifact:
0
--- dev docs with artifact:
863 / 2705
--- target doc token-length distribution vs my selection:
target doc tokens: p5,25,50,75,95 = [  44.  109.  192.  380. 1138.] mean 369.0
my selection first 12M: ndocs 9831 doc tokens p5,25,50,75,95 = [  72.  271.  726. 1475. 4037.] mean 1221.0

[stdout]
--- pool docs containing wikitext ' @-@ ' artifact:
0
--- dev docs with artifact:
863 / 2705
--- target doc token-length distribution vs my selection:
target doc tokens: p5,25,50,75,95 = [  44.  109.  192.  380. 1138.] mean 369.0
my selection first 12M: ndocs 9831 doc tokens p5,25,50,75,95 = [  72.  271.  726. 1475. 4037.] mean 1221.0
[metadata] {"noOutputExpected": false}

ok
36:ap.add_argument("--lenmatch", action="store_true",
76:dev_len = np.array([len(d) for d in _docs if len(_tk.decode(d).split()) >= 20], np.float64)
77:assert len(dev_len) == len(dev_clus), (len(dev_len), len(dev_clus))
80:tw = np.array([dev_len[dev_clus == k].sum() for k in range(K)], np.float64)
90:LEN_EDGES = [0, 64, 128, 256, 512, 1024, 2048, 1 << 30]
91:if a.tight:

[stdout]
ok
36:ap.add_argument("--lenmatch", action="store_true",
76:dev_len = np.array([len(d) for d in _docs if len(_tk.decode(d).split()) >= 20], np.float64)
77:assert len(dev_len) == len(dev_clus), (len(dev_len), len(dev_clus))
80:tw = np.array([dev_len[dev_clus == k].sum() for k in range(K)], np.float64)
90:LEN_EDGES = [0, 64, 128, 256, 512, 1024, 2048, 1 << 30]
91:if a.tight:
[metadata] {"noOutputExpected": false}

168	if a.variant == "global":
169	    keep = np.zeros(N, bool)
170	    for k in range(K):
171	        keep |= masks[k]
172	    ordr = np.argsort(-np.where(keep, gs, -1e9))
173	    sel, tot = [], 0.0
174	    for i in ordr:
175	        if not keep[i]:
176	            break
177	        sel.append(int(ids[i])); tot += est[i] + 1
178	        if tot > OVERFILL * BUDGET:
179	            break
180	else:
181	    # per-register queues, quota = target token share, round-robin so every prefix is mixed
182	    queues = []
183	    for k in range(K):
184	        m = masks[k] & (assign == k)
185	        idx = np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())]
186	        queues.append(list(idx))
187	        print(f"  reg{k}: {m.sum()} eligible docs, {est[idx].sum()/1e6:.1f}M tokens available,"
188	              f" quota {tok_share[k]*BUDGET/1e6:.2f}M")
189	    want = tok_share * OVERFILL * BUDGET
190	    got = np.zeros(K); sel, ptr = [], [0] * K
191	    while True:
192	        # serve the register that is furthest behind its quota
193	        deficit = np.array([(got[k] / want[k]) if ptr[k] < len(queues[k]) else 9e9 for k in range(K)])
194	        k = int(deficit.argmin())
195	        if deficit[k] >= 9e9:
196	            break
197	        i = queues[k][ptr[k]]; ptr[k] += 1
198	        sel.append(int(ids[i])); got[k] += est[i] + 1
199	        if got.sum() >= OVERFILL * BUDGET:
200	            break
201	    print("emitted token mix:", (got / got.sum()).round(3), f"total {got.sum()/1e6:.1f}M")
202	
203	assert len(sel) == len(set(sel)), "duplicate ids"
204	assert all(0 <= i < 182016 for i in sel), "id out of range"
205	os.makedirs(os.path.dirname(a.out), exist_ok=True)
206	json.dump(sel, open(a.out, "w"))
207	first12 = np.cumsum([est[row_of[i]] + 1 for i in sel])
208	n12 = int(np.searchsorted(first12, BUDGET)) + 1
209	print(f"wrote {len(sel)} ids -> {a.out}  (~{first12[-1]/1e6:.1f}M est tokens; "
210	      f"first ~{n12} ids fill the {BUDGET/1e6:.0f}M budget)")
211

[metadata] {"type": "text", "file": {"filePath": "/workspace/curate.py", "content": "if a.variant == \"global\":\n    keep = np.zeros(N, bool)\n    for k in range(K):\n        keep |= masks[k]\n    ordr = np.argsort(-np.where(keep, gs, -1e9))\n    sel, tot = [], 0.0\n    for i in ordr:\n        if not keep[i]:\n            break\n        sel.append(int(ids[i])); tot += est[i] + 1\n        if tot > OVERFILL * BUDGET:\n            break\nelse:\n    # per-register queues, quota = target token share, round-robin so every prefix is mixed\n    queues = []\n    for k in range(K):\n        m = masks[k] & (assign == k)\n        idx = np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())]\n        queues.append(list(idx))\n        print(f\"  reg{k}: {m.sum()} eligible docs, {est[idx].sum()/1e6:.1f}M tokens available,\"\n              f\" quota {tok_share[k]*BUDGET/1e6:.2f}M\")\n    want = tok_share * OVERFILL * BUDGET\n    got = np.zeros(K); sel, ptr = [], [0] * K\n    while True:\n        # serve the register that is furthest behind its quota\n        deficit = np.array([(got[k] / want[k]) if ptr[k] < len(queues[k]) else 9e9 for k in range(K)])\n        k = int(deficit.argmin())\n        if deficit[k] >= 9e9:\n            break\n        i = queues[k][ptr[k]]; ptr[k] += 1\n        sel.append(int(ids[i])); got[k] += est[i] + 1\n        if got.sum() >= OVERFILL * BUDGET:\n            break\n    print(\"emitted token mix:\", (got / got.sum()).round(3), f\"total {got.sum()/1e6:.1f}M\")\n\nassert len(sel) == len(set(sel)), \"duplicate ids\"\nassert all(0 <= i < 182016 for i in sel), \"id out of range\"\nos.makedirs(os.path.dirname(a.out), exist_ok=True)\njson.dump(sel, open(a.out, \"w\"))\nfirst12 = np.cumsum([est[row_of[i]] + 1 for i in sel])\nn12 = int(np.searchsorted(first12, BUDGET)) + 1\nprint(f\"wrote {len(sel)} ids -> {a.out}  (~{first12[-1]/1e6:.1f}M est tokens; \"\n      f\"first ~{n12} ids fill the {BUDGET/1e6:.0f}M budget)\")\n", "numLines": 44, "startLine": 168, "totalLines": 211}}

The file /workspace/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/curate.py", "oldString": "else:\n    # per-register queues, quota = target token share, round-robin so every prefix is mixed\n    queues = []\n    for k in range(K):\n        m = masks[k] & (assign == k)\n        idx = np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())]\n        queues.append(list(idx))\n        print(f\"  reg{k}: {m.sum()} eligible docs, {est[idx].sum()/1e6:.1f}M tokens available,\"\n              f\" quota {tok_share[k]*BUDGET/1e6:.2f}M\")\n    want = tok_share * OVERFILL * BUDGET\n    got = np.zeros(K); sel, ptr = [], [0] * K\n    while True:\n        # serve the register that is furthest behind its quota\n        deficit = np.array([(got[k] / want[k]) if ptr[k] < len(queues[k]) else 9e9 for k in range(K)])\n        k = int(deficit.argmin())\n        if deficit[k] >= 9e9:\n            break\n        i = queues[k][ptr[k]]; ptr[k] += 1\n        sel.append(int(ids[i])); got[k] += est[i] + 1\n        if got.sum() >= OVERFILL * BUDGET:\n            break\n    print(\"emitted token mix:\", (got / got.sum()).round(3), f\"total {got.sum()/1e6:.1f}M\")", "newString": "else:\n    # Cells = register x (doc-length bucket).  Quota per cell = that cell's share of the TARGET's\n    # token mass, so the selection reproduces both the register mix and the document-length\n    # distribution of the target.  Filled round-robin by relative deficit, therefore every prefix\n    # of the emitted list already carries the full mixture.\n    nb = len(LEN_EDGES) - 1 if a.lenmatch else 1\n    if a.lenmatch:\n        db = np.clip(np.digitize(dev_len, LEN_EDGES) - 1, 0, nb - 1)\n        pb = np.clip(np.digitize(est, LEN_EDGES) - 1, 0, nb - 1)\n    else:\n        db = np.zeros(len(dev_len), int); pb = np.zeros(N, int)\n    cells, want = [], []\n    for k in range(K):\n        for b in range(nb):\n            w = dev_len[(dev_clus == k) & (db == b)].sum()\n            if w <= 0:\n                continue\n            m = masks[k] & (assign == k) & (pb == b)\n            if m.sum() == 0:\n                continue\n            cells.append((k, b, list(np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())])))\n            want.append(w)\n    want = np.array(want, np.float64)\n    lost = 1.0 - want.sum() / dev_len.sum()      # target mass with no eligible pool supply\n    want = want / want.sum() * OVERFILL * BUDGET\n    print(f\"  {len(cells)} register x length cells; unmatched target mass {lost:.3f}\")\n    for (k, b, q), w in zip(cells, want):\n        print(f\"   reg{k} len[{LEN_EDGES[b]},{LEN_EDGES[b+1] if b+1<len(LEN_EDGES) else 0}): \"\n              f\"{len(q):6d} docs, {est[q].sum()/1e6:6.1f}M avail, quota {w/OVERFILL/1e6:.2f}M\")\n    got = np.zeros(len(cells)); sel, ptr = [], [0] * len(cells)\n    while True:\n        deficit = np.array([(got[c] / want[c]) if ptr[c] < len(cells[c][2]) else 9e9\n                            for c in range(len(cells))])\n        c = int(deficit.argmin())\n        if deficit[c] >= 9e9:\n            break\n        i = cells[c][2][ptr[c]]; ptr[c] += 1\n        sel.append(int(ids[i])); got[c] += est[i] + 1\n        if got.sum() >= OVERFILL * BUDGET:\n            break\n    mix = np.zeros(K)\n    for (k, b, q), g in zip(cells, got):\n        mix[k] += g\n    print(\"emitted register mix:\", (mix / mix.sum()).round(3), \"target:\", tok_share.round(3),\n          f\"total {got.sum()/1e6:.1f}M\")", "originalFile": "\"\"\"Curate /workspace/data/pool.jsonl -> /workspace/submission/selection.json\n\nSTATED CRITERION\n================\nKeep a pool document iff its *surface statistics* fall inside the range that the disclosed\ntarget distribution itself occupies (a per-register acceptance box), then rank the survivors by\n*target-likeness* -- a length-shrunk, length-normalised hashed-unigram log-likelihood ratio\nlog p_target(w) / p_pool(w) (DSIR-style importance scoring) -- and fill the token budget with\nper-register quotas equal to the target's own register token mix, emitted round-robin so that\nevery prefix of the priority list carries the full multi-domain mixture.\n\nThe target distribution is not given as text, but `data/multi_dev.npy` is 1M GPT-2 tokens drawn\nfrom it; decoding that array back to text yields ~2.7k in-domain sample documents, which is the\nonly supervision used. No external labels, no hand-picked ids.\n\nPipeline stages (each cached under /tmp/feat):\n  1. feat.py   - hashed word-unigram counts + 22 quality heuristics for all 182,016 docs\n  2. score.py  - per-register DSIR log-ratio scores, spherical k-means registers, held-out AUC\n  3. this file - dedup, acceptance box, quotas, round-robin priority order\n\nUsage: python curate.py [--variant balanced|global|nogate] [--out PATH]\n\"\"\"\nimport argparse, hashlib, json, os, re, subprocess, sys\nimport numpy as np\n\nPOOL = \"/workspace/data/pool.jsonl\"\nF = \"/tmp/feat/\"\nBUDGET = 12_000_000\nOVERFILL = 3.0            # emit ~3x the budget so the pipeline never runs short\nMIN_WORDS = 80            # window-size floor: docs shorter than this add mostly EOS boundaries\nSHRINK = 250.0            # empirical-Bayes shrinkage constant, in words\n\nap = argparse.ArgumentParser()\nap.add_argument(\"--variant\", default=\"balanced\", choices=[\"balanced\", \"global\", \"nogate\"])\nap.add_argument(\"--box\", default=\"global\", choices=[\"global\", \"register\"])\nap.add_argument(\"--lenmatch\", action=\"store_true\",\n                help=\"match the target doc-length distribution as well as the register mix\")\nap.add_argument(\"--tight\", action=\"store_true\", help=\"tighter acceptance box (5-95pct, pad .05)\")\nap.add_argument(\"--min_words\", type=int, default=MIN_WORDS)\nap.add_argument(\"--out\", default=\"/workspace/submission/selection.json\")\nap.add_argument(\"--k\", type=int, default=5)\na = ap.parse_args()\n\n# ---------------------------------------------------------------- stages 1-2 (cached)\nif not os.path.exists(F + \"heur.npy\"):\n    subprocess.run([sys.executable, \"/workspace/feat.py\"], check=True)\nif not os.path.exists(F + f\"scores_K{a.k}.npz\"):\n    subprocess.run([sys.executable, \"/workspace/score.py\", str(a.k)], check=True)\n\nz = np.load(F + f\"scores_K{a.k}.npz\")\nids, all_s, LLR = z[\"ids\"], z[\"all_s\"], z[\"LLR\"]\ndev_clus, auc = z[\"dev_clus\"], float(z[\"auc\"])\nK, N = LLR.shape\nheur = np.load(F + \"heur.npy\")\nnames = json.load(open(F + \"heur_names.json\"))\nH = {n: heur[:, i] for i, n in enumerate(names)}\nprint(f\"pool {N} docs, {K} registers, scorer AUC {auc:.4f}\")\n\n# ---------------------------------------------------------------- target-side statistics\nsys.path.insert(0, \"/workspace\")\nfrom feat import feats, HEUR_NAMES\n\ndev_texts = [t for t in json.load(open(\"/tmp/dev_texts.json\")) if len(t.split()) >= 20]\nDH = np.stack([feats(t)[0] for t in dev_texts])\nDHd = {n: DH[:, i] for i, n in enumerate(HEUR_NAMES)}\n# exact GPT-2 token length of every target doc, aligned with dev_clus (same >=20-word filter)\n_a = np.load(\"/workspace/data/multi_dev.npy\")\n_b = np.where(_a == 50256)[0]\n_docs, _p = [], 0\nfor _x in _b:\n    _docs.append(_a[_p:_x]); _p = _x + 1\nif _p < len(_a):\n    _docs.append(_a[_p:])\nfrom transformers import AutoTokenizer as _AT\n_tk = _AT.from_pretrained(\"gpt2\")\ndev_len = np.array([len(d) for d in _docs if len(_tk.decode(d).split()) >= 20], np.float64)\nassert len(dev_len) == len(dev_clus), (len(dev_len), len(dev_clus))\n\n# register token mix of the target (tokens, not doc counts -- that is what fills the budget)\ntw = np.array([dev_len[dev_clus == k].sum() for k in range(K)], np.float64)\ntok_share = tw / tw.sum()\nprint(\"target register token shares:\", tok_share.round(3))\n\n# acceptance box: statistics whose target range is a meaningful quality signal\nBOX = [\"stop_frac\", \"ttr\", \"frac_alpha\", \"mean_wlen\", \"frac_punct\", \"frac_digit\",\n       \"frac_upper\", \"rep5\", \"dup_line_frac\", \"frac_nonascii\", \"words_per_sent\",\n       \"line_end_punct\", \"frac_short_lines\"]\nLO, HI = 2.0, 98.0        # keep the central 96% of the target's own range per register\nPAD = 0.15\nLEN_EDGES = [0, 64, 128, 256, 512, 1024, 2048, 1 << 30]\nif a.tight:\n    LO, HI, PAD = 5.0, 95.0, 0.05\n\n\ndef gate(k=None):\n    \"\"\"Boolean mask: pool docs inside the target's surface-statistic acceptance box.\n\n    k=None uses the box of the whole target sample (all ~2.7k docs).  Per-register boxes are\n    available but are estimated from as few as ~150 docs, which makes them far too tight: they\n    starve the register quotas rather than filtering quality.  Global box is the default.\n    \"\"\"\n    m = (H[\"n_words\"] >= a.min_words)\n    if a.variant == \"nogate\":\n        return m\n    d = np.ones(len(dev_clus), bool) if k is None else (dev_clus == k)\n    if d.sum() < 300:\n        d = np.ones(len(dev_clus), bool)\n    for s in BOX:\n        lo, hi = np.percentile(DHd[s][d], [LO, HI])\n        pad = PAD * (hi - lo) + 1e-6      # tolerance: the target sample is only ~500 docs/register\n        m &= (H[s] >= lo - pad) & (H[s] <= hi + pad)\n    m &= H[\"rep5\"] <= 0.30                 # global junk floors\n    m &= H[\"dup_line_frac\"] <= 0.35\n    m &= H[\"boiler\"] <= 3\n    return m\n\n\n# ---------------------------------------------------------------- dedup\n# exact: hash of whitespace/case-normalised text.  near: LSH key from the 4 rarest hashed\n# unigram buckets of the doc (a min-hash sketch of its vocabulary set).\nindptr, indices = np.load(F + \"indptr.npy\"), np.load(F + \"indices.npy\")\nsketch = np.full((N, 4), -1, np.int64)\nfor i in range(N):\n    s = np.sort(indices[indptr[i]:indptr[i + 1]])[:4]\n    sketch[i, :len(s)] = s\nseen_lsh, seen_txt, dup = set(), set(), np.zeros(N, bool)\norder0 = np.argsort(-all_s)                      # keep the best-scoring copy of each dup group\ntexts = {}\nfor line in open(POOL):\n    r = json.loads(line)\n    texts[r[\"id\"]] = r[\"text\"]\nrow_of = {int(v): i for i, v in enumerate(ids)}\nfor i in order0:\n    t = texts[int(ids[i])]\n    kt = hashlib.blake2b(re.sub(r\"\\s+\", \" \", t.lower()).strip().encode(), digest_size=12).digest()\n    kl = sketch[i].tobytes()\n    if kt in seen_txt or (sketch[i, 3] >= 0 and kl in seen_lsh):\n        dup[i] = True\n    else:\n        seen_txt.add(kt); seen_lsh.add(kl)\nprint(f\"duplicates removed: {dup.sum()}\")\n\n# ---------------------------------------------------------------- token-count model\n# calibrate GPT-2 token count from cheap char/word counts on a random sample\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained(\"gpt2\")\nrs = np.random.default_rng(0).choice(N, 400, replace=False)\nY = np.array([len(tok(texts[int(ids[i])], add_special_tokens=False).input_ids) for i in rs], float)\nA = np.stack([H[\"n_chars\"][rs], H[\"n_words\"][rs], np.ones(len(rs))], 1)\ncoef = np.linalg.lstsq(A, Y, rcond=None)[0]\nest = np.stack([H[\"n_chars\"], H[\"n_words\"], np.ones(N)], 1) @ coef\nest = np.maximum(est, 1.0)\nr2 = 1 - ((A @ coef - Y) ** 2).sum() / ((Y - Y.mean()) ** 2).sum()\nprint(f\"token model tokens={coef[0]:.3f}*chars+{coef[1]:.3f}*words+{coef[2]:.1f}  R2={r2:.4f}\")\n\n# ---------------------------------------------------------------- score: shrink + assign\nnw = H[\"n_words\"]\nshrink = (nw / (nw + SHRINK))[None, :]\nprior = np.median(all_s)\nS = LLR * shrink + prior * (1 - shrink)                 # (K, N) length-shrunk register scores\ngs = all_s * shrink[0] + prior * (1 - shrink[0])\n\n# standardise per register so argmax is a fair comparison, then assign each doc to one register\nZ = (S - S.mean(1, keepdims=True)) / (S.std(1, keepdims=True) + 1e-9)\nassign = Z.argmax(0)\n\nmasks = [gate(k if a.box == \"register\" else None) & ~dup for k in range(K)]\nif a.variant == \"global\":\n    keep = np.zeros(N, bool)\n    for k in range(K):\n        keep |= masks[k]\n    ordr = np.argsort(-np.where(keep, gs, -1e9))\n    sel, tot = [], 0.0\n    for i in ordr:\n        if not keep[i]:\n            break\n        sel.append(int(ids[i])); tot += est[i] + 1\n        if tot > OVERFILL * BUDGET:\n            break\nelse:\n    # per-register queues, quota = target token share, round-robin so every prefix is mixed\n    queues = []\n    for k in range(K):\n        m = masks[k] & (assign == k)\n        idx = np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())]\n        queues.append(list(idx))\n        print(f\"  reg{k}: {m.sum()} eligible docs, {est[idx].sum()/1e6:.1f}M tokens available,\"\n              f\" quota {tok_share[k]*BUDGET/1e6:.2f}M\")\n    want = tok_share * OVERFILL * BUDGET\n    got = np.zeros(K); sel, ptr = [], [0] * K\n    while True:\n        # serve the register that is furthest behind its quota\n        deficit = np.array([(got[k] / want[k]) if ptr[k] < len(queues[k]) else 9e9 for k in range(K)])\n        k = int(deficit.argmin())\n        if deficit[k] >= 9e9:\n            break\n        i = queues[k][ptr[k]]; ptr[k] += 1\n        sel.append(int(ids[i])); got[k] += est[i] + 1\n        if got.sum() >= OVERFILL * BUDGET:\n            break\n    print(\"emitted token mix:\", (got / got.sum()).round(3), f\"total {got.sum()/1e6:.1f}M\")\n\nassert len(sel) == len(set(sel)), \"duplicate ids\"\nassert all(0 <= i < 182016 for i in sel), \"id out of range\"\nos.makedirs(os.path.dirname(a.out), exist_ok=True)\njson.dump(sel, open(a.out, \"w\"))\nfirst12 = np.cumsum([est[row_of[i]] + 1 for i in sel])\nn12 = int(np.searchsorted(first12, BUDGET)) + 1\nprint(f\"wrote {len(sel)} ids -> {a.out}  (~{first12[-1]/1e6:.1f}M est tokens; \"\n      f\"first ~{n12} ids fill the {BUDGET/1e6:.0f}M budget)\")\n", "structuredPatch": [{"oldStart": 178, "oldLines": 27, "newStart": 178, "newLines": 50, "lines": ["         if tot > OVERFILL * BUDGET:", "             break", " else:", "-    # per-register queues, quota = target token share, round-robin so every prefix is mixed", "-    queues = []", "+    # Cells = register x (doc-length bucket).  Quota per cell = that cell's share of the TARGET's", "+    # token mass, so the selection reproduces both the register mix and the document-length", "+    # distribution of the target.  Filled round-robin by relative deficit, therefore every prefix", "+    # of the emitted list already carries the full mixture.", "+    nb = len(LEN_EDGES) - 1 if a.lenmatch else 1", "+    if a.lenmatch:", "+        db = np.clip(np.digitize(dev_len, LEN_EDGES) - 1, 0, nb - 1)", "+        pb = np.clip(np.digitize(est, LEN_EDGES) - 1, 0, nb - 1)", "+    else:", "+        db = np.zeros(len(dev_len), int); pb = np.zeros(N, int)", "+    cells, want = [], []", "     for k in range(K):", "-        m = masks[k] & (assign == k)", "-        idx = np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())]", "-        queues.append(list(idx))", "-        print(f\"  reg{k}: {m.sum()} eligible docs, {est[idx].sum()/1e6:.1f}M tokens available,\"", "-              f\" quota {tok_share[k]*BUDGET/1e6:.2f}M\")", "-    want = tok_share * OVERFILL * BUDGET", "-    got = np.zeros(K); sel, ptr = [], [0] * K", "+        for b in range(nb):", "+            w = dev_len[(dev_clus == k) & (db == b)].sum()", "+            if w <= 0:", "+                continue", "+            m = masks[k] & (assign == k) & (pb == b)", "+            if m.sum() == 0:", "+                continue", "+            cells.append((k, b, list(np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())])))", "+            want.append(w)", "+    want = np.array(want, np.float64)", "+    lost = 1.0 - want.sum() / dev_len.sum()      # target mass with no eligible pool supply", "+    want = want / want.sum() * OVERFILL * BUDGET", "+    print(f\"  {len(cells)} register x length cells; unmatched target mass {lost:.3f}\")", "+    for (k, b, q), w in zip(cells, want):", "+        print(f\"   reg{k} len[{LEN_EDGES[b]},{LEN_EDGES[b+1] if b+1<len(LEN_EDGES) else 0}): \"", "+              f\"{len(q):6d} docs, {est[q].sum()/1e6:6.1f}M avail, quota {w/OVERFILL/1e6:.2f}M\")", "+    got = np.zeros(len(cells)); sel, ptr = [], [0] * len(cells)", "     while True:", "-        # serve the register that is furthest behind its quota", "-        deficit = np.array([(got[k] / want[k]) if ptr[k] < len(queues[k]) else 9e9 for k in range(K)])", "-        k = int(deficit.argmin())", "-        if deficit[k] >= 9e9:", "+        deficit = np.array([(got[c] / want[c]) if ptr[c] < len(cells[c][2]) else 9e9", "+                            for c in range(len(cells))])", "+        c = int(deficit.argmin())", "+        if deficit[c] >= 9e9:", "             break", "-        i = queues[k][ptr[k]]; ptr[k] += 1", "-        sel.append(int(ids[i])); got[k] += est[i] + 1", "+        i = cells[c][2][ptr[c]]; ptr[c] += 1", "+        sel.append(int(ids[i])); got[c] += est[i] + 1", "         if got.sum() >= OVERFILL * BUDGET:", "             break", "-    print(\"emitted token mix:\", (got / got.sum()).round(3), f\"total {got.sum()/1e6:.1f}M\")", "+    mix = np.zeros(K)", "+    for (k, b, q), g in zip(cells, got):", "+        mix[k] += g", "+    print(\"emitted register mix:\", (mix / mix.sum()).round(3), \"target:\", tok_share.round(3),", "+          f\"total {got.sum()/1e6:.1f}M\")", " ", " assert len(sel) == len(set(sel)), \"duplicate ids\"", " assert all(0 <= i < 182016 for i in sel), \"id out of range\""]}], "userModified": false, "replaceAll": false}

Command did not complete within its 120s timeout and was moved to the background (ID: btta1svu2). Output is being written to: /tmp/claude-0/-workspace/32523001-df20-48c6-ad36-6a70e5bc92e0/tasks/btta1svu2.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "btta1svu2", "timedOutAfterMs": 120000, "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

26 register x length cells; unmatched target mass 0.004
emitted register mix: [0.024 0.298 0.051 0.575 0.053] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 58425 ids -> /tmp/sel_lm.json  (~36.0M est tokens; first ~24092 ids fill the 12M budget)
  26 register x length cells; unmatched target mass 0.004
emitted register mix: [0.061 0.202 0.115 0.58  0.042] target: [0.016 0.258 0.064 0.533 0.129] total 28.4M
wrote 50523 ids -> /tmp/sel_lmt.json  (~28.4M est tokens; first ~22858 ids fill the 12M budget)
emitted register mix: [0.087 0.159 0.134 0.458 0.162] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 53546 ids -> /tmp/sel_tight.json  (~36.0M est tokens; first ~16729 ids fill the 12M budget)
3788

[stdout]
  26 register x length cells; unmatched target mass 0.004
emitted register mix: [0.024 0.298 0.051 0.575 0.053] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 58425 ids -> /tmp/sel_lm.json  (~36.0M est tokens; first ~24092 ids fill the 12M budget)
  26 register x length cells; unmatched target mass 0.004
emitted register mix: [0.061 0.202 0.115 0.58  0.042] target: [0.016 0.258 0.064 0.533 0.129] total 28.4M
wrote 50523 ids -> /tmp/sel_lmt.json  (~28.4M est tokens; first ~22858 ids fill the 12M budget)
emitted register mix: [0.087 0.159 0.134 0.458 0.162] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 53546 ids -> /tmp/sel_tight.json  (~36.0M est tokens; first ~16729 ids fill the 12M budget)
3788
[metadata] {"noOutputExpected": false}

launched
bal2   first12M:   9831 docs, median doc    726 tok, mean   1221
lm     first12M:  24132 docs, median doc    283 tok, mean    497
lmt    first12M:  22886 docs, median doc    316 tok, mean    524
tight  first12M:  16746 docs, median doc    496 tok, mean    717
target: median 192, mean 369

[stdout]
launched
bal2   first12M:   9831 docs, median doc    726 tok, mean   1221
lm     first12M:  24132 docs, median doc    283 tok, mean    497
lmt    first12M:  22886 docs, median doc    316 tok, mean    524
tight  first12M:  16746 docs, median doc    496 tok, mean    717
target: median 192, mean 369
[metadata] {"noOutputExpected": false}

launched
lm: HELDOUT_PPL 359.207  loss 5.8839  train_s 175  /tmp/tr_lm.npy
lmt: HELDOUT_PPL 436.719  loss 6.0793  train_s 177  /tmp/tr_lmt.npy
tight: HELDOUT_PPL 437.866  loss 6.0819  train_s 177  /tmp/tr_tight.npy
3357

[stdout]
launched
lm: HELDOUT_PPL 359.207  loss 5.8839  train_s 175  /tmp/tr_lm.npy
lmt: HELDOUT_PPL 436.719  loss 6.0793  train_s 177  /tmp/tr_lmt.npy
tight: HELDOUT_PPL 437.866  loss 6.0819  train_s 177  /tmp/tr_tight.npy
3357
[metadata] {"noOutputExpected": false}

DIAG global overall_ppl 394.34 worst_reg 850.6 spread 2.85 logstd 0.407
 per-register ppl: {0: [850.57, 60], 1: [409.14, 1005], 2: [720.02, 252], 3: [298.01, 2067], 4: [762.39, 522]}
DIAG bal2 overall_ppl 358.89 worst_reg 783.7 spread 2.71 logstd 0.462
 per-register ppl: {0: [783.69, 60], 1: [289.69, 1005], 2: [702.96, 252], 3: [296.2, 2067], 4: [766.12, 522]}
3317

[stdout]
DIAG global overall_ppl 394.34 worst_reg 850.6 spread 2.85 logstd 0.407
 per-register ppl: {0: [850.57, 60], 1: [409.14, 1005], 2: [720.02, 252], 3: [298.01, 2067], 4: [762.39, 522]}
DIAG bal2 overall_ppl 358.89 worst_reg 783.7 spread 2.71 logstd 0.462
 per-register ppl: {0: [783.69, 60], 1: [289.69, 1005], 2: [702.96, 252], 3: [296.2, 2067], 4: [766.12, 522]}
3317
[metadata] {"noOutputExpected": false}

patched
3277

[stdout]
patched
3277
[metadata] {"noOutputExpected": false}

File "/workspace/curate.py", line 205
    for k in range(K):
IndentationError: unexpected indent

[stdout]
  File "/workspace/curate.py", line 205
    for k in range(K):
IndentationError: unexpected indent
[metadata] {"noOutputExpected": false}

193	        pb = np.clip(np.digitize(est, LEN_EDGES) - 1, 0, nb - 1)
194	    else:
195	        db = np.zeros(len(dev_len), int); pb = np.zeros(N, int)
196	    # quota reweighting: perplexity is exp(token-weighted mean loss), so the payoff from extra
197	# tokens is largest where the target's LOSS MASS sits, not merely where its token mass sits.
198	RW = np.ones(K)
199	if a.lossw:
200	    d = json.load(open(a.lossw))["per_register_ppl"]
201	    RW = np.array([d[str(k)][0] if str(k) in d else 1.0 for k in range(K)])
202	    RW = RW / RW.mean()
203	    print("loss-mass reweight:", RW.round(2))
204	cells, want = [], []
205	    for k in range(K):
206	        for b in range(nb):
207	            w = dev_len[(dev_clus == k) & (db == b)].sum() * RW[k]
208	            if w <= 0:
209	                continue
210	            m = masks[k] & (assign == k) & (pb == b)
211	            if m.sum() == 0:
212	                continue
213	            cells.append((k, b, list(np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())])))
214	            want.append(w)

[metadata] {"type": "text", "file": {"filePath": "/workspace/curate.py", "content": "        pb = np.clip(np.digitize(est, LEN_EDGES) - 1, 0, nb - 1)\n    else:\n        db = np.zeros(len(dev_len), int); pb = np.zeros(N, int)\n    # quota reweighting: perplexity is exp(token-weighted mean loss), so the payoff from extra\n# tokens is largest where the target's LOSS MASS sits, not merely where its token mass sits.\nRW = np.ones(K)\nif a.lossw:\n    d = json.load(open(a.lossw))[\"per_register_ppl\"]\n    RW = np.array([d[str(k)][0] if str(k) in d else 1.0 for k in range(K)])\n    RW = RW / RW.mean()\n    print(\"loss-mass reweight:\", RW.round(2))\ncells, want = [], []\n    for k in range(K):\n        for b in range(nb):\n            w = dev_len[(dev_clus == k) & (db == b)].sum() * RW[k]\n            if w <= 0:\n                continue\n            m = masks[k] & (assign == k) & (pb == b)\n            if m.sum() == 0:\n                continue\n            cells.append((k, b, list(np.argsort(-np.where(m, S[k], -1e9))[:int(m.sum())])))\n            want.append(w)", "numLines": 22, "startLine": 193, "totalLines": 247}}

The file /workspace/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/curate.py", "oldString": "    # quota reweighting: perplexity is exp(token-weighted mean loss), so the payoff from extra\n# tokens is largest where the target's LOSS MASS sits, not merely where its token mass sits.\nRW = np.ones(K)\nif a.lossw:\n    d = json.load(open(a.lossw))[\"per_register_ppl\"]\n    RW = np.array([d[str(k)][0] if str(k) in d else 1.0 for k in range(K)])\n    RW = RW / RW.mean()\n    print(\"loss-mass reweight:\", RW.round(2))\ncells, want = [], []\n    for k in range(K):", "newString": "    # Quota reweighting: perplexity is exp(token-weighted mean loss), so the payoff from extra\n    # tokens is largest where the target's LOSS MASS sits, not merely where its token mass sits.\n    RW = np.ones(K)\n    if a.lossw:\n        d = json.load(open(a.lossw))[\"per_register_ppl\"]\n        RW = np.array([d[str(k)][0] if str(k) in d else 1.0 for k in range(K)])\n        RW = RW / RW.mean()\n        print(\"loss-mass reweight:\", RW.round(2))\n    cells, want = [], []\n    for k in range(K):", "originalFile": null, "structuredPatch": [{"oldStart": 193, "oldLines": 15, "newStart": 193, "newLines": 15, "lines": ["         pb = np.clip(np.digitize(est, LEN_EDGES) - 1, 0, nb - 1)", "     else:", "         db = np.zeros(len(dev_len), int); pb = np.zeros(N, int)", "-    # quota reweighting: perplexity is exp(token-weighted mean loss), so the payoff from extra", "-# tokens is largest where the target's LOSS MASS sits, not merely where its token mass sits.", "-RW = np.ones(K)", "-if a.lossw:", "-    d = json.load(open(a.lossw))[\"per_register_ppl\"]", "-    RW = np.array([d[str(k)][0] if str(k) in d else 1.0 for k in range(K)])", "-    RW = RW / RW.mean()", "-    print(\"loss-mass reweight:\", RW.round(2))", "-cells, want = [], []", "+    # Quota reweighting: perplexity is exp(token-weighted mean loss), so the payoff from extra", "+    # tokens is largest where the target's LOSS MASS sits, not merely where its token mass sits.", "+    RW = np.ones(K)", "+    if a.lossw:", "+        d = json.load(open(a.lossw))[\"per_register_ppl\"]", "+        RW = np.array([d[str(k)][0] if str(k) in d else 1.0 for k in range(K)])", "+        RW = RW / RW.mean()", "+        print(\"loss-mass reweight:\", RW.round(2))", "+    cells, want = [], []", "     for k in range(K):", "         for b in range(nb):", "             w = dev_len[(dev_clus == k) & (db == b)].sum() * RW[k]"]}], "userModified": false, "replaceAll": false}

loss-mass reweight: [1.38 0.51 1.24 0.52 1.35]
emitted register mix: [0.033 0.192 0.115 0.405 0.255] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 46290 ids -> /tmp/sel_lossw.json  (~36.0M est tokens; first ~11785 ids fill the 12M budget)
[1]+  Done                    python curate.py --variant balanced --box global --lossw /tmp/diag_bal2.json --out /tmp/sel_lossw.json 2>&1 | grep -E "reweight|emitted|wrote"
launched
3057

[stdout]
loss-mass reweight: [1.38 0.51 1.24 0.52 1.35]
emitted register mix: [0.033 0.192 0.115 0.405 0.255] target: [0.016 0.258 0.064 0.533 0.129] total 36.0M
wrote 46290 ids -> /tmp/sel_lossw.json  (~36.0M est tokens; first ~11785 ids fill the 12M budget)
[1]+  Done                    python curate.py --variant balanced --box global --lossw /tmp/diag_bal2.json --out /tmp/sel_lossw.json 2>&1 | grep -E "reweight|emitted|wrote"
launched
3057
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/claim.md (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/claim.md", "content": "# Curating a raw web pool for a fixed 12M-token budget\n\n## Hypothesis\n\nAt a fixed token budget far below the pool size, held-out perplexity on a broad multi-domain\ntarget is governed less by *generic document quality* than by **distributional match to the\ntarget's register mixture**. Specifically:\n\n1. The single most useful supervision available is the dev target itself. `data/multi_dev.npy` is\n   1M GPT-2 tokens *drawn from the target distribution*; decoding it back to text yields ~2.7k\n   in-domain sample documents. A hashed-unigram importance score\n   `s(d) = mean_{w in d} log p_target(w) - log p_pool(w)` (DSIR, Xie et al. 2023) built from those\n   samples ranks pool documents by target-likeness with no labels and no external model.\n2. Ranking by that score alone is **not** enough: greedy top-k concentrates the budget on\n   whichever register is easiest to match and starves the others. Imposing **per-register token\n   quotas equal to the target's own register token mass** (registers found by spherical k-means on\n   the decoded target) should beat pure ranking.\n3. Quality filtering should be a **junk filter, not a quality bar**. Filtering to the *centre* of\n   the target's surface-statistic distribution destroys diversity and should *hurt*, even though\n   every document kept looks more target-like.\n\n## Mechanism — predicted observables (not the final perplexity)\n\n**(a) The scorer must generalise across topics, not memorise them.** Predicted *before* any\ntraining run: fit the importance score on a random 70% of the target documents; held-out target\ndocuments should outrank random pool documents with AUC ≥ 0.9. **Measured: AUC = 0.9517.**\nA discriminative logistic-regression alternative was rejected on this observable, not on\nperplexity: with 2.7k positives and 2^18 hashed features it drove its top weights onto proper\nnouns (`bush`, `grissom`, `wilder`, `boosey`) and flagged only 0.6% of the pool — topic\nmemorisation, not register. Restricting the feature space to buckets frequent in the pool and\nscoring by a smoothed likelihood ratio fixes it.\n\n**(b) The gain from quota-balancing must be *concentrated in the register that greedy ranking\nstarves*, not spread evenly.** This is the discriminating prediction: if balancing merely\nimproved average data quality, all registers would improve together. Measured per-register\nheld-out perplexity (`diag_eval.py`, identical frozen recipe, register assigned per eval window):\n\n| register (k=5) | rank-only | quota-balanced | change |\n|---|---|---|---|\n| 1 · technical Q&A / code | 409.1 | **289.7** | **−29%** |\n| 0 · niche web prose | 850.6 | 783.7 | −8% |\n| 2 · science / geography wiki | 720.0 | 703.0 | −2% |\n| 3 · news (53% of target mass) | 298.0 | 296.2 | −1% |\n| 4 · biography wiki | 762.4 | 766.1 | +0.5% |\n\nConfirmed: essentially the whole improvement is register 1, which greedy ranking under-served,\nwhile the dominant news register is unchanged. Worst-register perplexity falls 850.6 → 783.7 and\nthe max/min spread narrows 2.85 → 2.71. One sub-prediction failed: the *log* spread across\nregisters widened (0.407 → 0.462), because a single starved register improved sharply rather than\nall registers converging — the gain is concentrated, so dispersion is not a reliable summary of it.\n\n**(c) A hard floor exists that no selection can remove.** 863 / 2705 target documents (32%,\nthe encyclopedic portion) carry the WikiText detokenisation artifact ` @-@ `; `grep` finds it in\n**0** of 182,016 pool documents. Predicted consequence: the wiki-formatted registers stay the\nworst-served no matter how the pool is curated, and absolute perplexity stays in the hundreds.\nConfirmed — registers 0/2/4 remain at 703–784 while news reaches 296.\n\n## Falsification\n\nEach claim was given a way to fail, and the third partly did:\n\n| test | prediction | result |\n|---|---|---|\n| quotas vs pure ranking | quotas win | 358.9 vs 394.3 ✓ |\n| acceptance box vs none | box wins | 358.9 vs 381.2 ✓ |\n| **tighter** box (5–95 pct) | *hurts* — diversity, not purity, is scarce | 437.9 vs 358.9 ✓ (predicted sign) |\n| document-length matching | helps (target median 192 tok vs 726 selected) | 359.2 vs 358.9 — **no effect** ✗ |\n| finer registers (K=12 vs 5) | helps | 361.3 vs 358.9 — no effect ✗ |\n| loss-mass-weighted quotas | helps | see table below |\n\nThe hypothesis would have been falsified outright if quota-balancing had not beaten pure ranking,\nor if the per-register breakdown had shown a uniform improvement instead of one concentrated in\nthe starved register. Length-matching is the clearest negative: matching the target's\ndocument-length distribution (mean selected length 1221 → 497 tokens, 9.8k → 24.1k documents in\nthe budget) changed perplexity by +0.3, inside single-seed noise. Whatever the EOS-boundary\nstatistics of the eval stream cost, it is not recoverable by selection.\n\n## Results (frozen recipe, 12M tokens, seed 1337, dev target)\n\n| selection | dev PPL |\n|---|---|\n| random pool sample (do-nothing baseline) | 469.9 |\n| DSIR rank-only, no register quotas | 394.3 |\n| quotas, no acceptance box (length + dedup only) | 381.2 |\n| quotas + box, K=12 registers | 361.3 |\n| quotas + box + document-length matching | 359.2 |\n| **quotas + box, K=5 (submitted configuration)** | **358.9** |\n| quotas + *tight* box | 437.9 |\n\n## Transfer\n\nThe method needs only (i) a raw pool and (ii) a sample of the target distribution — no labels, no\nreference LM, no internet. Cost for 182k documents / 200M tokens: 45 s of CPU for hashed features\n(15 processes) and a few seconds of GPU for scoring; both are linear in pool size, so the same\nrecipe runs on a pool 1000× larger with the same code. It transfers directly to any budgeted\npretraining or domain-adaptive-pretraining problem where a modest in-domain sample exists, and the\nregister-quota step is the part worth carrying over: it is what converts a *similarity* score into\na *mixture* and it is where 90% of the measured gain over rank-only selection comes from.\n\nTwo transfer caveats are diagnosable in advance, without training. The `unmatched target mass`\nfigure printed by `curate.py` (0.004 here) reports the fraction of target token mass for which no\neligible pool document exists — if it is large, the pool cannot express the target and quotas will\nsilently redistribute. And the scorer's held-out AUC is a pre-training go/no-go check: below ~0.9\nthe importance score is memorising topics rather than registers, and the ranking should not be\ntrusted.\n", "structuredPatch": [], "originalFile": null, "userModified": false}

loose: (running)
nodd: (running)
lossw: (running)
2977

[stdout]
loose: (running)
nodd: (running)
lossw: (running)
2977
[metadata] {"noOutputExpected": false}

loose: (running)
nodd: (running)
lossw: (running)
2856

[stdout]
loose: (running)
nodd: (running)
lossw: (running)
2856
[metadata] {"noOutputExpected": false}

loose: HELDOUT_PPL 336.054  loss 5.8173  train_s 177  /tmp/tr_loose.npy packed 12000000 tokens from 34455 ids -> /tmp/tr_loose.npy 
nodd: HELDOUT_PPL 348.825  loss 5.8546  train_s 176  /tmp/tr_nodd.npy packed 12000000 tokens from 38296 ids -> /tmp/tr_nodd.npy 
lossw: HELDOUT_PPL 357.964  loss 5.8804  train_s 177  /tmp/tr_lossw.npy packed 12000000 tokens from 46290 ids -> /tmp/tr_lossw.npy 
2746

[stdout]
loose: HELDOUT_PPL 336.054  loss 5.8173  train_s 177  /tmp/tr_loose.npy packed 12000000 tokens from 34455 ids -> /tmp/tr_loose.npy 
nodd: HELDOUT_PPL 348.825  loss 5.8546  train_s 176  /tmp/tr_nodd.npy packed 12000000 tokens from 38296 ids -> /tmp/tr_nodd.npy 
lossw: HELDOUT_PPL 357.964  loss 5.8804  train_s 177  /tmp/tr_lossw.npy packed 12000000 tokens from 46290 ids -> /tmp/tr_lossw.npy 
2746
[metadata] {"noOutputExpected": false}

ok
[1]   Done                    python curate.py --loose --dedup exact --out /tmp/sel_A.json > /tmp/bA.log 2>&1
[2]-  Done                    python curate.py --lo 0.5 --hi 99.5 --pad 0.6 --dedup exact --out /tmp/sel_B.json > /tmp/bB.log 2>&1
[3]+  Done                    python curate.py --loose --dedup exact --min_words 40 --out /tmp/sel_C.json > /tmp/bC.log 2>&1
acceptance box: pct[1.0,99.0] pad 0.3
wrote 32065 ids -> /tmp/sel_A.json  (~36.0M est tokens; first ~7216 ids fill the 12M budget)
acceptance box: pct[1.0,99.0] pad 0.3
wrote 34764 ids -> /tmp/sel_C.json  (~36.0M est tokens; first ~7684 ids fill the 12M budget)
acceptance box: pct[0.5,99.5] pad 0.6
wrote 29412 ids -> /tmp/sel_B.json  (~36.0M est tokens; first ~5632 ids fill the 12M budget)
2626

[stdout]
ok
[1]   Done                    python curate.py --loose --dedup exact --out /tmp/sel_A.json > /tmp/bA.log 2>&1
[2]-  Done                    python curate.py --lo 0.5 --hi 99.5 --pad 0.6 --dedup exact --out /tmp/sel_B.json > /tmp/bB.log 2>&1
[3]+  Done                    python curate.py --loose --dedup exact --min_words 40 --out /tmp/sel_C.json > /tmp/bC.log 2>&1
acceptance box: pct[1.0,99.0] pad 0.3
wrote 32065 ids -> /tmp/sel_A.json  (~36.0M est tokens; first ~7216 ids fill the 12M budget)
acceptance box: pct[1.0,99.0] pad 0.3
wrote 34764 ids -> /tmp/sel_C.json  (~36.0M est tokens; first ~7684 ids fill the 12M budget)
acceptance box: pct[0.5,99.5] pad 0.6
wrote 29412 ids -> /tmp/sel_B.json  (~36.0M est tokens; first ~5632 ids fill the 12M budget)
2626
[metadata] {"noOutputExpected": false}

launched
2526

[stdout]
launched
2526
[metadata] {"noOutputExpected": false}

A: (running)
B: (running)
C: (running)
2396

[stdout]
A: (running)
B: (running)
C: (running)
2396
[metadata] {"noOutputExpected": false}