claude-opus-5

{
  "command": "ps aux | grep -c \"[c]urate.py\"; pkill -f \"python3 submission/curate\" ; sleep 1; ps aux | grep -c \"[c]urate.py\""
}

{
  "command": "python3 submission/curate.py 2>&1 | tail -20",
  "run_in_background": true
}

{
  "command": "sleep 200; tail -20 /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bpv3jg2vs.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 110; tail -16 /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bpv3jg2vs.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "python3 - <<'PY'\np='submission/curate.py'; s=open(p).read()\nold = s[s.index('best = logp[:, :3].argmax(1)'):s.index('quota = {k:')]\nnew = '''# Each register ranks EVERY gated document by its own posterior; a document is\n# claimed by the first register that reaches it during the round-robin, so the\n# lists stay disjoint without any register being starved by an argmax tie-break.\nbuckets = {}\nfor r, name in enumerate([\"wiki\", \"qa\", \"web\"]):\n    buckets[name] = list(np.argsort(-(logp[:, r] + prior)))\n    top = buckets[name][:5000]\n    print(f\"{name}: mean logP of its top 5000 = {logp[top, r].mean():.3f}\", flush=True)\n\n'''\ns = s.replace(old, new)\ns = s.replace('''    k = min(cand, key=lambda k: got[k] / quota[k])   # furthest behind its share\n    j = buckets[k][ptr[k]]; ptr[k] += 1\n    sel.append(ids[keep[j]])\n    got[k] += lens[j]; tot += lens[j]''','''    k = min(cand, key=lambda k: got[k] / quota[k])   # furthest behind its share\n    j = buckets[k][ptr[k]]; ptr[k] += 1\n    if j in claimed:                                 # already taken by another register\n        continue\n    claimed.add(j)\n    sel.append(ids[keep[j]])\n    got[k] += lens[j]; tot += lens[j]''')\ns = s.replace(\"sel, tot = [], 0.0\",\"sel, tot, claimed = [], 0.0, set()\")\ns = s.replace('''3. STRATIFIED FILL. Budget shares mirror the target mixture: wiki 25%, qa 25%,\n   web+news 50% (the \"web\" class absorbs two of the four target domains). Each\n   document is claimed by the register that ranks it highest, so the per-register\n   lists are disjoint. The final list round-robins across registers in proportion\n   to those shares, best-first, so that truncating the list anywhere -- including\n   at the 12M-token budget -- preserves the mixture.''','''3. STRATIFIED FILL. Budget shares mirror the target mixture: wiki 25%, qa 25%,\n   web+news 50% (the \"web\" class absorbs two of the four target domains). Each\n   register ranks every gated document by its own posterior and the final list\n   round-robins across registers, always drawing next from whichever register is\n   furthest behind its share; a document already claimed is skipped, so the list\n   is duplicate-free. Because the fill is a round-robin rather than three\n   concatenated blocks, truncating the list anywhere -- including at the 12M-token\n   budget -- preserves the mixture.''')\nopen(p,'w').write(s)\nPY\npython3 submission/curate.py 2>&1 | tail -9"
}

{
  "command": "cp submission/selection.json /tmp/v2_selection.json; bash dev_eval.sh /tmp/v2_selection.json 2>&1 | grep -E \"HELDOUT|packed\"",
  "run_in_background": true
}

{
  "command": "sleep 235; tail -3 /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bh8e2rfcc.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "cat /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bh8e2rfcc.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "python3 - <<'PY'\nimport json,pickle,numpy as np\nids=[];texts=[]\nfor line in open('data/pool.jsonl'):\n    r=json.loads(line); ids.append(r['id']); texts.append(r['text'])\nkeep=pickle.load(open('/tmp/keep_cache_v2.pkl','rb'))\nlogp=np.load('/tmp/scores_v2.npy')\nlens=np.array([len(texts[j]) for j in keep],float)\nprior=0.15*np.log(lens/2000.0)\nTC=100_000_000\ndef build(SH,out):\n    buckets={n:list(np.argsort(-(logp[:,r]+prior))) for r,n in enumerate([\"wiki\",\"qa\",\"web\"])}\n    quota={k:SH[k]*TC for k in SH}; got={k:0. for k in SH}; ptr={k:0 for k in SH}\n    sel=[];tot=0.;claimed=set()\n    while tot<TC:\n        cand=[k for k in SH if ptr[k]<len(buckets[k])]\n        if not cand: break\n        k=min(cand,key=lambda k:got[k]/quota[k]); j=buckets[k][ptr[k]]; ptr[k]+=1\n        if j in claimed: continue\n        claimed.add(j); sel.append(ids[keep[j]]); got[k]+=lens[j]; tot+=lens[j]\n    json.dump(sel,open(out,'w')); print(out,len(sel),{k:round(v/1e6,1) for k,v in got.items()})\nbuild({\"wiki\":0.40,\"qa\":0.10,\"web\":0.50},'/tmp/selA.json')\nbuild({\"wiki\":0.34,\"qa\":0.33,\"web\":0.33},'/tmp/selB.json')\nbuild({\"wiki\":0.15,\"qa\":0.15,\"web\":0.70},'/tmp/selC.json')\nPY"
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\nfor v in A B C; do\n ( python3 pack_selection.py /tmp/sel$v.json /tmp/tr$v.npy >/dev/null 2>&1 && \\\n   python3 train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/$v.json 2>&1 | grep HELDOUT | sed \"s/^/$v /\" ) &\ndone; wait",
  "run_in_background": true
}

{
  "file_path": "/workspace/claim.md",
  "content": "# Claim: register-stratified domain matching beats undifferentiated quality ranking\n\n## Hypothesis\n\nFor a fixed, small token budget evaluated against a **multi-register** target, the\nbinding constraint is not \"how clean is each document\" but **how well the training\nmixture's register proportions match the target's**. Concretely:\n\n1. A single scalar quality/domain score, applied greedily, collapses the selection\n   onto whichever register is most abundant and most confidently scored (here:\n   general web prose). The under-represented registers of the target — encyclopedic\n   prose and technical Q&A — are then almost absent from a 12M-token budget, and\n   the model pays a large perplexity penalty on exactly those quarters of the\n   evaluation.\n2. Ranking documents *separately per target register* and filling a **per-register\n   quota** recovers those quarters at almost no cost to the majority register,\n   because the marginal web document at the 50%-quota boundary is barely worse\n   than the one at the 100%-quota boundary (the web candidate pool is ~30x larger\n   than the budget), whereas the marginal wiki-like document is enormously better\n   than nothing.\n\nSo: stratify, don't just rank.\n\n## Mechanism (prediction of an observable other than final perplexity)\n\nThe mechanism is **coverage of the target's register mixture**, not per-document\ncleanliness. That makes several non-perplexity predictions, checkable before any\nmodel is trained:\n\n- **M1 — Collapse is visible in the selection itself.** Under a single-score\n  ranking, the register composition of the selected 12M tokens should be\n  drastically skewed relative to the target's ~25/25/50 (wiki/qa/web) mixture.\n  *Observed:* the un-stratified variant's candidate mass is 417M chars of `web`\n  against 14M of `wiki` and 3M of `qa` — a greedy single ranking is >80% web.\n- **M2 — The gate alone is not the mechanism.** The hard quality gate keeps ~52%\n  of the pool; ranking within it by a single high-quality-vs-pool classifier\n  should give only a *small* gain over random. *Observed:* single-classifier\n  selection 434.4 dev PPL vs random baseline 459.8 — a 5.5% gain, i.e. cleanliness\n  alone explains little.\n- **M3 — Stratification is the mechanism.** Holding the gate, the features, and the\n  classifier fixed and changing *only* the fill rule from \"one ranking\" to\n  \"per-register quotas\" should produce a step change far larger than M2's.\n  *Observed:* 434.4 → 321.4 dev PPL (−26%), from a change that touches no\n  document scores at all, only how many documents each register contributes.\n- **M4 — Interior optimum in the quota vector.** If coverage is the mechanism,\n  perplexity should be a *concave* function of the quota shares with a minimum\n  near the target's true proportions: both starving a register and over-serving it\n  (spending budget on weak proxies at the expense of the abundant, well-matched\n  register) should be worse than matching. A pure quality story predicts monotone\n  improvement as we concentrate on the highest-scoring register instead.\n- **M5 — Register proxies need not match surface form.** The pool contains only 96\n  documents with an HTML `<p>` tag, so the `qa` quota cannot be filled with\n  markup-matched text; it fills with technical/instructional prose. If the\n  mechanism were surface-form mimicry, that quota would be worthless; if it is\n  register coverage, it should still help.\n\n## Falsification\n\nThe claim is wrong if any of these hold:\n\n- **F1.** A single-ranking selection using the *same* gate and *same* features,\n  with no stratification, matches or beats the stratified selection. Then the gain\n  came from the scorer, not from mixture matching. (Run: v1 = 434.4 vs\n  v2 = 321.4 — not observed.)\n- **F2.** Perplexity is monotone in the wiki (or any single) share, i.e. pushing a\n  share to ~100% keeps improving. Then \"pick the best-looking register and stop\"\n  is the real rule and stratification is incidental. (Tested by sweeping the share\n  vector; an interior optimum falsifies this alternative, a monotone curve\n  falsifies my claim.)\n- **F3.** Removing the `qa` quota entirely (share → 0) is neutral or better at\n  matched budget. That would falsify M5 and show the quota is filling with noise\n  rather than a usable register proxy.\n- **F4.** Randomly sub-sampling the gated pool to the same 25/25/50 *character*\n  proportions, but with documents drawn at random within each register rather\n  than by score, does as well. Then the per-register *ranking* contributes nothing\n  and only the proportions matter — a weaker claim than the one made here.\n\n## Transfer\n\n- **Where it should transfer.** Any fixed-budget curation problem whose evaluation\n  is a *known mixture* of registers, especially when budget << pool and the pool is\n  dominated by one register. The recipe needs only (a) a handful of in-domain\n  reference text to define registers, and (b) a cheap scorer; both scale to\n  fastText/n-gram classifiers over billions of documents at negligible cost. This\n  is why production pipelines interleave per-source quotas rather than globally\n  ranking by a quality score.\n- **What it needs to be true.** The abundant register must be over-supplied\n  relative to the budget (so quota-capping it is nearly free), and the scarce\n  registers must have *some* usable proxy in the pool. Both hold here.\n- **Where it should fail.** (i) If the budget approaches the size of the gated pool,\n  quotas can only reshuffle a fixed set and the gain vanishes. (ii) If the target\n  is single-register, stratification degenerates to ranking and buys nothing. (iii)\n  If a target register has no proxy at all in the pool, its quota actively wastes\n  budget — the honest move there is to shrink that share, which is exactly what the\n  share sweep tests.\n- **Scaling caveat.** The reference text used to define the registers here is the\n  dev target itself. That is legitimate domain disclosure, not label leakage — the\n  scored quantity is register membership, not held-out loss — but it does mean the\n  method inherits any register the disclosure omits. At larger scale one would\n  define registers from named sources (Wikipedia dump, a QA site dump, a news\n  corpus) instead of from an eval sample.\n\n## Results (dev PPL, frozen 30M GPT, 12M-token budget, `multi_dev.npy`)\n\n| selection | dev PPL |\n|---|---|\n| random pool sample (do-nothing baseline) | 459.8 |\n| gate + single quality/domain ranking (v1) | 434.4 |\n| gate + register-stratified quotas 25/25/50 (v2) | **321.4** |\n\nShare-sweep results (F2/F3) are reported in `RESULTS.md`.\n"
}

{
  "command": "sleep 105; cat /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bybew3bx6.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 210; cat /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bybew3bx6.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "cat /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bybew3bx6.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "python3 - <<'PY'\nimport json,pickle,zlib,re,numpy as np,os\nids=[];texts=[]\nfor line in open('data/pool.jsonl'):\n    r=json.loads(line); ids.append(r['id']); texts.append(r['text'])\nkeep=pickle.load(open('/tmp/keep_cache_v2.pkl','rb'))\nlogp=np.load('/tmp/scores_v2.npy'); lens=np.array([len(texts[j]) for j in keep],float)\nW=re.compile(r\"[a-z0-9']+\")\n# MinHash-16 over word 5-grams -> drop docs sharing a 4-hash band (near-dup)\nsig=np.empty((len(keep),16),np.uint32)\nfor n,j in enumerate(keep):\n    w=W.findall(texts[j].lower())\n    g=[zlib.crc32(' '.join(w[k:k+5]).encode()) for k in range(0,max(1,len(w)-4))]\n    a=np.array(g,np.uint64) if g else np.zeros(1,np.uint64)\n    for p in range(16):\n        sig[n,p]=((a*(2*p+1)+p*7919)%4294967291).min()\nseen=set(); dup=np.zeros(len(keep),bool)\norder=np.argsort(-lens)   # keep the longest representative of each cluster\nfor n in order:\n    bands=[(p,)+tuple(sig[n,4*p:4*p+4]) for p in range(4)]\n    if any(b in seen for b in bands): dup[n]=True\n    else: seen.update(bands)\nprint(\"near-dups flagged:\",dup.sum(),\"of\",len(keep))\nnp.save('/tmp/dup.npy',dup)\nTC=100_000_000\ndef build(SH,out,pw=0.15,nodup=True):\n    prior=pw*np.log(lens/2000.0)\n    pen=np.where(dup,-1e9,0.0) if nodup else 0.0\n    buckets={n:list(np.argsort(-(logp[:,r]+prior+pen))) for r,n in enumerate([\"wiki\",\"qa\",\"web\"])}\n    quota={k:SH[k]*TC for k in SH}; got={k:0. for k in SH}; ptr={k:0 for k in SH}\n    sel=[];tot=0.;claimed=set()\n    while tot<TC:\n        cand=[k for k in SH if ptr[k]<len(buckets[k])]\n        if not cand: break\n        k=min(cand,key=lambda k:got[k]/quota[k]); j=buckets[k][ptr[k]]; ptr[k]+=1\n        if j in claimed or (nodup and dup[j]): continue\n        claimed.add(j); sel.append(ids[keep[j]]); got[k]+=lens[j]; tot+=lens[j]\n    json.dump(sel,open(out,'w')); print(out,len(sel),round(tot/1e6,1))\nB={\"wiki\":0.34,\"qa\":0.33,\"web\":0.33}\nbuild(B,'/tmp/selD.json')                  # dedup\nbuild(B,'/tmp/selE.json',pw=0.45)          # stronger length prior, dedup\nbuild(B,'/tmp/selF.json',pw=0.0)           # no length prior, dedup\nPY\ncat /workspace/.timer/remaining_secs"
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\nfor v in D E F; do\n ( python3 pack_selection.py /tmp/sel$v.json /tmp/tr$v.npy >/dev/null 2>&1 && \\\n   python3 train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/$v.json 2>&1 | grep HELDOUT | sed \"s/^/$v /\" ) &\ndone; wait",
  "run_in_background": true
}

{
  "command": "sleep 235; grep HELD /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bijuh3qi0.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 170; grep HELD /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bijuh3qi0.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "grep HELD /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/bijuh3qi0.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "python3 - <<'PY'\nimport json,pickle,random,numpy as np\nids=[];texts=[]\nfor line in open('data/pool.jsonl'):\n    r=json.loads(line); ids.append(r['id']); texts.append(r['text'])\nkeep=pickle.load(open('/tmp/keep_cache_v2.pkl','rb'))\nlogp=np.load('/tmp/scores_v2.npy'); dup=np.load('/tmp/dup.npy')\nlens=np.array([len(texts[j]) for j in keep],float)\nTC=100_000_000\ndef build(SH,out,pw=0.0,dilute=0.0):\n    prior=pw*np.log(lens/2000.0); pen=np.where(dup,-1e9,0.0)\n    buckets={n:list(np.argsort(-(logp[:,r]+prior+pen))) for r,n in enumerate([\"wiki\",\"qa\",\"web\"]) if SH.get(n,0)>0}\n    SH={k:v for k,v in SH.items() if v>0}\n    quota={k:SH[k]*TC for k in SH}; got={k:0. for k in SH}; ptr={k:0 for k in SH}\n    sel=[];tot=0.;claimed=set()\n    while tot<TC:\n        cand=[k for k in SH if ptr[k]<len(buckets[k])]\n        if not cand: break\n        k=min(cand,key=lambda k:got[k]/quota[k]); j=buckets[k][ptr[k]]; ptr[k]+=1\n        if j in claimed or dup[j]: continue\n        claimed.add(j); sel.append(ids[keep[j]]); got[k]+=lens[j]; tot+=lens[j]\n    if dilute>0:   # interleave random gated docs to add register diversity\n        rng=random.Random(0); pool=[n for n in range(len(keep)) if n not in claimed and not dup[n]]\n        rng.shuffle(pool); step=int(1/dilute)\n        extra=[ids[keep[n]] for n in pool[:len(sel)//step]]\n        merged=[]\n        for i,x in enumerate(sel):\n            merged.append(x)\n            if i%step==step-1 and extra: merged.append(extra.pop())\n        sel=merged\n    json.dump(sel,open(out,'w')); print(out,len(sel))\nbuild({\"wiki\":0.34,\"qa\":0.33,\"web\":0.33},'/tmp/selG.json',dilute=0.15)\nbuild({\"wiki\":0.45,\"qa\":0.30,\"web\":0.25},'/tmp/selH.json')\nbuild({\"wiki\":0.50,\"qa\":0.0,\"web\":0.50},'/tmp/selI.json')\nPY\ncd /workspace; export TOKENIZERS_PARALLELISM=false\nfor v in G H I; do\n ( python3 pack_selection.py /tmp/sel$v.json /tmp/tr$v.npy >/dev/null 2>&1 && \\\n   python3 train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/$v.json 2>&1 | grep HELDOUT | sed \"s/^/$v /\" ) &\ndone; wait",
  "run_in_background": true
}

{
  "command": "python3 - <<'PY'\np='/workspace/submission/curate.py'; s=open(p).read()\ns=s.replace('SHARES = {\"wiki\": 0.25, \"qa\": 0.25, \"web\": 0.50}',\n            'SHARES = {\"wiki\": 0.34, \"qa\": 0.33, \"web\": 0.33}   # tuned by dev sweep')\n# insert near-dup removal after lens/prior block\nold = '''lens = np.array([len(texts[j]) for j in keep], np.float64)\n# Mild length prior: among equally in-register documents prefer longer ones --\n# fewer document boundaries per training token means more usable context.\nprior = 0.15 * np.log(lens / 2000.0)\n'''\nnew = '''lens = np.array([len(texts[j]) for j in keep], np.float64)\n\n# Near-duplicate removal (MinHash-16 over word 5-grams, 4 bands of 4).\n# Duplicated text wastes budget: at 12M tokens the model sees the selection\n# ~2x already, so a near-dup pair is effectively 4 passes over that text.\n# The longest member of each cluster is kept as the representative.\nWORDL = re.compile(r\"[a-z0-9']+\")\nsig = np.empty((len(keep), 16), np.uint32)\nfor n, j in enumerate(keep):\n    w = WORDL.findall(texts[j].lower())\n    grams = [zlib.crc32(\" \".join(w[k:k + 5]).encode()) for k in range(max(1, len(w) - 4))]\n    a = np.array(grams or [0], np.uint64)\n    for pp in range(16):\n        sig[n, pp] = ((a * (2 * pp + 1) + pp * 7919) % 4294967291).min()\n    if n % 20000 == 0:\n        print(f\"minhash {n}/{len(keep)}\", flush=True)\nseen, dup = set(), np.zeros(len(keep), bool)\nfor n in np.argsort(-lens):\n    bands = [(pp,) + tuple(sig[n, 4 * pp:4 * pp + 4]) for pp in range(4)]\n    if any(bd in seen for bd in bands):\n        dup[n] = True\n    else:\n        seen.update(bands)\nprint(f\"near-duplicates dropped: {int(dup.sum())} of {len(keep)}\", flush=True)\n\n# No length prior: a dev sweep showed ranking purely by register posterior beats\n# any length tilt (prior weight 0.0: 318.8 dev PPL; 0.15: 324.1; 0.45: 325.2).\nprior = np.where(dup, -1e9, 0.0)\n'''\nassert old in s\ns=s.replace(old,new)\ns=s.replace('''    if j in claimed:                                 # already taken by another register\n        continue''','''    if j in claimed or dup[j]:                       # already taken, or a near-dup\n        continue''')\ns=s.replace('''   register ranks every gated document by its own posterior and the final list''',\n            '''   register ranks every gated document by its own posterior (near-duplicates,\n   found by MinHash over word 5-grams, are excluded) and the final list''')\nopen(p,'w').write(s)\nprint(\"patched\")\nPY"
}

{
  "command": "sleep 230; grep HELD /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/b0efm8kwh.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 175; grep HELD /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/b0efm8kwh.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "grep HELD /tmp/claude-0/-workspace/9df847d0-e162-4016-8bd5-af03c1e3e23f/tasks/b0efm8kwh.output; cat /workspace/.timer/remaining_secs"
}

{
  "command": "cat > /tmp/build.py <<'PY'\nimport json,pickle,random,sys,numpy as np\nids=[];texts=[]\nfor line in open('/workspace/data/pool.jsonl'):\n    r=json.loads(line); ids.append(r['id']); texts.append(r['text'])\nkeep=pickle.load(open('/tmp/keep_cache_v2.pkl','rb'))\nlogp=np.load('/tmp/scores_v2.npy'); dup=np.load('/tmp/dup.npy')\nlens=np.array([len(texts[j]) for j in keep],float)\nTC=100_000_000\ndef build(SH,out,dilute=0.0):\n    pen=np.where(dup,-1e9,0.0)\n    SH={k:v for k,v in SH.items() if v>0}\n    buckets={n:list(np.argsort(-(logp[:,r]+pen))) for r,n in enumerate([\"wiki\",\"qa\",\"web\"]) if n in SH}\n    quota={k:SH[k]*TC for k in SH}; got={k:0. for k in SH}; ptr={k:0 for k in SH}\n    sel=[];tot=0.;claimed=set()\n    while tot<TC:\n        cand=[k for k in SH if ptr[k]<len(buckets[k])]\n        if not cand: break\n        k=min(cand,key=lambda k:got[k]/quota[k]); j=buckets[k][ptr[k]]; ptr[k]+=1\n        if j in claimed or dup[j]: continue\n        claimed.add(j); sel.append(ids[keep[j]]); got[k]+=lens[j]; tot+=lens[j]\n    if dilute>0:\n        rng=random.Random(0); pool=[n for n in range(len(keep)) if n not in claimed and not dup[n]]\n        rng.shuffle(pool); step=int(1/dilute); extra=[ids[keep[n]] for n in pool[:len(sel)//step]]\n        m=[]\n        for i,x in enumerate(sel):\n            m.append(x)\n            if i%step==step-1 and extra: m.append(extra.pop())\n        sel=m\n    json.dump(sel,open(out,'w')); print(out,len(sel),flush=True)\nfor spec in json.loads(sys.argv[1]):\n    build(spec['sh'],spec['out'],spec.get('dilute',0.0))\nPY\npython3 /tmp/build.py '[{\"sh\":{\"wiki\":0.30,\"qa\":0.45,\"web\":0.25},\"out\":\"/tmp/selJ.json\"},\n {\"sh\":{\"wiki\":0.45,\"qa\":0.30,\"web\":0.25},\"out\":\"/tmp/selK.json\",\"dilute\":0.15},\n {\"sh\":{\"wiki\":0.20,\"qa\":0.60,\"web\":0.20},\"out\":\"/tmp/selL.json\"}]'\ncd /workspace; export TOKENIZERS_PARALLELISM=false\nfor v in J K L; do\n ( python3 pack_selection.py /tmp/sel$v.json /tmp/tr$v.npy >/dev/null 2>&1 && \\\n   python3 train_nano.py --train_npy /tmp/tr$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/$v.json 2>&1 | grep HELDOUT | sed \"s/^/$v /\" ) &\ndone; wait",
  "run_in_background": true
}