claude-opus-5

{
  "command": "python3 - <<'EOF'\np='variants.py'; s=open(p).read()\ns=s.replace('''def build(toks, col=14, mix=(0.25, 0.25, 0.25, 0.25), wiki_boost=0.0, qa_boost=0.0,\n          len_pref=0.0, tail=6000, verbose=True):\n    S = score_cols(col, wiki_boost, qa_boost, len_pref)''',\n'''WIKI_RELAX = None   # set lazily\n\n\ndef relaxed_wiki_mask():\n    \"\"\"encyclopedic text is date/number/table dense - relax those shape rules for the wiki quota\"\"\"\n    nb, nalpha, stop, nonascii, awl, termp, dup, upper, dig, boil, naughty = [F[:, j] for j in range(11)]\n    m = (nb > 700) & (nb < 500_000) & (nalpha > 110)\n    m &= (stop > 0.055) & (nonascii < 0.06) & (awl > 3.0) & (awl < 8.5)\n    m &= (termp > 0.25) & (dup < 0.30) & (upper < 0.18) & (dig < 0.22) & (boil < 3.5) & (naughty <= 1)\n    return m\n\n\ndef build(toks, col=14, mix=(0.25, 0.25, 0.25, 0.25), wiki_boost=0.0, qa_boost=0.0,\n          len_pref=0.0, tail=6000, verbose=True, code_boost=0.0, wiki_relax=False):\n    S = score_cols(col, wiki_boost, qa_boost, len_pref)\n    if code_boost:\n        S[:, 3] += code_boost * np.minimum(F[:, 11], 4.0) / 4.0''')\ns=s.replace('''    for r in range(4):\n        ok = BASE | TECH if r == 3 else BASE\n        orders.append(np.argsort(-np.where(ok, S[:, r], -1e9)))''',\n'''    wmask = (BASE | relaxed_wiki_mask()) if wiki_relax else BASE\n    for r in range(4):\n        ok = (BASE | TECH) if r == 3 else (wmask if r == 0 else BASE)\n        orders.append(np.argsort(-np.where(ok, S[:, r], -1e9)))''')\ns=s.replace('''    specs = {\n        \"v2_contrast\":  dict(col=18),\n        \"v3_tilt\":      dict(col=14, mix=(0.20, 0.235, 0.235, 0.33)),\n        \"v4_boost\":     dict(col=14, wiki_boost=0.35, qa_boost=0.20),\n    }''',\n'''    specs = json.load(open(sys.argv[1])) if len(sys.argv) > 1 else {\n        \"v2_contrast\":  dict(col=18),\n        \"v3_tilt\":      dict(col=14, mix=(0.20, 0.235, 0.235, 0.33)),\n        \"v4_boost\":     dict(col=14, wiki_boost=0.35, qa_boost=0.20),\n    }''')\ns=s.replace('''    json.dump(flat_quality(toks), open(\"/tmp/sel_v5_flat.json\", \"w\"))\n    print(\"v5_flat (no register balance) written\")''',\n'''    if len(sys.argv) <= 1:\n        json.dump(flat_quality(toks), open(\"/tmp/sel_v5_flat.json\", \"w\"))\n        print(\"v5_flat (no register balance) written\")''')\nopen(p,'w').write(s); print('patched')\nEOF\ncat > /tmp/r3.json <<'EOF'\n{\"v6_techcomp\": {\"col\": 18, \"code_boost\": 1.2, \"qa_boost\": 0.5},\n \"v7_len\":      {\"col\": 18, \"len_pref\": 0.4},\n \"v8_wikirelax\":{\"col\": 18, \"wiki_relax\": true, \"wiki_boost\": 0.3},\n \"v9_wikitilt\": {\"col\": 18, \"mix\": [0.30, 0.23, 0.23, 0.24]}}\nEOF\nexport TOKENIZERS_PARALLELISM=false; python3 variants.py /tmp/r3.json 2>&1 | grep -v \"Token indices\" | tail -10",
  "description": "Add variant knobs and build round 3"
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\nfor v in v6_techcomp v7_len v8_wikirelax v9_wikitilt; do\n  python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1\ndone\nnohup bash -c 'for v in v6_techcomp v7_len v8_wikirelax v9_wikitilt; do (python3 /workspace/eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy /workspace/data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) & done; wait; echo ROUND3_DONE > /tmp/r3.done' > /tmp/r3.log 2>&1 &\necho \"round3 training launched\"",
  "description": "Pack round 3 and launch training in background",
  "timeout": 180000
}

{
  "command": "python3 - <<'EOF'\nimport numpy as np, json, re, sys\nfrom transformers import AutoTokenizer\ntok=AutoTokenizer.from_pretrained('gpt2')\nd=np.load('data/multi_dev.npy'); nb=len(d)//4\nwiki=d[:nb]\ntxt=tok.decode(wiki)\n# how much of the wiki block is artifact surface form?\nat = len(re.findall(r'@[-,.]@', txt)); sp = len(re.findall(r' [,.;:)]', txt))\nallp = len(re.findall(r'[,.;:)]', txt))\nprint(f'wiki block: {nb} tokens, {len(txt)} chars')\nprint(f'  \" @-@ \"-style artifacts: {at}  -> ~{3*at} tokens ({300*at/nb:.1f}% of block)')\nprint(f'  space-before-punct: {sp}/{allp} = {100*sp/max(1,allp):.1f}% of punctuation')\n# token-level: which token ids are the artifact tokens\nids_at = tok(' @-@')['input_ids'], tok(' @,@')['input_ids']\nprint('  artifact token ids', ids_at)\nfrac = np.isin(wiki, np.array([2488, 12, 31, 11])).mean()\nprint(f'  share of wiki tokens in artifact set: {100*frac:.2f}%')\n# does the POOL contain any space-before-punct docs?\nimport itertools\nn=0; hits=0; ex=[]\nfor line in itertools.islice(open('data/pool.jsonl'),0,182016,4):\n    t=json.loads(line)['text']; n+=1\n    a=len(re.findall(r' [,.](?:\\s|$)',t)); b=len(re.findall(r'[,.]',t))\n    if b>40 and a/b>0.25:\n        hits+=1\n        if len(ex)<3: ex.append(t[:180])\nprint(f'pool docs (1/4 sample) with >25% space-before-punct: {hits}/{n}')\nfor e in ex: print('   ',repr(e))\nEOF\ncat /workspace/.timer/remaining_secs",
  "description": "Quantify wiki surface-form artifacts and pool matches"
}

{
  "command": "python3 - <<'EOF'\np='variants.py'; s=open(p).read()\ns=s.replace('''def build(toks, col=14''','''def minhash_sig(a, nperm=24, W=8, cap=3072):\n    \"\"\"MinHash signature over 8-token shingles (token-level, so it is markup-agnostic)\"\"\"\n    x = a[:cap].astype(np.int64)\n    if len(x) < W + 1:\n        return None\n    sh = np.lib.stride_tricks.sliding_window_view(x, W)\n    h = (sh * _PRIMES).sum(1) & 0x7FFFFFFF\n    return ((h[None, :] * _A + _B) % 2147483647).min(1)\n\n\n_PRIMES = np.array([2654435761, 40503, 97, 1000003, 31, 15485863, 32452843, 49979687], dtype=np.int64)\n_rng = np.random.default_rng(7)\n_A = _rng.integers(1, 2 ** 31 - 1, size=(24, 1))\n_B = _rng.integers(0, 2 ** 31 - 1, size=(24, 1))\n\n\ndef build(toks, col=14''')\ns=s.replace('''          len_pref=0.0, tail=6000, verbose=True, code_boost=0.0, wiki_relax=False):''',\n'''          len_pref=0.0, tail=6000, verbose=True, code_boost=0.0, wiki_relax=False,\n          resample_T=0.0, band_dedup=False, rng_seed=3):''')\n# score-weighted resampling (DSIR-style) instead of strict top-K, and LSH dedup\ns=s.replace('''    wmask = (BASE | relaxed_wiki_mask()) if wiki_relax else BASE\n    for r in range(4):\n        ok = (BASE | TECH) if r == 3 else (wmask if r == 0 else BASE)\n        orders.append(np.argsort(-np.where(ok, S[:, r], -1e9)))''',\n'''    wmask = (BASE | relaxed_wiki_mask()) if wiki_relax else BASE\n    rng = np.random.default_rng(rng_seed)\n    for r in range(4):\n        ok = (BASE | TECH) if r == 3 else (wmask if r == 0 else BASE)\n        o = np.argsort(-np.where(ok, S[:, r], -1e9))\n        if resample_T:\n            # DSIR-style: keep a wide band of plausible docs, then sample without\n            # replacement with prob ~ exp(score/T). Trades a little peak score for\n            # topical diversity, which is what a *different* sample of the same\n            # domain rewards.\n            band = o[:int(3.2 * quota[r] / 900)]\n            w = np.exp((S[band, r] - S[band, r].max()) / resample_T)\n            o = band[rng.permutation(len(band))] if resample_T > 90 else \\\\\n                band[np.argsort(-(rng.random(len(band)) ** (1.0 / np.maximum(w, 1e-12))))]\n        orders.append(o)\n    bands = {} if band_dedup else None''')\ns=s.replace('''            k = C.fingerprint(a)\n            if k in seen:\n                continue\n            seen.add(k); picked.add(did); per[r].append(did); used[r] += len(a) + 1''',\n'''            k = C.fingerprint(a)\n            if k in seen:\n                continue\n            if bands is not None:\n                sg = minhash_sig(a)\n                if sg is not None:\n                    keys = [(bi, tuple(sg[bi * 4:(bi + 1) * 4].tolist())) for bi in range(6)]\n                    if any(kk in bands for kk in keys):\n                        continue\n                    for kk in keys:\n                        bands[kk] = 1\n            seen.add(k); picked.add(did); per[r].append(did); used[r] += len(a) + 1''')\nopen(p,'w').write(s); print('patched resample+dedup')\nEOF\nsleep 40; ls /tmp/r3.done 2>/dev/null; grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_v6*.txt /tmp/log_v7*.txt /tmp/log_v8*.txt /tmp/log_v9*.txt 2>/dev/null | sed 's/\\/tmp\\/tr_//'; cat /workspace/.timer/remaining_secs",
  "description": "Add resampling and MinHash dedup, poll round 3"
}

{
  "command": "while [ ! -f /tmp/r3.done ]; do sleep 10; done\ngrep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_v6_techcomp.txt /tmp/log_v7_len.txt /tmp/log_v8_wikirelax.txt /tmp/log_v9_wikitilt.txt | sed 's/\\/tmp\\/tr_//'\ncat /workspace/.timer/remaining_secs",
  "description": "Wait for round 3 results",
  "timeout": 300000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\ncat > /tmp/r4.json <<'EOF'\n{\"v10_combo\":    {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3},\n \"v11_dedup\":    {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true},\n \"v12_resample\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"resample_T\":0.15},\n \"v13_techmix\":  {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275]}}\nEOF\npython3 variants.py /tmp/r4.json 2>&1 | grep -v \"Token indices\" | tail -9\nfor v in v10_combo v11_dedup v12_resample v13_techmix; do\n  python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1\ndone\nrm -f /tmp/r4.done\nnohup bash -c 'for v in v10_combo v11_dedup v12_resample v13_techmix; do (python3 /workspace/eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy /workspace/data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) & done; wait; echo done > /tmp/r4.done' > /tmp/r4.log 2>&1 &\necho LAUNCHED; cat /workspace/.timer/remaining_secs",
  "description": "Build, pack and launch round 4 combos",
  "timeout": 240000
}

{
  "file_path": "/workspace/claim.md",
  "content": "# Register coverage, not document \"quality\", is the binding constraint\n\n## Hypothesis\n\nFor a **broad, multi-register** evaluation target under a **small fixed token budget**\n(12M tokens ≈ 6% of the pool), the quantity that determines held-out perplexity is\nhow the budget is **allocated across the registers that make up the target**, not how\n\"high-quality\" the selected documents are in a register-agnostic sense.\n\nConcretely: the disclosed target is four equally-weighted registers (encyclopedic /\ngeneral HQ web prose / news / technical Q&A) and the reported metric is\n`exp(mean CE)` over windows drawn uniformly from all four — i.e. the *geometric mean*\nof four per-register perplexities. Because each per-register loss is convex-decreasing\nin that register's share of the training tokens, the optimum of the average is close to\n**equal token quotas matched to the evaluation weights**, and any selection rule that is\nblind to register — including a strong quality ranker — will starve the register that is\nrarest in the pool and pay for it super-linearly.\n\nOperationally I select documents by a **register-contrastive Moore–Lewis score**\n`s_r(d) = H_{¬r}(d) − H_r(d)` (per-word cross-entropy under a Dirichlet-smoothed backoff\nbigram model of register `r` vs. a mixture of the other registers plus a pool-estimated\nbackground), gate on mechanical junk filters, drop near-duplicates, and then fill\n**3M tokens per register**.\n\n## Mechanism → prediction on an observable other than final perplexity\n\nThe mechanism is per-register loss reallocation. It is directly observable in the\n**four per-register block perplexities** of the held-out target (the dev target is four\ncontiguous 250k-token blocks), so I predicted *before* tuning any mixture:\n\n1. **The technical Q&A block is the most responsive** to register-matched data, because\n   it is the register that generic web crawl covers worst.\n2. **A register-blind \"quality-only\" ranking will raise the techqa block far above the\n   random baseline** while leaving the prose blocks roughly unchanged — i.e. it converts\n   a broad target into a narrow one.\n3. **The encyclopedic block will respond least**, because its difficulty is dominated by\n   a *surface form* that no amount of selection can supply: it is WikiText-style\n   detokenized text — measured on the dev block, **94.5% of its punctuation marks are\n   space-separated** (` ,` / ` .`) and **2.8% of its tokens are ` @-@ `/` @,@ ` artifact\n   tokens**. A scan of the pool finds only ~112 documents in 182,016 with that\n   punctuation style, so the mismatch is irreducible by curation.\n\nAll three predictions held (dev target, frozen trainer, seed 1337):\n\n| selection | overall PPL | wiki | webprose | news | techqa |\n|---|---|---|---|---|---|\n| random 12M (do-nothing baseline) | 485.5 | 1021.7 | 335.0 | 446.7 | 363.7 |\n| **register-blind quality top-K** | **577.3** | 1094.1 | 284.9 | 428.0 | **830.1** |\n| register-balanced (this method) | 380.5 | 930.7 | 281.4 | 350.5 | 227.5 |\n| final (balanced + technical/encyclopedic recall) | **~377** | 883–929 | ~283 | ~347 | ~214 |\n\nReading the mechanism off the blocks: balancing buys **−0.47 nats on techqa** but only\n**−0.09 nats on wiki** (prediction 1 and 3), and the register-blind ranker — which\n*looks* like a better quality filter and is the standard \"train on the cleanest docs\"\nrecipe — sends techqa from 364 → **830** (+1.03 nats) with webprose unchanged\n(335 → 285 ≈ its balanced value), making it **worse than doing nothing at all**\n(577 vs 485) (prediction 2).\n\n## Falsification\n\nThe claim is falsifiable and I ran the tests that would break it:\n\n- **If register balance were not the operative variable**, the register-blind quality\n  ranking should have matched or beaten the balanced selection at equal budget.\n  It lost by 52% (577.3 vs 380.5) — *and* lost to random. Not falsified.\n- **If the rule were \"spend the budget where the loss is highest\"** rather than \"match\n  the evaluation weights / equalize marginal gain\", then tilting the budget toward the\n  worst block (wiki, ~930 ppl) should have helped. Tilting wiki to 30% gave **386.7**\n  vs 377.0 for the equal quota — worse. Symmetrically, over-tilting toward the most\n  responsive block (techqa 33%) gave **382.2** — also worse. The claim survives in its\n  specific form: allocate by *evaluation weight / marginal gain*, not by absolute loss.\n- **A cheap confound would be document length or raw count.** Preferring long documents\n  (same scores, same quotas, 4.2k docs instead of 12.9k) gave **388.4** — worse, so the\n  gain is not \"fewer, bigger documents\".\n- **Remaining way to falsify:** if the hidden target's register mixture is *not* the\n  disclosed equal four-way mix, an equal quota is mis-specified and a selection tuned to\n  the true mixture would beat it. The method exposes this as one vector (`mix`), so the\n  test is a single re-run.\n\n## Transfer\n\nThe recipe needs **no labels, no GPU, and no internet**: decode the disclosed dev target,\nsplit it into its registers, fit cheap hashed n-gram models per register plus a background\nmodel from the pool itself, score every pool document by the contrastive cross-entropy\ndifference, and fill a per-register token quota equal to the register's weight in the\nevaluation. Scoring 182,016 documents takes ~90s on 15 CPU cores, so the whole pipeline is\ndominated by tokenization, not by selection.\n\nIt transfers to (a) other budgets — the quota vector is the only thing that changes;\n(b) other target mixtures, including unequal ones, by setting `mix` to the evaluation\nweights; and (c) other pools, since the background model is estimated from the pool being\nfiltered rather than from an external corpus.\n\nThe transferable *negative* result is equally useful: **a register-blind quality filter is\nnot safe for a broad target.** Under a tight budget it can be worse than random selection,\nand the damage is invisible in the aggregate number until you break perplexity out by\nregister. The other boundary condition is prediction 3 — when a target register's\n*surface form* is absent from the pool (detokenized WikiText here), curation cannot\nrecover it; that gain has to come from upstream text normalization instead.\n"
}

{
  "command": "while [ ! -f /tmp/r4.done ]; do sleep 10; done\nfor v in v10_combo v11_dedup v12_resample v13_techmix; do grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_$v.txt | sed 's/\\/tmp\\/tr_//'; done\ncat /workspace/.timer/remaining_secs",
  "description": "Collect round 4 results",
  "timeout": 300000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\ncat > /tmp/r5.json <<'EOF'\n{\"v14_t30\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.215,0.2425,0.2425,0.30]},\n \"v15_t34\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.20,0.23,0.23,0.34]},\n \"v16_cb2\":  {\"col\":18,\"code_boost\":2.2,\"qa_boost\":0.8,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275]},\n \"v17_vsbg\": {\"col\":14,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275]}}\nEOF\npython3 variants.py /tmp/r5.json 2>&1 | grep -v \"Token indices\" | tail -9\nfor v in v14_t30 v15_t34 v16_cb2 v17_vsbg; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done\nrm -f /tmp/r5.done\nnohup bash -c 'for v in v14_t30 v15_t34 v16_cb2 v17_vsbg; do (python3 /workspace/eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy /workspace/data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) & done; wait; echo done > /tmp/r5.done' > /tmp/r5.log 2>&1 &\necho LAUNCHED",
  "description": "Build, pack, launch round 5",
  "timeout": 240000
}

{
  "file_path": "/workspace/submission/curate_final.py",
  "content": "#!/usr/bin/env python3\n\"\"\"\nCurate a 12M-token pretraining selection from a raw, unlabeled web pool for a BROAD\nmulti-register high-quality English target (encyclopedic / general web prose / news /\ntechnical Q&A).   No labels, no internet, no GPU: ~3 min end-to-end on 15 CPU cores.\n\n=============================== STATED CRITERION ===============================\nLet the disclosed dev target be split into its R = 4 equally-weighted registers\n(the target file is four contiguous equal blocks).  For each register r we fit a\nDirichlet-smoothed backoff bigram model over hashed word n-grams, plus (a) a\nBACKGROUND model estimated from the pool itself and (b) a \"not-r\" model (equal\nmixture of the other registers and the background).  A document d is scored by the\nregister-contrastive Moore-Lewis cross-entropy difference\n\n        s_r(d) = H_{not r}(d) - H_r(d)                                       (1)\n\ni.e. how much better register r explains d than everything else does.  Technical\nrecall is sharpened by additive bonuses for code/Q&A surface markers, and\nencyclopedic recall by a bonus for wiki-derived markers (see BONUS below).\n\nA document is ELIGIBLE if it passes fixed mechanical junk filters (length,\nEnglish-ness via stopword rate, non-ASCII rate, mean word length, terminal-\npunctuation rate, duplicate-line rate, caps/digit rate, boilerplate density,\nadult/spam terms).  The digit/table/boilerplate rules are relaxed for the\nencyclopedic register (dates and infobox rows are signal there, not noise) and the\nprose-shape rules are relaxed for documents that carry >= 3 code markers.\n\nSELECTION = fill a per-register TOKEN QUOTA equal to that register's weight in the\nevaluation, taking documents in descending s_r(d) and skipping near-duplicates\n(MinHash LSH over 8-token shingles).  The quota vector MIX is the only tuned knob;\nit is near-uniform because the metric averages loss uniformly over the registers.\nDocuments are emitted round-robin across registers so that ANY prefix of the list\n(the trainer consumes the list in order until the budget is full) is register-\nbalanced, followed by a tail buffer of next-best ids that guards the 12M floor.\n================================================================================\n\nUsage:  python3 curate_final.py [--out selection.json] [--mix w,x,y,z]\nStages are cached under /tmp/curate_cache so re-runs are cheap.\n\"\"\"\nimport argparse, html, json, os, re, sys, zlib\nimport multiprocessing as mp\nimport numpy as np\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV = \"/workspace/data/multi_dev.npy\"\nCACHE = \"/tmp/curate_cache\"\nBUDGET = 12_000_000\nN_REG = 4\nREG_NAMES = [\"wiki\", \"webprose\", \"news\", \"techqa\"]\nos.makedirs(CACHE, exist_ok=True)\n\nWORD_RE = re.compile(r\"[a-z]+|[0-9]+|[^\\sa-z0-9]\")\nV_UNI = 1 << 19            # hashed unigram table\nV_BI = 1 << 21             # hashed bigram table\nMAXW = 6000                # words scored per document\nALPHA = 6.0                # Dirichlet backoff strength (bigram -> unigram)\n\n# ---- tuned selection knobs (chosen on the dev target; see claim.md) -----------\nMIX = (0.225, 0.25, 0.25, 0.275)   # token quota per register\nCODE_BOOST = 1.2                   # additive bonus, techqa score, per code marker (capped)\nQA_BOOST = 0.5                     # additive bonus, techqa score, per Q&A marker (capped)\nWIKI_BOOST = 0.3                   # additive bonus, wiki score, per wiki marker (capped)\n\n\n# --------------------------------------------------------------------- hashing\n_HMEMO = {}\n\n\ndef hash_words(words):\n    \"\"\"word -> deterministic 32-bit id (crc32; independent of PYTHONHASHSEED)\"\"\"\n    m = _HMEMO\n    out = []\n    for w in words:\n        h = m.get(w)\n        if h is None:\n            h = m[w] = zlib.crc32(w.encode(\"utf-8\", \"ignore\"))\n        out.append(h)\n    return np.array(out, dtype=np.int64)\n\n\ndef feat_ids(words):\n    h = hash_words(words)\n    u = h % V_UNI\n    if len(h) < 2:\n        return u, np.zeros(0, dtype=np.int64)\n    return u, (h[:-1] * 1000003 + h[1:]) % V_BI\n\n\n# ------------------------------------------------------------------ references\ndef normalize_ref(text, kind):\n    \"\"\"strip surface artifacts so a reference measures register/content, not markup\"\"\"\n    if kind == \"wiki\":                       # WikiText-103 detokenization artifacts\n        text = text.replace(\" @-@ \", \"-\").replace(\" @,@ \", \",\").replace(\" @.@ \", \".\")\n        text = re.sub(r\" ([,.;:!?\\)\\]])\", r\"\\1\", text)\n        text = re.sub(r\"([\\(\\[]) \", r\"\\1\", text)\n    if kind == \"techqa\":                     # StackExchange HTML wrapper\n        text = re.sub(r\"<[^>\\n]{1,40}>\", \" \", text)\n        text = html.unescape(text)\n    return text\n\n\ndef build_references():\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    d = np.load(DEV)\n    n = len(d) // N_REG\n    out = []\n    for r in range(N_REG):\n        t = tok.decode(d[r * n:(r + 1) * n]).replace(\"<|endoftext|>\", \"\\n\")\n        out.append(normalize_ref(t, REG_NAMES[r] if REG_NAMES[r] in (\"wiki\", \"techqa\") else \"x\"))\n    return out\n\n\ndef count_model(texts):\n    uc = np.zeros(V_UNI, dtype=np.float64)\n    bc = np.zeros(V_BI, dtype=np.float64)\n    for t in texts:\n        w = WORD_RE.findall(t.lower())\n        for i in range(0, len(w), 200000):\n            u, b = feat_ids(w[i:i + 200000])\n            uc += np.bincount(u, minlength=V_UNI)\n            bc += np.bincount(b, minlength=V_BI)\n    return uc, bc\n\n\ndef logprob_tables(uc, bc):\n    return uc, (uc + 0.25) / (uc.sum() + 0.25 * V_UNI), bc\n\n\ndef doc_ce(u, b, model):\n    \"\"\"per-word cross-entropy under a Dirichlet-smoothed backoff bigram model\"\"\"\n    uc, p_uni, bc = model\n    pb = (bc[b] + ALPHA * p_uni[u][1:]) / (uc[u[:-1]] + ALPHA)\n    return -np.log(pb).mean()\n\n\n# -------------------------------------------------------------------- features\nSTOP = set(\"the and of to in a is that for it was as on with be by are this at from or an \"\n           \"have has not but they he she we you\".split())\nBOILER = [\"sign in\", \"log in\", \"cookie\", \"privacy policy\", \"all rights reserved\", \"click here\",\n          \"javascript\", \"terms of service\", \"subscribe\", \"read more\", \"share this\", \"posted by\",\n          \"add to cart\", \"©\", \"follow us\", \"newsletter\", \"advertisement\", \"skip to content\",\n          \"leave a reply\", \"your email address\", \"no comments\", \"related posts\", \"tags:\"]\nNAUGHTY = [\"porn\", \"xxx\", \"sex chat\", \"viagra\", \"casino\", \"escort\", \"nude\", \"milf\",\n           \"camgirl\", \"betting odds\", \"payday loan\", \"cialis\", \"hentai\", \"fuck\", \"slut\"]\nCODE_RE = [re.compile(p, re.M | re.I) for p in [\n    r\"[{};]\\s*\\n\", r\"\\b\\w+\\([\\w\\$\\\"']\",\n    r\"^\\s*(?:import |from \\w+ import|#include|using \\w|require\\(|def |class |function |public |\"\n    r\"private |var |const |let |\\$)\", r\"&(?:lt|gt|quot|amp);\",\n    r\"(?:sudo |npm |pip install|apt-get|git clone|SELECT .* FROM|</?\\w+>|==|!=|=>|->|\\[\\]|::)\"]]\nWIKI_RE = [re.compile(p, re.M | re.I) for p in [\n    r\"\\[edit\\]\", r\"\\bcitation needed\\b\", r\"Retrieved from\", r\"External links\", r\"^\\|.*\\|\\|\",\n    r\"\\bFrom Wikipedia\\b\", r\"^References$\", r\"\\bdisambiguation\\b\", r\"\\bthumb\\|\", r\"\\{\\{\"]]\nQA_RE = [re.compile(p, re.M | re.I) for p in [\n    r\"^(Q|A):\", r\"\\banswer\\b.*\\bquestion\\b\", r\"\\bany (ideas|help|suggestions)\\b\",\n    r\"\\bthanks in advance\\b\", r\"\\bI'?m trying to\\b\", r\"\\bhow (do|can) i\\b\"]]\nTERM_PUNCT = ('.', '!', '?', '\"', \"'\", ')', ':', ';', '”', '’')\nNF = 24   # 14 shape + 4 vs-background + 4 contrastive + wiki/qa marker counts\n\n\ndef doc_features(text, models):\n    head, body = text[:20000], text[:60000]\n    nh = max(1, len(head))\n    low = body.lower()\n    words = WORD_RE.findall(low)\n    alpha = [w for w in words if w[0].isalpha()]\n    nw = max(1, len(alpha))\n    lines = [l for l in text.split(\"\\n\") if l.strip()]\n    nl = max(1, len(lines))\n    letters = sum(c.isalpha() for c in head)\n    f = np.zeros(NF, dtype=np.float32)\n    f[0] = len(text)\n    f[1] = len(alpha)\n    f[2] = sum(1 for w in alpha if w in STOP) / nw               # English-ness\n    f[3] = sum(1 for c in head if ord(c) > 127) / nh\n    f[4] = sum(len(w) for w in alpha) / nw\n    f[5] = sum(1 for l in lines if l.rstrip().endswith(TERM_PUNCT)) / nl\n    f[6] = 1.0 - len(set(lines)) / nl                            # duplicate lines\n    f[7] = sum(1 for c in head if c.isupper()) / max(1, letters)\n    f[8] = sum(1 for c in head if c.isdigit()) / nh\n    f[9] = sum(low.count(b) for b in BOILER) / (nw / 100.0 + 1)\n    f[10] = sum(low.count(b) for b in NAUGHTY)\n    f[11] = sum(1 for r in CODE_RE if r.search(body))\n    f[12] = sum(len(r.findall(body)) for r in CODE_RE[:2]) / (nw / 100.0 + 1)\n    f[13] = np.mean([len(l) for l in lines]) if lines else 0\n    w = words[:MAXW]\n    if len(w) >= 30:\n        u, b = feat_ids(w)\n        ce = np.array([doc_ce(u, b, m) for m in models])\n        f[14:18] = ce[N_REG] - ce[:N_REG]                    # H_background - H_r\n        f[18:22] = ce[N_REG + 1:2 * N_REG + 1] - ce[:N_REG]  # H_{not r}   - H_r   (eq. 1)\n    else:\n        f[14:22] = -10.0\n    t40 = text[:40000]\n    f[22] = sum(1 for r in WIKI_RE if r.search(t40))\n    f[23] = sum(1 for r in QA_RE if r.search(t40))\n    return f\n\n\n# ------------------------------------------------------------- parallel scoring\n_G = {}\n\n\ndef _init(models):\n    _G[\"models\"] = models\n\n\ndef _work(args):\n    start, end = args\n    fs, ids = [], []\n    with open(POOL, \"rb\") as fh:\n        if start:\n            fh.seek(start - 1)\n            fh.readline()\n        while fh.tell() < end:\n            line = fh.readline()\n            if not line:\n                break\n            try:\n                r = json.loads(line)\n            except Exception:\n                continue\n            ids.append(r[\"id\"])\n            fs.append(doc_features(r[\"text\"], _G[\"models\"]))\n    return np.array(ids, dtype=np.int64), np.array(fs, dtype=np.float32)\n\n\ndef score_pool(nproc=15):\n    cf = os.path.join(CACHE, \"scores24.npz\")\n    if os.path.exists(cf):\n        z = np.load(cf)\n        return z[\"ids\"], z[\"feats\"]\n    print(\"[1/3] fitting register / background / not-r language models ...\", flush=True)\n    refs = build_references()\n    raw = [count_model([refs[r]]) for r in range(N_REG)]\n    bg = [json.loads(l)[\"text\"] for i, l in enumerate(open(POOL)) if i % 13 == 0]\n    print(f\"      background sample: {len(bg)} pool docs\", flush=True)\n    raw.append(count_model(bg))\n    del bg\n    T = 2.0e6\n    scale = [T / max(1.0, uc.sum()) for uc, _ in raw]\n    models = [logprob_tables(*raw[r]) for r in range(N_REG + 1)]\n    for r in range(N_REG):\n        models.append(logprob_tables(\n            sum(raw[q][0] * scale[q] for q in range(N_REG + 1) if q != r),\n            sum(raw[q][1] * scale[q] for q in range(N_REG + 1) if q != r)))\n    sz = os.path.getsize(POOL)\n    bounds = [(sz * k // nproc, sz * (k + 1) // nproc) for k in range(nproc)]\n    print(f\"[2/3] scoring the pool on {nproc} procs ...\", flush=True)\n    with mp.Pool(nproc, initializer=_init, initargs=(models,)) as p:\n        res = p.map(_work, bounds)\n    ids = np.concatenate([r[0] for r in res])\n    feats = np.concatenate([r[1] for r in res])\n    o = np.argsort(ids)\n    ids, feats = ids[o], feats[o]\n    np.savez(cf, ids=ids, feats=feats)\n    print(f\"      scored {len(ids)} docs\", flush=True)\n    return ids, feats\n\n\n# --------------------------------------------------------------- tokenizer cache\ndef _tok_work(chunk):\n    from transformers import AutoTokenizer\n    tok = _G.get(\"tok\")\n    if tok is None:\n        tok = _G[\"tok\"] = AutoTokenizer.from_pretrained(\"gpt2\")\n    ids, texts = chunk\n    return ids, [np.array(e, dtype=np.uint16)\n                 for e in tok(texts, add_special_tokens=False)[\"input_ids\"]]\n\n\ndef tokenize_subset(cand_ids, tag, nproc=15):\n    \"\"\"exact GPT-2 token ids for a candidate subset (quotas are counted in real tokens)\"\"\"\n    cf = os.path.join(CACHE, f\"tok_{tag}.npz\")\n    if os.path.exists(cf):\n        z = np.load(cf)\n        flat, off, tid = z[\"flat\"], z[\"off\"], z[\"ids\"]\n        return {int(tid[k]): flat[off[k]:off[k + 1]] for k in range(len(tid))}\n    want = set(int(i) for i in cand_ids)\n    tids, texts = [], []\n    for line in open(POOL):\n        r = json.loads(line)\n        if r[\"id\"] in want:\n            tids.append(r[\"id\"])\n            texts.append(r[\"text\"])\n    step = max(1, len(tids) // (nproc * 4))\n    chunks = [(tids[i:i + step], texts[i:i + step]) for i in range(0, len(tids), step)]\n    os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n    with mp.Pool(nproc, initializer=_init, initargs=(None,)) as p:\n        res = p.map(_tok_work, chunks)\n    out = {}\n    for ii, aa in res:\n        for i, arr in zip(ii, aa):\n            out[int(i)] = arr\n    keys = sorted(out)\n    off = np.concatenate([[0], np.cumsum([len(out[k]) for k in keys])]).astype(np.int64)\n    np.savez(cf, ids=np.array(keys, dtype=np.int64), off=off,\n             flat=np.concatenate([out[k] for k in keys]))\n    return out\n\n\n# ------------------------------------------------------------------ near-dup LSH\n_PR = np.array([2654435761, 40503, 97, 1000003, 31, 15485863, 32452843, 49979687], dtype=np.int64)\n_rng = np.random.default_rng(7)\n_A = _rng.integers(1, 2 ** 31 - 1, size=(24, 1))\n_B = _rng.integers(0, 2 ** 31 - 1, size=(24, 1))\n\n\ndef minhash_sig(a, W=8, cap=3072):\n    \"\"\"MinHash over 8-token shingles: token-level, so it is markup/format agnostic\"\"\"\n    x = a[:cap].astype(np.int64)\n    if len(x) < W + 1:\n        return None\n    h = (np.lib.stride_tricks.sliding_window_view(x, W) * _PR).sum(1) & 0x7FFFFFFF\n    return ((h[None, :] * _A + _B) % 2147483647).min(1)\n\n\n# ---------------------------------------------------------------------- filters\ndef keep_mask(F, tech=False, wiki=False):\n    \"\"\"mechanical junk filters; `tech` relaxes prose shape, `wiki` relaxes digits/tables\"\"\"\n    nb, nal, stop, na, awl, tp, dup, up, dig, boil, bad = [F[:, j] for j in range(11)]\n    m = (nb > 700) & (nb < 500_000) & (nal > 110) & (na < 0.06) & (dup < 0.30) & (bad <= 1)\n    m &= awl > 3.0\n    if tech:\n        m &= (stop > 0.045) & (awl < 11.0) & (tp > 0.15) & (up < 0.16) & (dig < 0.14) & (boil < 5.0)\n    elif wiki:\n        m &= (stop > 0.055) & (awl < 8.5) & (tp > 0.25) & (up < 0.18) & (dig < 0.22) & (boil < 3.5)\n    else:\n        m &= (stop > 0.075) & (awl < 8.0) & (tp > 0.40) & (up < 0.16) & (dig < 0.09) & (boil < 2.2)\n    return m\n\n\n# -------------------------------------------------------------------- selection\ndef build_selection(ids, F, mix=MIX, tail=6000, verbose=True):\n    S = F[:, 18:22].copy()                                   # contrastive scores, eq. (1)\n    S[:, 0] += WIKI_BOOST * np.minimum(F[:, 22], 3.0)        # encyclopedic recall\n    S[:, 3] += QA_BOOST * np.minimum(F[:, 23], 3.0)          # technical recall\n    S[:, 3] += CODE_BOOST * np.minimum(F[:, 11], 4.0) / 4.0\n    base = keep_mask(F)\n    elig = [base | keep_mask(F, wiki=True), base, base,\n            base | (keep_mask(F, tech=True) & (F[:, 11] >= 3))]\n    quota = [m * BUDGET for m in mix]\n    orders = [np.argsort(-np.where(elig[r], S[:, r], -1e9)) for r in range(N_REG)]\n\n    # candidate superset -> exact token counts (a generous multiple of each quota)\n    cand = set()\n    for r in range(N_REG):\n        cand.update(ids[orders[r][:int(4.0 * quota[r] / 900) + 4000]].tolist())\n    if verbose:\n        print(f\"[3/3] eligible: {base.sum()} prose-clean of {len(ids)}; \"\n              f\"tokenizing {len(cand)} candidates\", flush=True)\n    toks = tokenize_subset(sorted(cand), \"final\")\n\n    picked, bands, per, used, ndup = set(), {}, [[] for _ in range(N_REG)], [0.0] * N_REG, 0\n    for r in range(N_REG):\n        for i in orders[r]:\n            if used[r] >= quota[r]:\n                break\n            did = int(ids[i])\n            if did in picked:\n                continue\n            a = toks.get(did)\n            if a is None or len(a) < 64:\n                continue\n            sg = minhash_sig(a)\n            if sg is not None:\n                keys = [(bi, tuple(sg[bi * 4:(bi + 1) * 4].tolist())) for bi in range(6)]\n                if any(k in bands for k in keys):\n                    ndup += 1\n                    continue\n                for k in keys:\n                    bands[k] = 1\n            picked.add(did)\n            per[r].append(did)\n            used[r] += len(a) + 1\n    if verbose:\n        for r in range(N_REG):\n            print(f\"      {REG_NAMES[r]:9s} {len(per[r]):5d} docs  {used[r]/1e6:.2f}M tok \"\n                  f\"(quota {quota[r]/1e6:.2f}M)\", flush=True)\n        print(f\"      near-duplicates skipped: {ndup}\", flush=True)\n\n    sel = []                                   # round-robin: every prefix stays balanced\n    for j in range(max(len(x) for x in per)):\n        for r in range(N_REG):\n            if j < len(per[r]):\n                sel.append(per[r][j])\n    extra = []                                 # tail buffer past the budget\n    for r in range(N_REG):\n        for i in orders[r]:\n            did = int(ids[i])\n            if did not in picked and toks.get(did) is not None and len(extra) < tail:\n                picked.add(did)\n                extra.append(did)\n    return sel + extra, toks\n\n\ndef main():\n    ap = argparse.ArgumentParser()\n    ap.add_argument(\"--out\", default=\"/workspace/submission/selection.json\")\n    ap.add_argument(\"--mix\", default=\",\".join(str(m) for m in MIX))\n    a = ap.parse_args()\n    mix = [float(x) for x in a.mix.split(\",\")]\n    ids, F = score_pool()\n    sel, toks = build_selection(ids, F, mix=mix)\n    assert len(sel) == len(set(sel)), \"duplicate ids\"\n    assert min(sel) >= 0 and max(sel) < len(ids), \"id out of range\"\n    tot = sum(len(toks[i]) + 1 for i in sel)\n    json.dump(sel, open(a.out, \"w\"))\n    print(f\"wrote {len(sel)} ids, {tot/1e6:.2f}M tokens available \"\n          f\"for a {BUDGET/1e6:.0f}M budget -> {a.out}\")\n\n\nif __name__ == \"__main__\":\n    main()\n"
}

{
  "command": "while [ ! -f /tmp/r5.done ]; do sleep 10; done\nfor v in v14_t30 v15_t34 v16_cb2 v17_vsbg; do grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_$v.txt | sed 's/\\/tmp\\/tr_//'; done\ncat /workspace/.timer/remaining_secs",
  "description": "Collect round 5 results",
  "timeout": 300000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\nmv submission/curate.py /tmp/curate_v1_scoring.py && mv submission/curate_final.py submission/curate.py\ntime python3 submission/curate.py --out /workspace/submission/selection.json 2>&1 | grep -v \"Token indices\" | tail -12\ncat /workspace/.timer/remaining_secs",
  "description": "Regenerate selection with final curate.py",
  "timeout": 420000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\nnohup bash -c 'python3 /workspace/pack_selection.py /workspace/submission/selection.json /tmp/off_train.npy > /tmp/off_pack.log 2>&1; echo done > /tmp/off_pack.done' >/dev/null 2>&1 &\ncat > /tmp/r6.json <<'EOF'\n{\"v18_wikib\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.7,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275]},\n \"v19_mix\":   {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.24,0.24,0.24,0.28]},\n \"v20_qab\":   {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.9,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275]}}\nEOF\npython3 variants.py /tmp/r6.json 2>&1 | grep -v \"Token indices\" | tail -6\nfor v in v18_wikib v19_mix v20_qab; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done\nwhile [ ! -f /tmp/off_pack.done ]; do sleep 5; done; cat /tmp/off_pack.log | tail -2\npython3 -c \"\nimport numpy as np\na=np.load('/tmp/off_train.npy'); b=np.load('/tmp/tr_v13_techmix.npy')\nprint('official pack', a.shape, a.dtype, 'v13 pack', b.shape)\nprint('identical to v13 pack:', np.array_equal(a,b))\nprint('first 20 tokens equal:', np.array_equal(a[:20],b[:20]))\"",
  "description": "Run official packer and build final probes",
  "timeout": 500000
}

{
  "command": "cd /workspace\nnohup bash -c 'python3 /workspace/eval_blocks.py --train_npy /tmp/off_train.npy --heldout_npy /workspace/data/multi_dev.npy --out_json /tmp/e_official.json > /tmp/log_official.txt 2>&1; echo done > /tmp/off_train.done' >/dev/null 2>&1 &\npython3 - <<'EOF'\np='variants.py'; s=open(p).read()\ns=s.replace('import curate as C','''import curate as C\n\n\ndef _fingerprint(arr):\n    a = arr[:512].astype(np.int64)\n    if len(a) < 16:\n        return a.tobytes()\n    sh = np.lib.stride_tricks.sliding_window_view(a, 12)\n    h = (sh * np.arange(1, 13, dtype=np.int64) * 2654435761).sum(1) & 0x7FFFFFFF\n    return tuple(np.sort(h)[:8].tolist())''')\ns=s.replace('C.fingerprint(a)','_fingerprint(a)')\nopen(p,'w').write(s); print('fixed variants.py')\nEOF\nexport TOKENIZERS_PARALLELISM=false\npython3 variants.py /tmp/r6.json 2>&1 | grep -v \"Token indices\" | tail -5\nfor v in v18_wikib v19_mix v20_qab; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done\ncat /workspace/.timer/remaining_secs",
  "description": "Launch official verification and rebuild probes",
  "timeout": 400000
}

{
  "command": "cd /workspace\nfor v in v18_wikib v19_mix v20_qab; do\n  (python3 eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) &\ndone\nwait; while [ ! -f /tmp/off_train.done ]; do sleep 5; done\necho \"=== OFFICIAL (frozen pack_selection.py + submission/selection.json) ===\"; grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_official.txt\necho \"=== probes ===\"; for v in v18_wikib v19_mix v20_qab; do grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_$v.txt | sed 's/\\/tmp\\/tr_//'; done\ncat /workspace/.timer/remaining_secs",
  "description": "Train probes and collect official verification",
  "timeout": 450000
}

{
  "command": "python3 - <<'EOF'\nimport json, re, os, numpy as np, multiprocessing as mp\nPOOL='data/pool.jsonl'\nPUNC=re.compile(r'[,.;:)]'); SPP=re.compile(r' [,.;:)]')\ndef work(rng):\n    s,e=rng; out=[]\n    with open(POOL,'rb') as f:\n        if s: f.seek(s-1); f.readline()\n        while f.tell()<e:\n            l=f.readline()\n            if not l: break\n            r=json.loads(l); t=r['text']\n            b=len(PUNC.findall(t))\n            out.append((r['id'], len(SPP.findall(t))/b if b>30 else 0.0, b))\n    return out\nsz=os.path.getsize(POOL); n=15\nwith mp.Pool(n) as p: res=p.map(work,[(sz*k//n,sz*(k+1)//n) for k in range(n)])\nrows=[x for r in res for x in r]\nids=np.array([r[0] for r in rows]); rat=np.array([r[1] for r in rows],dtype=np.float32)\no=np.argsort(ids); ids,rat=ids[o],rat[o]\nnp.savez('/tmp/curate_cache/spp.npz', ids=ids, ratio=rat)\nfor th in (0.5,0.3,0.2,0.12,0.08,0.05):\n    m=rat>=th\n    print(f'  ratio>={th}: {m.sum()} docs')\nz=np.load('/tmp/curate_cache/scores24.npz'); F=z['feats']\nimport sys; sys.path.insert(0,'submission'); import curate as C\nbase=C.keep_mask(F)|C.keep_mask(F,wiki=True)\nfor th in (0.3,0.2,0.12,0.08):\n    m=(rat>=th)&base\n    print(f'  ratio>={th} & eligible: {m.sum()} docs, ~{F[m,0].sum()/4.0/1e6:.2f}M tokens')\nEOF\ncat /workspace/.timer/remaining_secs",
  "description": "Scan pool for space-before-punctuation surface matches",
  "timeout": 300000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\npython3 - <<'EOF'\np='variants.py'; s=open(p).read()\ns=s.replace(\"MK = np.load(\\\"/tmp/curate_cache/markers.npz\\\")\",\n\"\"\"SPP = np.load(\"/tmp/curate_cache/spp.npz\")[\"ratio\"]\nMK = np.load(\"/tmp/curate_cache/markers.npz\")\"\"\")\ns=s.replace('''          resample_T=0.0, band_dedup=False, rng_seed=3):''',\n            '''          resample_T=0.0, band_dedup=False, rng_seed=3, surf_tokens=0.0, surf_min=0.12):''')\n# prepend surface-form matched docs into the encyclopedic bucket\ns=s.replace('''    for r in range(4):\n        for i in orders[r]:\n            if used[r] >= quota[r]:\n                break''','''    if surf_tokens:\n        cands = np.where((SPP >= surf_min) & (BASE | relaxed_wiki_mask()))[0]\n        cands = cands[np.argsort(-SPP[cands])]\n        orders[0] = np.concatenate([cands, orders[0]])\n        quota_surf = surf_tokens\n    for r in range(4):\n        for i in orders[r]:\n            if used[r] >= quota[r]:\n                break\n            if r == 0 and surf_tokens and used[0] < quota_surf and SPP[i] < surf_min:\n                continue   # reserve the first surf_tokens of the wiki quota for surface matches''')\nopen(p,'w').write(s); print('patched surf knob')\nEOF\ncat > /tmp/r7.json <<'EOF'\n{\"v21_surf09\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":900000},\n \"v22_surf18\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.3,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":1800000,\"surf_min\":0.06},\n \"v23_surfwb\":  {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.7,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":900000}}\nEOF\npython3 variants.py /tmp/r7.json 2>&1 | grep -v \"Token indices\" | tail -5\nfor v in v21_surf09 v22_surf18 v23_surfwb; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done",
  "description": "Build and pack surface-form variants",
  "timeout": 400000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\npython3 - <<'EOF'\np='variants.py'; s=open(p).read()\n# correct dose control: cap surface docs at surf_tokens, never skip ordinary docs\ns=s.replace('''    if surf_tokens:\n        cands = np.where((SPP >= surf_min) & (BASE | relaxed_wiki_mask()))[0]\n        cands = cands[np.argsort(-SPP[cands])]\n        orders[0] = np.concatenate([cands, orders[0]])\n        quota_surf = surf_tokens''','''    surf_used = 0.0\n    if surf_tokens:\n        cands = np.where((SPP >= surf_min) & (BASE | relaxed_wiki_mask()))[0]\n        cands = cands[np.argsort(-SPP[cands])]\n        orders[0] = np.concatenate([cands, orders[0]])''')\ns=s.replace('''            if r == 0 and surf_tokens and used[0] < quota_surf and SPP[i] < surf_min:\n                continue   # reserve the first surf_tokens of the wiki quota for surface matches''',\n'''            if surf_tokens and SPP[i] >= surf_min and surf_used >= surf_tokens:\n                continue   # surface-form dose already met''')\ns=s.replace('''            seen.add(k); picked.add(did); per[r].append(did); used[r] += len(a) + 1''',\n'''            seen.add(k); picked.add(did); per[r].append(did); used[r] += len(a) + 1\n            if surf_tokens and SPP[i] >= surf_min:\n                surf_used += len(a) + 1''')\n# include surface-form docs in the tokenized superset\ns=s.replace('''    s.update(IDS[(WIKI_MK >= 1) & BASE].tolist())''',\n'''    s.update(IDS[(WIKI_MK >= 1) & BASE].tolist())\n    s.update(IDS[(SPP >= 0.05) & (BASE | relaxed_wiki_mask())].tolist())''')\ns=s.replace('toks = C.tokenize_subset(cand, \"super\")','toks = C.tokenize_subset(cand, \"super2\")')\nopen(p,'w').write(s); print('fixed')\nEOF\npython3 variants.py /tmp/r7.json 2>&1 | grep -v \"Token indices\" | tail -5\nfor v in v21_surf09 v22_surf18 v23_surfwb; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done\ncat /workspace/.timer/remaining_secs",
  "description": "Fix dose logic, rebuild surface variants",
  "timeout": 400000
}

{
  "command": "cd /workspace\nfor v in v21_surf09 v22_surf18 v23_surfwb; do\n  (python3 eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) &\ndone\nwait\nfor v in v21_surf09 v22_surf18 v23_surfwb; do grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_$v.txt | sed 's/\\/tmp\\/tr_//'; done\necho \"--- reference: official selection.json = 359.50 (wiki 870.8) ; v13 = 355.57\"\ncat /workspace/.timer/remaining_secs",
  "description": "Train surface-form variants",
  "timeout": 400000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\ncat > /tmp/r8.json <<'EOF'\n{\"v24_s18b\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.7,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":1800000,\"surf_min\":0.06},\n \"v25_sall\":  {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.7,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":2700000,\"surf_min\":0.05},\n \"v26_s135\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.7,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":1350000,\"surf_min\":0.10}}\nEOF\npython3 variants.py /tmp/r8.json 2>&1 | grep -v \"Token indices\" | tail -4\nfor v in v24_s18b v25_sall v26_s135; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done\nrm -f /tmp/r8.done\nnohup bash -c 'for v in v24_s18b v25_sall v26_s135; do (python3 /workspace/eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy /workspace/data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) & done; wait; echo done > /tmp/r8.done' >/dev/null 2>&1 &\necho LAUNCHED; cat /workspace/.timer/remaining_secs",
  "description": "Launch dose-sweep round 8",
  "timeout": 400000
}

{
  "command": "cd /workspace; python3 - <<'EOF'\np='submission/curate.py'; s=open(p).read()\ns=s.replace('NF = 24   # 14 shape + 4 vs-background + 4 contrastive + wiki/qa marker counts',\n            'NF = 25   # 14 shape + 4 vs-background + 4 contrastive + wiki/qa markers + surface-form rate')\ns=s.replace('''TERM_PUNCT = ('.', '!', '?', '\"', \"'\", ')', ':', ';', '”', '’')''',\n'''TERM_PUNCT = ('.', '!', '?', '\"', \"'\", ')', ':', ';', '”', '’')\nPUNC_RE = re.compile(r\"[,.;:)]\")\nSPP_RE = re.compile(r\" [,.;:)]\")      # space-BEFORE-punctuation: the WikiText surface form''')\ns=s.replace('''    f[23] = sum(1 for r in QA_RE if r.search(t40))\n    return f''','''    f[23] = sum(1 for r in QA_RE if r.search(t40))\n    npunc = len(PUNC_RE.findall(text))\n    f[24] = len(SPP_RE.findall(text)) / npunc if npunc > 30 else 0.0\n    return f''')\ns=s.replace('cf = os.path.join(CACHE, \"scores24.npz\")','cf = os.path.join(CACHE, \"scores25.npz\")')\ns=s.replace('''WIKI_BOOST = 0.3                   # additive bonus, wiki score, per wiki marker (capped)''',\n'''WIKI_BOOST = 0.7                   # additive bonus, wiki score, per wiki marker (capped)\nSURF_TOKENS = 900_000              # tokens of the wiki quota reserved for surface-form matches\nSURF_MIN = 0.12                    # min space-before-punctuation rate to count as a match''')\n# selection: reserve part of the encyclopedic quota for surface-form matches\ns=s.replace('''    quota = [m * BUDGET for m in mix]\n    orders = [np.argsort(-np.where(elig[r], S[:, r], -1e9)) for r in range(N_REG)]''',\n'''    quota = [m * BUDGET for m in mix]\n    orders = [np.argsort(-np.where(elig[r], S[:, r], -1e9)) for r in range(N_REG)]\n    # The encyclopedic target is detokenized WikiText: ~94% of its punctuation marks are\n    # space-separated (\" ,\") and no pool document shares that style by default. Reserve\n    # SURF_TOKENS of the encyclopedic quota for the pool documents that DO share it,\n    # highest rate first: a surface-form match the content score cannot express.\n    surf = np.where((F[:, 24] >= SURF_MIN) & elig[0])[0]\n    surf = surf[np.argsort(-F[surf, 24])]\n    orders[0] = np.concatenate([surf, orders[0]])''')\ns=s.replace('''    cand = set()\n    for r in range(N_REG):\n        cand.update(ids[orders[r][:int(4.0 * quota[r] / 900) + 4000]].tolist())''',\n'''    cand = set(ids[surf].tolist())\n    for r in range(N_REG):\n        cand.update(ids[orders[r][:int(4.0 * quota[r] / 900) + 4000]].tolist())''')\ns=s.replace('''    picked, bands, per, used, ndup = set(), {}, [[] for _ in range(N_REG)], [0.0] * N_REG, 0\n    for r in range(N_REG):\n        for i in orders[r]:\n            if used[r] >= quota[r]:\n                break\n            did = int(ids[i])\n            if did in picked:\n                continue''','''    picked, bands, per, used, ndup = set(), {}, [[] for _ in range(N_REG)], [0.0] * N_REG, 0\n    surf_used = 0.0\n    for r in range(N_REG):\n        for i in orders[r]:\n            if used[r] >= quota[r]:\n                break\n            is_surf = F[i, 24] >= SURF_MIN\n            if is_surf and surf_used >= SURF_TOKENS:\n                continue                      # surface-form dose already met\n            did = int(ids[i])\n            if did in picked:\n                continue''')\ns=s.replace('''            picked.add(did)\n            per[r].append(did)\n            used[r] += len(a) + 1''','''            picked.add(did)\n            per[r].append(did)\n            used[r] += len(a) + 1\n            if is_surf:\n                surf_used += len(a) + 1''')\ns=s.replace('''        print(f\"      near-duplicates skipped: {ndup}\", flush=True)''',\n'''        print(f\"      near-duplicates skipped: {ndup}; \"\n              f\"surface-form tokens: {surf_used/1e6:.2f}M\", flush=True)''')\ns=s.replace('''and (c) other pools, since the background model is estimated from the pool being\nfiltered rather than from an external corpus.''','''and (c) other pools, since the background model is estimated from the pool being\nfiltered rather than from an external corpus.''')\ns=s.replace('''ap.add_argument(\"--mix\", default=\",\".join(str(m) for m in MIX))''',\n'''ap.add_argument(\"--mix\", default=\",\".join(str(m) for m in MIX))\n    ap.add_argument(\"--surf\", type=float, default=SURF_TOKENS)\n    ap.add_argument(\"--surf_min\", type=float, default=SURF_MIN)''')\ns=s.replace('''    mix = [float(x) for x in a.mix.split(\",\")]''',\n'''    mix = [float(x) for x in a.mix.split(\",\")]\n    global SURF_TOKENS, SURF_MIN\n    SURF_TOKENS, SURF_MIN = a.surf, a.surf_min''')\n# document the surface-form rule in the stated criterion\ns=s.replace('''SELECTION = fill a per-register TOKEN QUOTA equal''','''SURFACE FORM: the encyclopedic target is detokenized WikiText (94.5% of its\npunctuation marks are space-separated, 2.8% of its tokens are \" @-@ \" artifacts) - a\nstyle the content score cannot express because it is orthogonal to word identity.\nSURF_TOKENS of the encyclopedic quota are therefore reserved for the pool documents\nwith the highest space-before-punctuation rate (>= SURF_MIN), highest first.\n\nSELECTION = fill a per-register TOKEN QUOTA equal''')\nopen(p,'w').write(s)\nimport ast; ast.parse(s); print('curate.py patched + syntax OK')\nEOF\nwhile [ ! -f /tmp/r8.done ]; do sleep 8; done\nfor v in v24_s18b v25_sall v26_s135; do grep -hE \"BLOCK_PPL|HELDOUT_PPL\" /tmp/log_$v.txt | sed 's/\\/tmp\\/tr_//'; done\necho \"--- v23_surfwb=334.29  v22_surf18=339.12  v21_surf09=348.94  no-surf=359.50\"; cat /workspace/.timer/remaining_secs",
  "description": "Fold surface rule into curate.py, collect dose sweep",
  "timeout": 400000
}

{
  "command": "cd /workspace; export TOKENIZERS_PARALLELISM=false\nrm -f /tmp/regen.done\nnohup bash -c 'python3 /workspace/submission/curate.py --out /workspace/submission/selection.json > /tmp/regen.log 2>&1; echo done > /tmp/regen.done' >/dev/null 2>&1 &\ncat > /tmp/r9.json <<'EOF'\n{\"v27_s045\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":0.7,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":450000,\"surf_min\":0.20},\n \"v28_wb11\": {\"col\":18,\"code_boost\":1.2,\"qa_boost\":0.5,\"wiki_relax\":true,\"wiki_boost\":1.1,\"band_dedup\":true,\"mix\":[0.225,0.25,0.25,0.275],\"surf_tokens\":900000,\"surf_min\":0.12}}\nEOF\npython3 variants.py /tmp/r9.json 2>&1 | grep -v \"Token indices\" | tail -3\nfor v in v27_s045 v28_wb11; do python3 fast_pack.py /tmp/sel_$v.json /tmp/tr_$v.npy 2>&1 | grep -v \"Token indices\" | tail -1; done\nrm -f /tmp/r9.done\nnohup bash -c 'for v in v27_s045 v28_wb11; do (python3 /workspace/eval_blocks.py --train_npy /tmp/tr_$v.npy --heldout_npy /workspace/data/multi_dev.npy --out_json /tmp/e_$v.json > /tmp/log_$v.txt 2>&1) & done; wait; echo done > /tmp/r9.done' >/dev/null 2>&1 &\nwhile [ ! -f /tmp/regen.done ]; do sleep 5; done; tail -8 /tmp/regen.log | grep -v \"Token indices\"\ncat /workspace/.timer/remaining_secs",
  "description": "Regenerate deliverable, launch final probes",
  "timeout": 500000
}

{
  "command": "cd /workspace; sed -n '415,450p' submission/curate.py",
  "description": "Inspect main() around global decl"
}

{
  "replace_all": false,
  "file_path": "/workspace/submission/curate.py",
  "old_string": "def main():\n    ap = argparse.ArgumentParser()",
  "new_string": "def main():\n    global SURF_TOKENS, SURF_MIN\n    ap = argparse.ArgumentParser()"
}

{
  "replace_all": false,
  "file_path": "/workspace/submission/curate.py",
  "old_string": "    mix = [float(x) for x in a.mix.split(\",\")]\n    global SURF_TOKENS, SURF_MIN\n    SURF_TOKENS, SURF_MIN = a.surf, a.surf_min",
  "new_string": "    mix = [float(x) for x in a.mix.split(\",\")]\n    SURF_TOKENS, SURF_MIN = a.surf, a.surf_min"
}