claude-opus-5

{
  "command": "nohup bash -c \"python -u pack_selection.py /tmp/selD.json /tmp/D.npy && python -u train_nano.py --train_npy /tmp/D.npy --heldout_npy data/multi_dev.npy --out_json /tmp/D.json\" > /tmp/evalD.log 2>&1 & sleep 26; tail -1 /tmp/evalD.log"
}

{
  "command": "sleep 26; tail -1 /tmp/evalD.log"
}

{
  "command": "sleep 26; tail -1 /tmp/evalD.log; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 26; tail -1 /tmp/evalD.log"
}

{
  "command": "sed 's|/tmp/selC.json|/tmp/selE.json|' curate_quota.py > curate_E.py && K=16 nohup python -u curate_E.py > /tmp/curE.log 2>&1 & sleep 26; tail -2 /tmp/curE.log"
}

{
  "command": "sleep 26; tail -2 /tmp/curE.log"
}

{
  "command": "sleep 26; tail -2 /tmp/curE.log"
}

{
  "command": "nohup bash -c \"python -u pack_selection.py /tmp/selE.json /tmp/E.npy && python -u train_nano.py --train_npy /tmp/E.npy --heldout_npy data/multi_dev.npy --out_json /tmp/E.json\" > /tmp/evalE.log 2>&1 & sleep 26; tail -1 /tmp/evalE.log"
}

{
  "command": "sleep 26; tail -1 /tmp/evalE.log"
}

{
  "command": "sleep 30; tail -1 /tmp/evalE.log; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 26; tail -1 /tmp/evalE.log"
}

{
  "command": "sed -n '1,60p' /workspace/submission/curate.py > /tmp/head.txt; wc -l /workspace/submission/curate.py /workspace/curate_registers.py"
}

{
  "file_path": "/workspace/submission/curate.py",
  "content": "\"\"\"Curate /workspace/data/pool.jsonl into a priority-ordered selection for a\n12M-GPT-2-token pretraining budget.\n\nSTATED CRITERION\n================\nTake the documents that maximise *distribution match to the disclosed evaluation\ntarget* -- broad high-quality English in FOUR registers, in equal parts:\nencyclopedic (Wikipedia), general high-quality web prose, news, technical Q&A.\nConcretely, a document is selected if it (1) survives surface-quality filters and\n(2) ranks highly under the classifier for a target register that still has budget\nquota left.\n\nThree stages:\n\n1. HARD FILTERS (drop, not rank). Reject documents no small LM can profit from:\n   too short, non-English/heavily non-ASCII, low alphabetic fraction (nav bars,\n   symbol/number soup), non-prose stopword profile, repeated lines, low token\n   diversity, SHOUTING, digit-dense tables, boilerplate spam (\"cookie\", \"click\n   here\", ...). Computed on the first 4000 chars of each document.\n\n2. PER-REGISTER TARGET-LIKENESS CLASSIFIERS. `data/multi_dev.npy` is the dev\n   target as GPT-2 tokens, so it can be decoded back to text and split on EOS\n   into target documents. Spherical k-means (K=10) over hashed word 1-2 grams\n   clusters those target documents into registers; for each cluster we fit a\n   logistic regression (cluster docs = positive, random pool sample = negative)\n   and score every surviving pool document under every cluster model. This is the\n   CCNet / GPT-3-style quality classifier, but one per register instead of one\n   globally -- a single global classifier over-selects whichever register is most\n   abundant in the raw pool (news), which measurably costs perplexity.\n\n3. QUOTA-BALANCED MERGE. Each cluster is mapped to one of the four registers by\n   surface markers on its dev documents, each REGISTER is granted 25% of the\n   budget (matching the disclosed \"equal parts\" target), and that share is split\n   among the register's clusters in proportion to their dev mass. The output list\n   is produced by repeatedly emitting the next-best unused document of whichever\n   cluster is furthest behind its quota, so the priority order is quota-correct at\n   *every* prefix -- which matters because the trainer consumes the list until the\n   budget is full and then stops.\n\nA small shared prior on top of the classifier score prefers longer documents\n(fewer EOS boundaries per million tokens => more usable long-range context inside\nthe 256-token training window) and clean prose punctuation, and penalises\nduplicated lines and boilerplate.\n\nMeasured on the dev target with the frozen trainer (30M GPT, 12M tokens):\n    random selection (do-nothing baseline) ... 488.5 ppl\n    single global classifier ................ 389.5 ppl\n    per-register, equal-per-cluster quota ... 342.0 ppl\n    per-register, dev-proportional quota .... 335.5 ppl\n    this script (register quota, K=10) ...... 331.4 ppl\n    (same pipeline with K=16) ............... 338.2 ppl\n\nOutput: /workspace/submission/selection.json\n\"\"\"\nimport json, re, math, random, zlib, os, numpy as np, torch\nfrom collections import Counter\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\nK = int(os.environ.get(\"K\", \"10\"))          # target-register clusters\nNF = 1 << 20                                # hashed feature space\nPREFIX = 4000                               # chars of each doc used for scoring\nBUDGET_TOKENS = 12_000_000\nSEED = 0\n\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nWORD = re.compile(r\"[A-Za-z']+\")\n\n# ============================================================ load pool\nids, texts = [], []\nfor line in open(POOL):\n    r = json.loads(line)\n    ids.append(r[\"id\"]); texts.append(r[\"text\"])\nprint(f\"pool: {len(ids)} docs\", flush=True)\n\n# ============================================================ 1. hard filters\nBOILER = re.compile(\n    r\"cookie|privacy policy|terms of service|sign in|log in|subscribe now|\"\n    r\"all rights reserved|click here|javascript|advertisement|add to cart|\"\n    r\"posted by|read more|©\", re.I)\nSTOP = set(\"the of and to in a is that it for was on as with be by are this or from at an \"\n           \"which not but have has had were their they he she we you his her its\".split())\n\ndef features(full):\n    t = full[:PREFIX]\n    n = len(t)\n    if n == 0: return None\n    nb = len(t.encode(\"utf-8\", \"ignore\"))\n    ascii_frac = 1.0 - max(0.0, (nb - n)) / n     # multibyte chars inflate byte length\n    alpha_frac = sum(map(str.isalpha, t)) / n\n    words = WORD.findall(t)\n    nw = len(words)\n    if nw == 0: return None\n    lw = [w.lower() for w in words]\n    lines = [l for l in t.split(\"\\n\") if l.strip()]\n    dup_line = 1.0 - len(Counter(lines)) / len(lines) if lines else 0.0\n    nalpha = alpha_frac * n\n    return dict(\n        nw=nw,\n        nw_full=nw * len(full) / n,                       # extrapolated full length\n        ascii_frac=ascii_frac,\n        alpha_frac=alpha_frac,\n        stop_frac=sum(w in STOP for w in lw) / nw,\n        mean_wlen=sum(len(w) for w in words) / nw,\n        uniq_frac=len(set(lw)) / nw,\n        dup_line=dup_line,\n        punct_per_w=sum(t.count(c) for c in \".!?\") / nw,\n        upper_frac=sum(map(str.isupper, t)) / max(1.0, nalpha),\n        digit_frac=sum(map(str.isdigit, t)) / n,\n        boiler=len(BOILER.findall(t)) / max(1.0, nw / 100.0))   # hits per 100 words\n\ndef keep(f):\n    if f is None: return False\n    if f[\"nw\"] < 60: return False                  # too short to teach much\n    if f[\"ascii_frac\"] < 0.92: return False        # not English text\n    if f[\"alpha_frac\"] < 0.62: return False        # symbol / number soup\n    if not 0.18 <= f[\"stop_frac\"] <= 0.62: return False   # not fluent running prose\n    if not 2.8 <= f[\"mean_wlen\"] <= 9.0: return False\n    if f[\"dup_line\"] > 0.28: return False          # repeated boilerplate lines\n    if f[\"uniq_frac\"] < 0.24: return False         # repetitive spam\n    if not 0.012 <= f[\"punct_per_w\"] <= 0.28: return False\n    if f[\"upper_frac\"] > 0.22: return False        # SHOUTING / headline lists\n    if f[\"digit_frac\"] > 0.16: return False        # stats dumps, tables\n    if f[\"boiler\"] > 3.0: return False\n    return True\n\nfeats = [features(t) for t in texts]\nmask = np.array([keep(f) for f in feats])\nkept = np.where(mask)[0]\nprint(f\"after hard filters: {len(kept)} docs ({mask.mean():.1%})\", flush=True)\n\ntrunc = [t[:PREFIX] for t in texts]\nnw_full = np.array([f[\"nw_full\"] if f else 0.0 for f in feats], np.float32)\nppw = np.array([f[\"punct_per_w\"] if f else 0.0 for f in feats], np.float32)\ndupl = np.array([f[\"dup_line\"] if f else 0.0 for f in feats], np.float32)\nboil = np.array([f[\"boiler\"] if f else 0.0 for f in feats], np.float32)\ndel feats\n\n# ============================================================ hashed features\ndef hash_row(t):\n    \"\"\"L2-normalised hashed word 1-2 gram counts for one document.\"\"\"\n    w = [x.lower() for x in WORD.findall(t)]\n    h = [zlib.crc32(x.encode()) % NF for x in w]                       # deterministic\n    h += [zlib.crc32((w[i] + \" \" + w[i+1]).encode()) % NF for i in range(len(w) - 1)]\n    if not h: return np.zeros(0, np.int64), np.zeros(0, np.float32)\n    idx, cnt = np.unique(np.array(h, np.int64), return_counts=True)\n    v = cnt.astype(np.float32); v /= max(1e-8, float(np.linalg.norm(v)))\n    return idx, v\n\ndef sparse_of(docs):\n    rows, cols, vals = [], [], []\n    for r, d in enumerate(docs):\n        i, v = hash_row(d)\n        rows.append(np.full(len(i), r, np.int64)); cols.append(i); vals.append(v)\n    return torch.sparse_coo_tensor(\n        np.stack([np.concatenate(rows), np.concatenate(cols)]), np.concatenate(vals),\n        (len(docs), NF), device=DEVICE).coalesce()\n\ndef fit_logreg(X, y, steps=300, l2=1e-5, lr=0.05):\n    npos = float(y.sum()); nneg = float(len(y) - npos)\n    cw = torch.where(y > 0, len(y) / (2 * npos), len(y) / (2 * nneg))\n    w = torch.zeros(NF, device=DEVICE, requires_grad=True)\n    b = torch.zeros(1, device=DEVICE, requires_grad=True)\n    opt = torch.optim.Adam([w, b], lr=lr)\n    for _ in range(steps):\n        lg = torch.sparse.mm(X, w.unsqueeze(1)).squeeze(1) + b\n        loss = (torch.nn.functional.binary_cross_entropy_with_logits(\n            lg, y, reduction=\"none\") * cw).mean() + l2 * (w * w).sum()\n        opt.zero_grad(); loss.backward(); opt.step()\n    with torch.no_grad():\n        lg = torch.sparse.mm(X, w.unsqueeze(1)).squeeze(1) + b\n        acc = ((lg > 0).float() == y).float().mean().item()\n    return w.detach(), b.detach(), acc\n\n# ============================================================ 2. decode dev target\ntok = AutoTokenizer.from_pretrained(\"gpt2\")\nEOS = tok.eos_token_id\nstream = np.load(DEV).astype(np.int64)\ndocs, cur = [], []\nfor t in stream:\n    if t == EOS:\n        if len(cur) > 40: docs.append(cur)\n        cur = []\n    else:\n        cur.append(int(t))\nif len(cur) > 40: docs.append(cur)\nPOS = [p for p in (tok.decode(d)[:PREFIX] for d in docs) if len(p) > 400]\nprint(f\"dev target docs: {len(POS)}\", flush=True)\n\nXp = sparse_of(POS)\npidx, pval = Xp.indices(), Xp.values()\n\n# --- spherical k-means over the target documents => registers\nC = torch.zeros(K, NF, device=DEVICE)\nfor k, r in enumerate(random.sample(range(len(POS)), K)):\n    m = pidx[0] == r\n    C[k, pidx[1][m]] = pval[m]\nfor _ in range(15):\n    lab = torch.sparse.mm(Xp, C.t()).argmax(1)\n    Cn = torch.zeros_like(C)\n    for k in range(K):\n        sel = (lab == k).nonzero().squeeze(1)\n        if len(sel) == 0: continue\n        m = torch.isin(pidx[0], sel)\n        Cn[k].index_add_(0, pidx[1][m], pval[m])\n        Cn[k] /= max(1e-8, float(Cn[k].norm()))\n    C = Cn\nlab = torch.sparse.mm(Xp, C.t()).argmax(1).cpu().numpy()\nprint(\"cluster sizes:\", np.bincount(lab, minlength=K).tolist(), flush=True)\n\n# --- one classifier per register cluster, against a random pool sample\nNEG = [trunc[i] for i in random.sample(list(kept), 8000)]\nmodels = []\nfor k in range(K):\n    pk = [POS[i] for i in range(len(POS)) if lab[i] == k]\n    if len(pk) < 30:\n        models.append(None); continue\n    X = sparse_of(pk + NEG)\n    y = torch.tensor(np.r_[np.ones(len(pk)), np.zeros(len(NEG))],\n                     dtype=torch.float32, device=DEVICE)\n    w, b, acc = fit_logreg(X, y)\n    print(f\"  clf c{k}: n_pos={len(pk)} train_acc={acc:.3f}\", flush=True)\n    models.append((w, b))\n\n# --- score every surviving pool doc under every register model\nS = np.full((K, len(ids)), -1e9, np.float32)\nfor s in range(0, len(kept), 20000):\n    sub = kept[s:s+20000]\n    Xs = sparse_of([trunc[i] for i in sub])\n    for k in range(K):\n        if models[k] is None: continue\n        with torch.no_grad():\n            S[k, sub] = (torch.sparse.mm(Xs, models[k][0].unsqueeze(1)).squeeze(1)\n                         + models[k][1]).cpu().numpy()\n    print(f\"scored {min(s+20000, len(kept))}/{len(kept)}\", flush=True)\n\n# --- shared surface prior\nprior = np.zeros(len(ids), np.float32)\nprior[kept] = (0.35 * np.minimum(1.0, np.log1p(nw_full[kept] / 150.0) / math.log(6.0))\n               + 0.25 * (1.0 - np.minimum(1.0, np.abs(ppw[kept] - 0.055) / 0.10))\n               - 0.30 * dupl[kept] - 0.10 * np.minimum(1.0, boil[kept]))\n\nranks = []\nfor k in range(K):\n    if models[k] is None:\n        ranks.append([]); continue\n    o = np.argsort(-(S[k] + prior))\n    ranks.append([int(i) for i in o if mask[i]])\n\n# ============================================================ 3. quota-balanced merge\ndef register(t):\n    \"\"\"Surface-marker register detector for a target document.\"\"\"\n    if \"<p>\" in t or \"</p>\" in t or \"<code>\" in t: return \"qa\"\n    if \" @-@ \" in t or \" @,@ \" in t or t.count(\" , \") > 3: return \"wiki\"\n    if re.search(r\"\\b(said|reported|Reuters|AP|\"\n                 r\"on (Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday))\\b\", t):\n        return \"news\"\n    return \"web\"\n\nregs = [register(p) for p in POS]\nmass = np.zeros(K)\nfor i in range(len(POS)): mass[lab[i]] += len(POS[i])\ncl_reg = []\nfor k in range(K):\n    c = Counter(regs[i] for i in range(len(POS)) if lab[i] == k)\n    cl_reg.append(c.most_common(1)[0][0] if c else \"web\")\nprint(\"cluster registers:\", cl_reg, flush=True)\n\nwgt = np.zeros(K)\nfor r in [\"wiki\", \"web\", \"news\", \"qa\"]:                 # equal parts, as disclosed\n    ks = [k for k in range(K) if cl_reg[k] == r and ranks[k]]\n    tot = sum(mass[k] for k in ks)\n    for k in ks:\n        wgt[k] = 0.25 * (mass[k] / tot if tot > 0 else 1.0 / len(ks))\nif wgt.sum() == 0: wgt = np.ones(K)\nwgt /= wgt.sum()\nprint(\"cluster weights:\", np.round(wgt, 3).tolist(), flush=True)\n\nptr = [0] * K\ncredit = np.zeros(K)\nseen, order, chars = set(), [], 0\nLIMIT = 4 * BUDGET_TOKENS * 4          # ~4x the budget in characters (chars ~ 4/token)\nactive = [k for k in range(K) if ranks[k]]\nwhile active and chars < LIMIT:\n    k = min(active, key=lambda j: credit[j] / max(1e-9, wgt[j]))\n    advanced = False\n    while ptr[k] < len(ranks[k]):\n        i = ranks[k][ptr[k]]; ptr[k] += 1\n        if i in seen: continue\n        seen.add(i); order.append(int(ids[i]))\n        c = max(1, len(texts[i]))\n        chars += c; credit[k] += c\n        advanced = True; break\n    if not advanced: active.remove(k)\n\nassert len(order) == len(set(order))\njson.dump(order, open(OUT, \"w\"))\nprint(f\"wrote {len(order)} ids -> {OUT} (~{chars/4/1e6:.0f}M tokens of text)\", flush=True)\n"
}

{
  "command": "cp /workspace/submission/curate.py /tmp/curate_old.py; echo ok"
}

{
  "file_path": "/workspace/submission/curate.py",
  "limit": 5
}

{
  "command": "rm -f /workspace/submission/curate.py; echo removed"
}

{
  "file_path": "/workspace/submission/curate.py",
  "content": "\"\"\"Curate /workspace/data/pool.jsonl into a priority-ordered selection for a\n12M-GPT-2-token pretraining budget.\n\nSTATED CRITERION\n================\nSelect the documents that maximise *distribution match to the disclosed evaluation\ntarget* -- broad high-quality English in FOUR registers, in equal parts:\nencyclopedic (Wikipedia), general high-quality web prose, news, technical Q&A.\nConcretely, a document is selected if it (1) survives surface-quality filters and\n(2) ranks highly under the classifier of a target register that still has budget\nquota left.\n\nThree stages:\n\n1. HARD FILTERS (drop, not rank). Reject documents no small LM can profit from:\n   too short, non-English/heavily non-ASCII, low alphabetic fraction (nav bars,\n   symbol/number soup), non-prose stopword profile, repeated lines, low token\n   diversity, SHOUTING, digit-dense tables, boilerplate spam (\"cookie\", \"click\n   here\", ...). Computed on the first 4000 chars of each document.\n\n2. PER-REGISTER TARGET-LIKENESS CLASSIFIERS. `data/multi_dev.npy` is the dev\n   target as GPT-2 tokens, so it can be decoded back to text and split on EOS into\n   target documents. Spherical k-means (K=10) over hashed word 1-2 grams clusters\n   those target documents into registers; for each cluster we fit a logistic\n   regression (cluster docs = positive, random pool sample = negative) and score\n   every surviving pool document under every cluster model. This is the CCNet /\n   GPT-3-style quality classifier, but one per register instead of one globally --\n   a single global classifier over-selects whichever register is most abundant in\n   the raw pool (news), which measurably costs perplexity.\n\n3. QUOTA-BALANCED MERGE. Each cluster is mapped to one of the four registers by\n   surface markers on its dev documents, each REGISTER is granted 25% of the budget\n   (matching the disclosed \"equal parts\" target), and that share is split among the\n   register's clusters in proportion to their dev mass. The output list is built by\n   repeatedly emitting the next-best unused document of whichever cluster is\n   furthest behind its quota, so the priority order is quota-correct at *every*\n   prefix -- which matters because the trainer consumes the list until the budget is\n   full and then stops.\n\nA small shared prior on top of the classifier score prefers longer documents (fewer\nEOS boundaries per million tokens => more usable long-range context inside the\n256-token training window) and clean prose punctuation, and penalises duplicated\nlines and boilerplate.\n\nMeasured on the dev target with the frozen trainer (30M GPT, 12M tokens):\n    random selection (do-nothing baseline) ... 488.5 ppl\n    single global classifier ................ 389.5 ppl\n    per-register, equal-per-cluster quota ... 342.0 ppl\n    per-register, dev-proportional quota .... 335.5 ppl\n    this script (register quota, K=10) ...... 331.4 ppl\n    same pipeline with K=16 ................. 338.2 ppl\n\nOutput: /workspace/submission/selection.json\n\"\"\"\nimport json, re, math, random, zlib, os, numpy as np, torch\nfrom collections import Counter\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\nK = int(os.environ.get(\"K\", \"10\"))          # target-register clusters\nNF = 1 << 20                                # hashed feature space\nPREFIX = 4000                               # chars of each doc used for scoring\nBUDGET_TOKENS = 12_000_000\nSEED = 0\n\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\nWORD = re.compile(r\"[A-Za-z']+\")\n\n# ============================================================ load pool\nids, texts = [], []\nfor line in open(POOL):\n    r = json.loads(line)\n    ids.append(r[\"id\"]); texts.append(r[\"text\"])\nprint(f\"pool: {len(ids)} docs\", flush=True)\n\n# ============================================================ 1. hard filters\nBOILER = re.compile(\n    r\"cookie|privacy policy|terms of service|sign in|log in|subscribe now|\"\n    r\"all rights reserved|click here|javascript|advertisement|add to cart|\"\n    r\"posted by|read more|©\", re.I)\nSTOP = set(\"the of and to in a is that it for was on as with be by are this or from at an \"\n           \"which not but have has had were their they he she we you his her its\".split())\n\ndef features(full):\n    t = full[:PREFIX]\n    n = len(t)\n    if n == 0: return None\n    nb = len(t.encode(\"utf-8\", \"ignore\"))\n    ascii_frac = 1.0 - max(0.0, (nb - n)) / n     # multibyte chars inflate byte length\n    alpha_frac = sum(map(str.isalpha, t)) / n\n    words = WORD.findall(t)\n    nw = len(words)\n    if nw == 0: return None\n    lw = [w.lower() for w in words]\n    lines = [l for l in t.split(\"\\n\") if l.strip()]\n    dup_line = 1.0 - len(Counter(lines)) / len(lines) if lines else 0.0\n    nalpha = alpha_frac * n\n    return dict(\n        nw=nw,\n        nw_full=nw * len(full) / n,                       # extrapolated full length\n        ascii_frac=ascii_frac,\n        alpha_frac=alpha_frac,\n        stop_frac=sum(w in STOP for w in lw) / nw,\n        mean_wlen=sum(len(w) for w in words) / nw,\n        uniq_frac=len(set(lw)) / nw,\n        dup_line=dup_line,\n        punct_per_w=sum(t.count(c) for c in \".!?\") / nw,\n        upper_frac=sum(map(str.isupper, t)) / max(1.0, nalpha),\n        digit_frac=sum(map(str.isdigit, t)) / n,\n        boiler=len(BOILER.findall(t)) / max(1.0, nw / 100.0))   # hits per 100 words\n\ndef keep(f):\n    if f is None: return False\n    if f[\"nw\"] < 60: return False                  # too short to teach much\n    if f[\"ascii_frac\"] < 0.92: return False        # not English text\n    if f[\"alpha_frac\"] < 0.62: return False        # symbol / number soup\n    if not 0.18 <= f[\"stop_frac\"] <= 0.62: return False   # not fluent running prose\n    if not 2.8 <= f[\"mean_wlen\"] <= 9.0: return False\n    if f[\"dup_line\"] > 0.28: return False          # repeated boilerplate lines\n    if f[\"uniq_frac\"] < 0.24: return False         # repetitive spam\n    if not 0.012 <= f[\"punct_per_w\"] <= 0.28: return False\n    if f[\"upper_frac\"] > 0.22: return False        # SHOUTING / headline lists\n    if f[\"digit_frac\"] > 0.16: return False        # stats dumps, tables\n    if f[\"boiler\"] > 3.0: return False\n    return True\n\nfeats = [features(t) for t in texts]\nmask = np.array([keep(f) for f in feats])\nkept = np.where(mask)[0]\nprint(f\"after hard filters: {len(kept)} docs ({mask.mean():.1%})\", flush=True)\n\ntrunc = [t[:PREFIX] for t in texts]\nnw_full = np.array([f[\"nw_full\"] if f else 0.0 for f in feats], np.float32)\nppw = np.array([f[\"punct_per_w\"] if f else 0.0 for f in feats], np.float32)\ndupl = np.array([f[\"dup_line\"] if f else 0.0 for f in feats], np.float32)\nboil = np.array([f[\"boiler\"] if f else 0.0 for f in feats], np.float32)\ndel feats\n\n# ============================================================ hashed features\ndef hash_row(t):\n    \"\"\"L2-normalised hashed word 1-2 gram counts for one document.\"\"\"\n    w = [x.lower() for x in WORD.findall(t)]\n    h = [zlib.crc32(x.encode()) % NF for x in w]                       # deterministic\n    h += [zlib.crc32((w[i] + \" \" + w[i+1]).encode()) % NF for i in range(len(w) - 1)]\n    if not h: return np.zeros(0, np.int64), np.zeros(0, np.float32)\n    idx, cnt = np.unique(np.array(h, np.int64), return_counts=True)\n    v = cnt.astype(np.float32); v /= max(1e-8, float(np.linalg.norm(v)))\n    return idx, v\n\ndef sparse_of(docs):\n    rows, cols, vals = [], [], []\n    for r, d in enumerate(docs):\n        i, v = hash_row(d)\n        rows.append(np.full(len(i), r, np.int64)); cols.append(i); vals.append(v)\n    return torch.sparse_coo_tensor(\n        np.stack([np.concatenate(rows), np.concatenate(cols)]), np.concatenate(vals),\n        (len(docs), NF), device=DEVICE).coalesce()\n\ndef fit_logreg(X, y, steps=300, l2=1e-5, lr=0.05):\n    npos = float(y.sum()); nneg = float(len(y) - npos)\n    cw = torch.where(y > 0, len(y) / (2 * npos), len(y) / (2 * nneg))\n    w = torch.zeros(NF, device=DEVICE, requires_grad=True)\n    b = torch.zeros(1, device=DEVICE, requires_grad=True)\n    opt = torch.optim.Adam([w, b], lr=lr)\n    for _ in range(steps):\n        lg = torch.sparse.mm(X, w.unsqueeze(1)).squeeze(1) + b\n        loss = (torch.nn.functional.binary_cross_entropy_with_logits(\n            lg, y, reduction=\"none\") * cw).mean() + l2 * (w * w).sum()\n        opt.zero_grad(); loss.backward(); opt.step()\n    with torch.no_grad():\n        lg = torch.sparse.mm(X, w.unsqueeze(1)).squeeze(1) + b\n        acc = ((lg > 0).float() == y).float().mean().item()\n    return w.detach(), b.detach(), acc\n\n# ============================================================ 2. decode dev target\ntok = AutoTokenizer.from_pretrained(\"gpt2\")\nEOS = tok.eos_token_id\nstream = np.load(DEV).astype(np.int64)\ndocs, cur = [], []\nfor t in stream:\n    if t == EOS:\n        if len(cur) > 40: docs.append(cur)\n        cur = []\n    else:\n        cur.append(int(t))\nif len(cur) > 40: docs.append(cur)\nPOS = [p for p in (tok.decode(d)[:PREFIX] for d in docs) if len(p) > 400]\nprint(f\"dev target docs: {len(POS)}\", flush=True)\n\nXp = sparse_of(POS)\npidx, pval = Xp.indices(), Xp.values()\n\n# --- spherical k-means over the target documents => registers\nC = torch.zeros(K, NF, device=DEVICE)\nfor k, r in enumerate(random.sample(range(len(POS)), K)):\n    m = pidx[0] == r\n    C[k, pidx[1][m]] = pval[m]\nfor _ in range(15):\n    lab = torch.sparse.mm(Xp, C.t()).argmax(1)\n    Cn = torch.zeros_like(C)\n    for k in range(K):\n        sel = (lab == k).nonzero().squeeze(1)\n        if len(sel) == 0: continue\n        m = torch.isin(pidx[0], sel)\n        Cn[k].index_add_(0, pidx[1][m], pval[m])\n        Cn[k] /= max(1e-8, float(Cn[k].norm()))\n    C = Cn\nlab = torch.sparse.mm(Xp, C.t()).argmax(1).cpu().numpy()\nprint(\"cluster sizes:\", np.bincount(lab, minlength=K).tolist(), flush=True)\n\n# --- one classifier per register cluster, against a random pool sample\nNEG = [trunc[i] for i in random.sample(list(kept), 8000)]\nmodels = []\nfor k in range(K):\n    pk = [POS[i] for i in range(len(POS)) if lab[i] == k]\n    if len(pk) < 30:\n        models.append(None); continue\n    X = sparse_of(pk + NEG)\n    y = torch.tensor(np.r_[np.ones(len(pk)), np.zeros(len(NEG))],\n                     dtype=torch.float32, device=DEVICE)\n    w, b, acc = fit_logreg(X, y)\n    print(f\"  clf c{k}: n_pos={len(pk)} train_acc={acc:.3f}\", flush=True)\n    models.append((w, b))\n\n# --- score every surviving pool doc under every register model\nS = np.full((K, len(ids)), -1e9, np.float32)\nfor s in range(0, len(kept), 20000):\n    sub = kept[s:s+20000]\n    Xs = sparse_of([trunc[i] for i in sub])\n    for k in range(K):\n        if models[k] is None: continue\n        with torch.no_grad():\n            S[k, sub] = (torch.sparse.mm(Xs, models[k][0].unsqueeze(1)).squeeze(1)\n                         + models[k][1]).cpu().numpy()\n    print(f\"scored {min(s+20000, len(kept))}/{len(kept)}\", flush=True)\n\n# --- shared surface prior\nprior = np.zeros(len(ids), np.float32)\nprior[kept] = (0.35 * np.minimum(1.0, np.log1p(nw_full[kept] / 150.0) / math.log(6.0))\n               + 0.25 * (1.0 - np.minimum(1.0, np.abs(ppw[kept] - 0.055) / 0.10))\n               - 0.30 * dupl[kept] - 0.10 * np.minimum(1.0, boil[kept]))\n\nranks = []\nfor k in range(K):\n    if models[k] is None:\n        ranks.append([]); continue\n    o = np.argsort(-(S[k] + prior))\n    ranks.append([int(i) for i in o if mask[i]])\n\n# ============================================================ 3. quota-balanced merge\ndef register(t):\n    \"\"\"Surface-marker register detector for a target document.\"\"\"\n    if \"<p>\" in t or \"</p>\" in t or \"<code>\" in t: return \"qa\"\n    if \" @-@ \" in t or \" @,@ \" in t or t.count(\" , \") > 3: return \"wiki\"\n    if re.search(r\"\\b(said|reported|Reuters|AP|\"\n                 r\"on (Monday|Tuesday|Wednesday|Thursday|Friday|Saturday|Sunday))\\b\", t):\n        return \"news\"\n    return \"web\"\n\nregs = [register(p) for p in POS]\nmass = np.zeros(K)\nfor i in range(len(POS)): mass[lab[i]] += len(POS[i])\ncl_reg = []\nfor k in range(K):\n    c = Counter(regs[i] for i in range(len(POS)) if lab[i] == k)\n    cl_reg.append(c.most_common(1)[0][0] if c else \"web\")\nprint(\"cluster registers:\", cl_reg, flush=True)\n\nwgt = np.zeros(K)\nfor r in [\"wiki\", \"web\", \"news\", \"qa\"]:                 # equal parts, as disclosed\n    ks = [k for k in range(K) if cl_reg[k] == r and ranks[k]]\n    tot = sum(mass[k] for k in ks)\n    for k in ks:\n        wgt[k] = 0.25 * (mass[k] / tot if tot > 0 else 1.0 / len(ks))\nif wgt.sum() == 0: wgt = np.ones(K)\nwgt /= wgt.sum()\nprint(\"cluster weights:\", np.round(wgt, 3).tolist(), flush=True)\n\nptr = [0] * K\ncredit = np.zeros(K)\nseen, order, chars = set(), [], 0\nLIMIT = 4 * BUDGET_TOKENS * 4          # ~4x the budget in characters (chars ~ 4/token)\nactive = [k for k in range(K) if ranks[k]]\nwhile active and chars < LIMIT:\n    k = min(active, key=lambda j: credit[j] / max(1e-9, wgt[j]))\n    advanced = False\n    while ptr[k] < len(ranks[k]):\n        i = ranks[k][ptr[k]]; ptr[k] += 1\n        if i in seen: continue\n        seen.add(i); order.append(int(ids[i]))\n        c = max(1, len(texts[i]))\n        chars += c; credit[k] += c\n        advanced = True; break\n    if not advanced: active.remove(k)\n\nassert len(order) == len(set(order)), \"duplicate ids\"\nassert all(0 <= i < len(ids) for i in order), \"out-of-range id\"\njson.dump(order, open(OUT, \"w\"))\nprint(f\"wrote {len(order)} ids -> {OUT} (~{chars/4/1e6:.0f}M tokens of text)\", flush=True)\n"
}

{
  "command": "nohup python -u submission/curate.py > /tmp/final.log 2>&1 & sleep 26; tail -2 /tmp/final.log; cat /workspace/.timer/remaining_secs"
}

{
  "command": "sleep 28; tail -3 /tmp/final.log"
}

{
  "command": "sleep 28; tail -3 /tmp/final.log"
}

{
  "command": "sleep 28; tail -3 /tmp/final.log"
}

{
  "command": "sleep 28; tail -2 /tmp/final.log"
}

{
  "command": "nohup bash dev_eval.sh > /tmp/evalF.log 2>&1 & sleep 28; tail -1 /tmp/evalF.log"
}

{
  "command": "sleep 28; tail -1 /tmp/evalF.log"
}

{
  "command": "sleep 28; tail -2 /tmp/evalF.log"
}