claude-opus-5

186	        for k in ("alpha", "digit", "punct", "nonascii"): o[k] /= cl
187	        return o
188	    ST = [stats(s) for s in segs]
189	    TS = {k: np.array([s[k] for s in ST]) for k in ST[0]}
190	    LO = {k: float(np.percentile(TS[k], a.gate_p)) for k in TS}
191	    HI = {k: float(np.percentile(TS[k], 100.0 - a.gate_p)) for k in TS}
192	    gate = (lens >= a.min_tokens) & (rep < a.rep_max)
193	    for k in ("alpha", "stop", "isword"):                       # prose-ness: floor
194	        gate &= Hc[k] >= LO[k]
195	    for k in ("pipe", "digit", "punct", "nonascii", "nl"):      # junk markers: ceiling
196	        gate &= Hc[k] <= HI[k]
197	    gate &= (Hc["clen"] >= LO["clen"]) & (Hc["clen"] <= HI["clen"])   # BPE fertility, two-sided
198	    log(f"stage1 gate: {gate.sum()} docs ({gate.mean():.1%}), {lens[gate].sum()/1e6:.0f}M tokens")
199	
200	    # ---- stage 3a: group the target into registers ----
201	    S = len(segs)
202	    if a.groups == "registers":
203	        # The four disclosed registers, identified by unambiguous surface markers: wikitext
204	        # detokenisation artefacts, HTML-wrapped Q&A bodies, reporting verbs in short news
205	        # paragraphs, and general HQ web prose as the remainder.
206	        txt = tk.batch_decode(segs)
207	
208	        def register_of(t):
209	            if " @-@ " in t or " @,@ " in t or " @.@ " in t: return 0     # encyclopedic (wikitext)
210	            if "<p>" in t or "<code>" in t or "<pre>" in t: return 2      # technical Q&A
211	            if re.search(r"\b(said|told|reported|announced|according to)\b", t) and t.count("\n") <= 3:
212	                return 1                                                  # news
213	            return 3                                                      # general HQ web prose
214	        lab = np.array([register_of(t) for t in txt])
215	        K = 4
216	        mass = np.bincount(lab, weights=seglen, minlength=K)
217	        groups = sorted(set(lab.tolist()))
218	        log(f"{len(groups)} registers, token shares {np.round(mass[groups]/mass.sum(), 3)}")
219	        return_after_group = True
220	    else:
221	        K = a.k
222	        return_after_group = False
223	    if not return_after_group:
224	        lab, mass, groups = kmeans_groups(segs, seglen, S, a.k, gpu)
225	    _ = None
226	    Xt = torch.zeros(0, device=gpu) if True else None
227	    del Xt
228	    Xt = torch.zeros(S, V, device=gpu) if False else None
229	    for i, s in enumerate(segs):
230	        Xt[i].index_add_(0, torch.from_numpy(s).to(gpu), torch.ones(len(s), device=gpu))
231	    idf = torch.log(1.0 / ((Xt > 0).float().mean(0) + 1e-3))
232	    Xt = torch.log1p(Xt) * idf
233	    Xt /= Xt.norm(dim=1, keepdim=True).clamp(min=1e-6)
234	    gen = torch.Generator(device=gpu.type).manual_seed(0)
235	    C = Xt[torch.randperm(S, generator=gen, device=gpu)[:a.k]].clone()
236	    for _ in range(25):
237	        lab_t = (Xt @ C.T).argmax(1)
238	        for k in range(a.k):
239	            m = lab_t == k
240	            if m.any(): C[k] = Xt[m].mean(0)
241	        C /= C.norm(dim=1, keepdim=True).clamp(min=1e-6)
242	    lab = lab_t.cpu().numpy()
243	    del Xt; torch.cuda.empty_cache()
244	
245	    mass = np.bincount(lab, weights=seglen, minlength=a.k)
246	    small = [k for k in range(a.k) if mass[k] < 0.25 * mass.sum() / a.k]
247	    if small:                                  # fold clusters too small for a reliable n-gram model

[metadata] {"type": "text", "file": {"filePath": "/workspace/submission/curate.py", "content": "        for k in (\"alpha\", \"digit\", \"punct\", \"nonascii\"): o[k] /= cl\n        return o\n    ST = [stats(s) for s in segs]\n    TS = {k: np.array([s[k] for s in ST]) for k in ST[0]}\n    LO = {k: float(np.percentile(TS[k], a.gate_p)) for k in TS}\n    HI = {k: float(np.percentile(TS[k], 100.0 - a.gate_p)) for k in TS}\n    gate = (lens >= a.min_tokens) & (rep < a.rep_max)\n    for k in (\"alpha\", \"stop\", \"isword\"):                       # prose-ness: floor\n        gate &= Hc[k] >= LO[k]\n    for k in (\"pipe\", \"digit\", \"punct\", \"nonascii\", \"nl\"):      # junk markers: ceiling\n        gate &= Hc[k] <= HI[k]\n    gate &= (Hc[\"clen\"] >= LO[\"clen\"]) & (Hc[\"clen\"] <= HI[\"clen\"])   # BPE fertility, two-sided\n    log(f\"stage1 gate: {gate.sum()} docs ({gate.mean():.1%}), {lens[gate].sum()/1e6:.0f}M tokens\")\n\n    # ---- stage 3a: group the target into registers ----\n    S = len(segs)\n    if a.groups == \"registers\":\n        # The four disclosed registers, identified by unambiguous surface markers: wikitext\n        # detokenisation artefacts, HTML-wrapped Q&A bodies, reporting verbs in short news\n        # paragraphs, and general HQ web prose as the remainder.\n        txt = tk.batch_decode(segs)\n\n        def register_of(t):\n            if \" @-@ \" in t or \" @,@ \" in t or \" @.@ \" in t: return 0     # encyclopedic (wikitext)\n            if \"<p>\" in t or \"<code>\" in t or \"<pre>\" in t: return 2      # technical Q&A\n            if re.search(r\"\\b(said|told|reported|announced|according to)\\b\", t) and t.count(\"\\n\") <= 3:\n                return 1                                                  # news\n            return 3                                                      # general HQ web prose\n        lab = np.array([register_of(t) for t in txt])\n        K = 4\n        mass = np.bincount(lab, weights=seglen, minlength=K)\n        groups = sorted(set(lab.tolist()))\n        log(f\"{len(groups)} registers, token shares {np.round(mass[groups]/mass.sum(), 3)}\")\n        return_after_group = True\n    else:\n        K = a.k\n        return_after_group = False\n    if not return_after_group:\n        lab, mass, groups = kmeans_groups(segs, seglen, S, a.k, gpu)\n    _ = None\n    Xt = torch.zeros(0, device=gpu) if True else None\n    del Xt\n    Xt = torch.zeros(S, V, device=gpu) if False else None\n    for i, s in enumerate(segs):\n        Xt[i].index_add_(0, torch.from_numpy(s).to(gpu), torch.ones(len(s), device=gpu))\n    idf = torch.log(1.0 / ((Xt > 0).float().mean(0) + 1e-3))\n    Xt = torch.log1p(Xt) * idf\n    Xt /= Xt.norm(dim=1, keepdim=True).clamp(min=1e-6)\n    gen = torch.Generator(device=gpu.type).manual_seed(0)\n    C = Xt[torch.randperm(S, generator=gen, device=gpu)[:a.k]].clone()\n    for _ in range(25):\n        lab_t = (Xt @ C.T).argmax(1)\n        for k in range(a.k):\n            m = lab_t == k\n            if m.any(): C[k] = Xt[m].mean(0)\n        C /= C.norm(dim=1, keepdim=True).clamp(min=1e-6)\n    lab = lab_t.cpu().numpy()\n    del Xt; torch.cuda.empty_cache()\n\n    mass = np.bincount(lab, weights=seglen, minlength=a.k)\n    small = [k for k in range(a.k) if mass[k] < 0.25 * mass.sum() / a.k]\n    if small:                                  # fold clusters too small for a reliable n-gram model", "numLines": 62, "startLine": 186, "totalLines": 316}}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    # ---- stage 3a: group the target into registers ----\n    S = len(segs)\n    if a.groups == \"registers\":\n        # The four disclosed registers, identified by unambiguous surface markers: wikitext\n        # detokenisation artefacts, HTML-wrapped Q&A bodies, reporting verbs in short news\n        # paragraphs, and general HQ web prose as the remainder.\n        txt = tk.batch_decode(segs)\n\n        def register_of(t):\n            if \" @-@ \" in t or \" @,@ \" in t or \" @.@ \" in t: return 0     # encyclopedic (wikitext)\n            if \"<p>\" in t or \"<code>\" in t or \"<pre>\" in t: return 2      # technical Q&A\n            if re.search(r\"\\b(said|told|reported|announced|according to)\\b\", t) and t.count(\"\\n\") <= 3:\n                return 1                                                  # news\n            return 3                                                      # general HQ web prose\n        lab = np.array([register_of(t) for t in txt])\n        K = 4\n        mass = np.bincount(lab, weights=seglen, minlength=K)\n        groups = sorted(set(lab.tolist()))\n        log(f\"{len(groups)} registers, token shares {np.round(mass[groups]/mass.sum(), 3)}\")\n        return_after_group = True\n    else:\n        K = a.k\n        return_after_group = False\n    if not return_after_group:\n        lab, mass, groups = kmeans_groups(segs, seglen, S, a.k, gpu)\n    _ = None\n    Xt = torch.zeros(0, device=gpu) if True else None\n    del Xt\n    Xt = torch.zeros(S, V, device=gpu) if False else None\n    for i, s in enumerate(segs):\n        Xt[i].index_add_(0, torch.from_numpy(s).to(gpu), torch.ones(len(s), device=gpu))\n    idf = torch.log(1.0 / ((Xt > 0).float().mean(0) + 1e-3))\n    Xt = torch.log1p(Xt) * idf\n    Xt /= Xt.norm(dim=1, keepdim=True).clamp(min=1e-6)\n    gen = torch.Generator(device=gpu.type).manual_seed(0)\n    C = Xt[torch.randperm(S, generator=gen, device=gpu)[:a.k]].clone()\n    for _ in range(25):\n        lab_t = (Xt @ C.T).argmax(1)\n        for k in range(a.k):\n            m = lab_t == k\n            if m.any(): C[k] = Xt[m].mean(0)\n        C /= C.norm(dim=1, keepdim=True).clamp(min=1e-6)\n    lab = lab_t.cpu().numpy()\n    del Xt; torch.cuda.empty_cache()\n\n    mass = np.bincount(lab, weights=seglen, minlength=a.k)\n    small = [k for k in range(a.k) if mass[k] < 0.25 * mass.sum() / a.k]\n    if small:                                  # fold clusters too small for a reliable n-gram model\n        big = int(np.argmax(mass))\n        lab = np.array([big if x in small else x for x in lab])\n        mass = np.bincount(lab, weights=seglen, minlength=a.k)\n    groups = sorted(set(lab.tolist()))\n    log(f\"{len(groups)} target groups, token shares {np.round(mass[groups]/mass.sum(), 3)}\")", "newString": "    # ---- stage 3a: group the target into registers ----\n    if a.groups == \"registers\":\n        lab, mass = register_groups(segs, seglen, tk)\n    else:\n        lab, mass = kmeans_groups(segs, seglen, a.k, gpu)\n    groups = sorted(set(lab.tolist()))\n    log(f\"{len(groups)} target groups, token shares {np.round(mass[groups]/mass.sum(), 3)}\")", "originalFile": null, "structuredPatch": [{"oldStart": 198, "oldLines": 56, "newStart": 198, "newLines": 10, "lines": ["     log(f\"stage1 gate: {gate.sum()} docs ({gate.mean():.1%}), {lens[gate].sum()/1e6:.0f}M tokens\")", " ", "     # ---- stage 3a: group the target into registers ----", "-    S = len(segs)", "     if a.groups == \"registers\":", "-        # The four disclosed registers, identified by unambiguous surface markers: wikitext", "-        # detokenisation artefacts, HTML-wrapped Q&A bodies, reporting verbs in short news", "-        # paragraphs, and general HQ web prose as the remainder.", "-        txt = tk.batch_decode(segs)", "-", "-        def register_of(t):", "-            if \" @-@ \" in t or \" @,@ \" in t or \" @.@ \" in t: return 0     # encyclopedic (wikitext)", "-            if \"<p>\" in t or \"<code>\" in t or \"<pre>\" in t: return 2      # technical Q&A", "-            if re.search(r\"\\b(said|told|reported|announced|according to)\\b\", t) and t.count(\"\\n\") <= 3:", "-                return 1                                                  # news", "-            return 3                                                      # general HQ web prose", "-        lab = np.array([register_of(t) for t in txt])", "-        K = 4", "-        mass = np.bincount(lab, weights=seglen, minlength=K)", "-        groups = sorted(set(lab.tolist()))", "-        log(f\"{len(groups)} registers, token shares {np.round(mass[groups]/mass.sum(), 3)}\")", "-        return_after_group = True", "+        lab, mass = register_groups(segs, seglen, tk)", "     else:", "-        K = a.k", "-        return_after_group = False", "-    if not return_after_group:", "-        lab, mass, groups = kmeans_groups(segs, seglen, S, a.k, gpu)", "-    _ = None", "-    Xt = torch.zeros(0, device=gpu) if True else None", "-    del Xt", "-    Xt = torch.zeros(S, V, device=gpu) if False else None", "-    for i, s in enumerate(segs):", "-        Xt[i].index_add_(0, torch.from_numpy(s).to(gpu), torch.ones(len(s), device=gpu))", "-    idf = torch.log(1.0 / ((Xt > 0).float().mean(0) + 1e-3))", "-    Xt = torch.log1p(Xt) * idf", "-    Xt /= Xt.norm(dim=1, keepdim=True).clamp(min=1e-6)", "-    gen = torch.Generator(device=gpu.type).manual_seed(0)", "-    C = Xt[torch.randperm(S, generator=gen, device=gpu)[:a.k]].clone()", "-    for _ in range(25):", "-        lab_t = (Xt @ C.T).argmax(1)", "-        for k in range(a.k):", "-            m = lab_t == k", "-            if m.any(): C[k] = Xt[m].mean(0)", "-        C /= C.norm(dim=1, keepdim=True).clamp(min=1e-6)", "-    lab = lab_t.cpu().numpy()", "-    del Xt; torch.cuda.empty_cache()", "-", "-    mass = np.bincount(lab, weights=seglen, minlength=a.k)", "-    small = [k for k in range(a.k) if mass[k] < 0.25 * mass.sum() / a.k]", "-    if small:                                  # fold clusters too small for a reliable n-gram model", "-        big = int(np.argmax(mass))", "-        lab = np.array([big if x in small else x for x in lab])", "-        mass = np.bincount(lab, weights=seglen, minlength=a.k)", "+        lab, mass = kmeans_groups(segs, seglen, a.k, gpu)", "     groups = sorted(set(lab.tolist()))", "     log(f\"{len(groups)} target groups, token shares {np.round(mass[groups]/mass.sum(), 3)}\")", " "]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "# --------------------------------------------------------------------------- n-gram machinery\ndef counts_of(arrs):", "newString": "# --------------------------------------------------------------------------- target grouping\ndef register_groups(segs, seglen, tk):\n    \"\"\"The four disclosed registers, split by unambiguous surface markers: wikitext\n    detokenisation artefacts, HTML-wrapped Q&A bodies, reporting verbs in short news\n    paragraphs, and general high-quality web prose as the remainder.\"\"\"\n    def register_of(t):\n        if \" @-@ \" in t or \" @,@ \" in t or \" @.@ \" in t: return 0        # encyclopedic (wikitext)\n        if \"<p>\" in t or \"<code>\" in t or \"<pre>\" in t: return 2         # technical Q&A\n        if re.search(r\"\\b(said|told|reported|announced|according to)\\b\", t) and t.count(\"\\n\") <= 3:\n            return 1                                                     # news\n        return 3                                                         # general HQ web prose\n    lab = np.array([register_of(t) for t in tk.batch_decode(segs)])\n    return lab, np.bincount(lab, weights=seglen, minlength=4)\n\n\ndef kmeans_groups(segs, seglen, K, gpu):\n    \"\"\"Spherical k-means over tf-idf unigram vectors of the target segments.\"\"\"\n    S = len(segs)\n    Xt = torch.zeros(S, V, device=gpu)\n    for i, s in enumerate(segs):\n        Xt[i].index_add_(0, torch.from_numpy(s).to(gpu), torch.ones(len(s), device=gpu))\n    idf = torch.log(1.0 / ((Xt > 0).float().mean(0) + 1e-3))\n    Xt = torch.log1p(Xt) * idf\n    Xt /= Xt.norm(dim=1, keepdim=True).clamp(min=1e-6)\n    gen = torch.Generator(device=gpu.type).manual_seed(0)\n    C = Xt[torch.randperm(S, generator=gen, device=gpu)[:K]].clone()\n    for _ in range(25):\n        lab_t = (Xt @ C.T).argmax(1)\n        for k in range(K):\n            m = lab_t == k\n            if m.any(): C[k] = Xt[m].mean(0)\n        C /= C.norm(dim=1, keepdim=True).clamp(min=1e-6)\n    lab = lab_t.cpu().numpy()\n    del Xt\n    torch.cuda.empty_cache()\n    mass = np.bincount(lab, weights=seglen, minlength=K)\n    small = [k for k in range(K) if mass[k] < 0.25 * mass.sum() / K]\n    if small:                       # fold clusters too small for a reliable n-gram estimate\n        big = int(np.argmax(mass))\n        lab = np.array([big if x in small else x for x in lab])\n        mass = np.bincount(lab, weights=seglen, minlength=K)\n    return lab, mass\n\n\n# --------------------------------------------------------------------------- n-gram machinery\ndef counts_of(arrs):", "originalFile": null, "structuredPatch": [{"oldStart": 144, "oldLines": 6, "newStart": 144, "newLines": 50, "lines": ["     return Hn, rep, sig", " ", " ", "+# --------------------------------------------------------------------------- target grouping", "+def register_groups(segs, seglen, tk):", "+    \"\"\"The four disclosed registers, split by unambiguous surface markers: wikitext", "+    detokenisation artefacts, HTML-wrapped Q&A bodies, reporting verbs in short news", "+    paragraphs, and general high-quality web prose as the remainder.\"\"\"", "+    def register_of(t):", "+        if \" @-@ \" in t or \" @,@ \" in t or \" @.@ \" in t: return 0        # encyclopedic (wikitext)", "+        if \"<p>\" in t or \"<code>\" in t or \"<pre>\" in t: return 2         # technical Q&A", "+        if re.search(r\"\\b(said|told|reported|announced|according to)\\b\", t) and t.count(\"\\n\") <= 3:", "+            return 1                                                     # news", "+        return 3                                                         # general HQ web prose", "+    lab = np.array([register_of(t) for t in tk.batch_decode(segs)])", "+    return lab, np.bincount(lab, weights=seglen, minlength=4)", "+", "+", "+def kmeans_groups(segs, seglen, K, gpu):", "+    \"\"\"Spherical k-means over tf-idf unigram vectors of the target segments.\"\"\"", "+    S = len(segs)", "+    Xt = torch.zeros(S, V, device=gpu)", "+    for i, s in enumerate(segs):", "+        Xt[i].index_add_(0, torch.from_numpy(s).to(gpu), torch.ones(len(s), device=gpu))", "+    idf = torch.log(1.0 / ((Xt > 0).float().mean(0) + 1e-3))", "+    Xt = torch.log1p(Xt) * idf", "+    Xt /= Xt.norm(dim=1, keepdim=True).clamp(min=1e-6)", "+    gen = torch.Generator(device=gpu.type).manual_seed(0)", "+    C = Xt[torch.randperm(S, generator=gen, device=gpu)[:K]].clone()", "+    for _ in range(25):", "+        lab_t = (Xt @ C.T).argmax(1)", "+        for k in range(K):", "+            m = lab_t == k", "+            if m.any(): C[k] = Xt[m].mean(0)", "+        C /= C.norm(dim=1, keepdim=True).clamp(min=1e-6)", "+    lab = lab_t.cpu().numpy()", "+    del Xt", "+    torch.cuda.empty_cache()", "+    mass = np.bincount(lab, weights=seglen, minlength=K)", "+    small = [k for k in range(K) if mass[k] < 0.25 * mass.sum() / K]", "+    if small:                       # fold clusters too small for a reliable n-gram estimate", "+        big = int(np.argmax(mass))", "+        lab = np.array([big if x in small else x for x in lab])", "+        mass = np.bincount(lab, weights=seglen, minlength=K)", "+    return lab, mass", "+", "+", " # --------------------------------------------------------------------------- n-gram machinery", " def counts_of(arrs):", "     cu = np.zeros(V); cb = np.zeros(HB)"]}], "userModified": false, "replaceAll": false}

ok
syntax ok
--- loose=314.96
3267

[stdout]
ok
syntax ok
--- loose=314.96
3267
[metadata] {"noOutputExpected": false}

== knobs (loose=314.96) ==
== per-register ==
3026

[stdout]
== knobs (loose=314.96) ==
== per-register ==
3026
[metadata] {"noOutputExpected": false}

== knobs (loose=314.96 mass=325.97) ==
== per-register ==
topk wiki pending
topk news pending
topk qa pending
topk web pending
mass wiki pending
mass news pending
mass qa pending
mass web pending

[stdout]
== knobs (loose=314.96 mass=325.97) ==
== per-register ==
topk wiki pending
topk news pending
topk qa pending
topk web pending
mass wiki pending
mass news pending
mass qa pending
mass web pending
[metadata] {"noOutputExpected": false}

15
topk   wiki  -
topk   news  -
topk   qa    -
topk   web   -
mass   wiki  -
mass   news  -
mass   qa    -
mass   web   -
2776

[stdout]
15
topk   wiki  -
topk   news  -
topk   qa    -
topk   web   -
mass   wiki  -
mass   news  -
mass   qa    -
mass   web   -
2776
[metadata] {"noOutputExpected": false}

560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
2628 --output-format=stream-json --disallowedTools
2628 rm /tmp/ri-exec-0f161f04679741d9860567ac88fcb99c.pid
2628 set pipefail;

[stdout]
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
560 --train_npy --heldout_npy
2628 --output-format=stream-json --disallowedTools
2628 rm /tmp/ri-exec-0f161f04679741d9860567ac88fcb99c.pid
2628 set pipefail;
[metadata] {"noOutputExpected": false}

HELDOUT_PPL 312.422  loss 5.7444  train_s 645  /tmp/tr_lqa.npy
HELDOUT_PPL 314.058  loss 5.7496  train_s 645  /tmp/tr_lweb.npy
HELDOUT_PPL 311.550  loss 5.7416  train_s 644  /tmp/tr_loose3.npy
HELDOUT_PPL 306.709  loss 5.7259  train_s 645  /tmp/tr_loose2.npy

topk   wiki  PPL 819.164
topk   news  PPL 251.713
topk   qa    PPL 524.848
topk   web   PPL 344.019
mass   wiki  PPL 710.872
mass   news  PPL 265.387
mass   qa    PPL 213.962
mass   web   PPL 334.256
2646

[stdout]
HELDOUT_PPL 312.422  loss 5.7444  train_s 645  /tmp/tr_lqa.npy
HELDOUT_PPL 314.058  loss 5.7496  train_s 645  /tmp/tr_lweb.npy
HELDOUT_PPL 311.550  loss 5.7416  train_s 644  /tmp/tr_loose3.npy
HELDOUT_PPL 306.709  loss 5.7259  train_s 645  /tmp/tr_loose2.npy

topk   wiki  PPL 819.164
topk   news  PPL 251.713
topk   qa    PPL 524.848
topk   web   PPL 344.019
mass   wiki  PPL 710.872
mass   news  PPL 265.387
mass   qa    PPL 213.962
mass   web   PPL 334.256
2646
[metadata] {"noOutputExpected": false}

[g0] [  3.1s] gate keeps 135601 (74.5%), 168M tokens
[g0] [  5.6s] mass: 38197 ids / 40.0M tokens; first 12M = 11863 docs, mean len 1011
[l2qa] [  2.9s] gate keeps 123766 (68.0%), 151M tokens
[l2qa] [  5.2s] mass: 38333 ids / 40.0M tokens; first 12M = 12280 docs, mean len 976
[l2wq] [  2.8s] gate keeps 123766 (68.0%), 151M tokens
[l2wq] [  5.3s] mass: 38754 ids / 40.0M tokens; first 12M = 12071 docs, mean len 993
[l2k12] [  2.8s] gate keeps 123766 (68.0%), 151M tokens
[l2k12] [  5.5s] clus12: 38126 ids / 40.0M tokens; first 12M = 11775 docs, mean len 1018
launched
2576

[stdout]
[g0] [  3.1s] gate keeps 135601 (74.5%), 168M tokens
[g0] [  5.6s] mass: 38197 ids / 40.0M tokens; first 12M = 11863 docs, mean len 1011
[l2qa] [  2.9s] gate keeps 123766 (68.0%), 151M tokens
[l2qa] [  5.2s] mass: 38333 ids / 40.0M tokens; first 12M = 12280 docs, mean len 976
[l2wq] [  2.8s] gate keeps 123766 (68.0%), 151M tokens
[l2wq] [  5.3s] mass: 38754 ids / 40.0M tokens; first 12M = 12071 docs, mean len 993
[l2k12] [  2.8s] gate keeps 123766 (68.0%), 151M tokens
[l2k12] [  5.5s] clus12: 38126 ids / 40.0M tokens; first 12M = 11775 docs, mean len 1018
launched
2576
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/claim.md (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/claim.md", "content": "# Under a fixed token budget, coverage of the target *mixture* beats average document quality\n\n## Hypothesis\n\nFor a broad multi-register target (equal-ish parts encyclopedic / general HQ web prose / news /\ntechnical Q&A), the binding constraint on held-out perplexity at a fixed 12M-token budget is\n**how well the selection covers every register in the target mixture**, not how target-like the\naverage selected document is.\n\nThe standard recipe — score each document for target-likeness and take the top-k — optimises the\nwrong objective. Held-out perplexity is `exp(mean token loss)`, and the mean runs over an eval set\nthat is a *mixture*. A register that the training set starves contributes an enormous loss term\nthat no amount of extra quality elsewhere can offset. So the prediction is: a selection that\n**reserves a token quota for each register** beats a strictly higher-scoring global top-k\nselection, even though its documents are *individually less* target-like on average.\n\nConcretely I claim: gate for prose quality, collapse near-duplicates, then rank documents by a\nper-register importance weight (`log p_register / p_pool` over hashed unigrams+bigrams, per token)\nand **interleave the per-register rankings to the target's token proportions**, so that every\nprefix of the list — including the exact prefix the budget cuts — is mixture-matched.\n\n## Mechanism, and the observable it predicts (not the final perplexity)\n\nMechanism: the pool is not uniform over registers. Raw web text is overwhelmingly general prose\nand news-like; HTML-wrapped technical Q&A is rare. A single global ranking therefore fills the\nbudget from the modes of the pool that best match the target *on average* and leaves the rare\nregister almost unrepresented. Quota-based interleaving forces the rare register in, trading a\nlittle fit on the abundant registers for a large gain on the starved one.\n\n**Predicted observable — the per-register perplexity breakdown.** Going from global top-k to\nmixture-matched interleaving (same gate, same dedup, same budget, same seed), I predicted:\n\n1. the largest improvement lands on **technical Q&A**, the register the pool under-supplies;\n2. **general web prose / news get slightly worse** — top-k over-serves them, so the quota takes\n   tokens away;\n3. it is *not* a uniform quality improvement — the mixture-matched set has a **lower** mean\n   importance weight than top-k on 3 of the 4 registers, so any account based on \"better documents\n   on average\" predicts the opposite sign.\n\nMeasured (frozen `train_nano.py`, 30M GPT, 12M tokens, seed 1337; per-register dev slices):\n\n| dev slice | global top-k | mixture-matched | Δ |\n|---|---|---|---|\n| technical Q&A | 524.8 | **214.0** | **−310.8** |\n| encyclopedic (wikitext) | 819.2 | 710.9 | −108.3 |\n| general web prose | 344.0 | 334.3 | −9.8 |\n| news | 251.7 | **265.4** | **+13.7** (worse, as predicted) |\n| aggregate | 418.4 | 326.0 | −92.4 |\n\nMean per-register importance weight of the selected 12M tokens (higher = more target-like):\n\n| | wiki | news | Q&A | web |\n|---|---|---|---|---|\n| global top-k | −1.358 | **−0.156** | −1.940 | **+0.051** |\n| mixture-matched | −1.469 | −0.359 | **−1.784** | −0.084 |\n\nTop-k wins the average on 3 of 4 registers and still loses by 92 perplexity points. The entire\naggregate gain is bought on the one register it starved (Q&A, a 2.45× loss ratio), at the cost of\nthe register it over-served (news). That is the mechanism, visible without looking at the score.\n\nA second prediction from the same mechanism, also confirmed: because the mechanism is *coverage*,\nnot *purity*, tightening the quality gate should eventually hurt — it shrinks the candidate pool\nfor the rare register faster than it removes junk. Loosening the gate from the target's 1st/99th\npercentile to its 0.05th/99.95th (48% → 68% of the pool admitted) improved aggregate dev\nperplexity monotonically: 326.0 → 315.0 → 306.7. Conversely, adding diversity *noise* within a\nregister (Gumbel-perturbed ranking, T=0.3/1.0) hurt sharply (359.2 / 421.4), and restricting to\nlong documents (≥600 tokens) hurt (345.9). Coverage across registers helps; noise within a\nregister does not.\n\n## Falsification\n\nThe claim is falsified if any of these had come out otherwise, and each is a cheap re-run:\n\n- **Per-register signs.** If mixture-matching had improved *all four* registers, the mechanism\n  would be \"it selects better documents\", not \"it fixes a starved register\". It improved three and\n  degraded news — the predicted trade.\n- **Average-quality account.** If the mixture-matched set had also had a higher mean importance\n  weight per register, the result would be explained by document quality alone. It does not (3 of\n  4 registers lower).\n- **Quota direction.** If register quotas were irrelevant, reallocating them would be inert.\n  Shifting quota toward the starved register (Q&A 25%→40%) helped slightly (315.0 → 312.4) while\n  shifting toward the abundant one (web →50%) did not (314.1) — weakly consistent, and the honest\n  reading is that quota *fine-tuning* is a second-order knob once every register is non-empty.\n  This is the weakest leg of the claim: the effect is near the run-to-run resolution of a single\n  seed, so I do not claim an optimal quota, only that non-zero coverage of each register matters.\n- **Ordering, not filtering.** If the gain came from the filters rather than the mixture, then\n  gate+dedup with a random order would score near the mixture-matched number. It scores 460.3\n  against a 470.1 random baseline — the filters are worth ~10 points, the mixture ~134.\n- The disclosed target and the hidden target are different samples of the same domain. If the\n  effect were dev-set memorisation rather than distribution matching, it would not survive the\n  hidden sample; the criterion touches the dev text only through ~1M tokens of *n-gram statistics\n  and register proportions*, never through document identity, and it selects only real pool\n  documents.\n\n## Transfer\n\nThe recipe needs only a small unlabeled sample of the target and no quality labels, so it\ntransfers to any budget-constrained pretraining or continued-pretraining mix where the evaluation\nis known to be a mixture: swap the four registers for the domains you care about (languages, code\nvs prose, medical vs legal), keep the per-group importance weights and the quota interleave.\n\nThe transferable rule is sharper than \"filter for quality\": **when the objective is a mean over a\nmixture, select by per-group quotas, and make the ordering safe under truncation** — since a\nbudget cuts the list at an arbitrary point, every prefix must already be mixture-matched, which\ninterleaving guarantees and a global ranking does not. The same argument applies to any\nbudget-truncated data pipeline (RL prompt mixes, eval-set construction, retrieval index budgets).\n\nWhere it should *not* transfer: single-domain targets (then global top-k is right, there is no\nstarved register), and regimes where the budget is large enough to cover every register anyway —\nthe effect here is a fixed-budget scarcity effect, and it should shrink as the budget grows.\n\n## Selection actually submitted\n\n`submission/curate.py` (stated criterion, no hand-picked ids) → `submission/selection.json`:\ntarget-calibrated quality gate, minhash near-duplicate collapse, per-register importance ranking\ninterleaved to the target's measured token proportions. Dev perplexity of the submitted selection\nis reported in `submission/RESULTS.md` alongside every variant above; random baseline 470.1.\n", "structuredPatch": [], "originalFile": null, "userModified": false}

--- loose2=306.71 loose=314.96 lqa=312.42
2426

[stdout]
--- loose2=306.71 loose=314.96 lqa=312.42
2426
[metadata] {"noOutputExpected": false}

HELDOUT_PPL 308.938  loss 5.7331  train_s 216  /tmp/tr_l2wq.npy
HELDOUT_PPL 308.423  loss 5.7315  train_s 215  /tmp/tr_l2qa.npy
HELDOUT_PPL 311.696  loss 5.7420  train_s 216  /tmp/tr_l2k12.npy
HELDOUT_PPL 306.658  loss 5.7257  train_s 216  /tmp/tr_g0.npy
--- loose2=306.71
2305

[stdout]
HELDOUT_PPL 308.938  loss 5.7331  train_s 216  /tmp/tr_l2wq.npy
HELDOUT_PPL 308.423  loss 5.7315  train_s 215  /tmp/tr_l2qa.npy
HELDOUT_PPL 311.696  loss 5.7420  train_s 216  /tmp/tr_l2k12.npy
HELDOUT_PPL 306.658  loss 5.7257  train_s 216  /tmp/tr_g0.npy
--- loose2=306.71
2305
[metadata] {"noOutputExpected": false}

[   0.4s] pool: 182016 docs, 199.9M tokens
[   0.4s] target: 2657 segments, 996k tokens
[   4.1s] doc features ready
[   4.8s] stage1 gate: 123766 docs (68.0%), 151M tokens
[   5.0s] 4 target groups, token shares [0.164 0.173 0.251 0.412]
[   6.3s] importance weights done
[   6.3s] stage2 candidates: 108540 docs, 121M tokens
[   6.6s] wrote 38420 ids (40.0M tokens) -> /workspace/submission/selection.json
[   6.6s] budget prefix: 12026 docs, mean len 997 tokens, group shares [0.164 0.173 0.251 0.412]

[stdout]
[   0.4s] pool: 182016 docs, 199.9M tokens
[   0.4s] target: 2657 segments, 996k tokens
[   4.1s] doc features ready
[   4.8s] stage1 gate: 123766 docs (68.0%), 151M tokens
[   5.0s] 4 target groups, token shares [0.164 0.173 0.251 0.412]
[   6.3s] importance weights done
[   6.3s] stage2 candidates: 108540 docs, 121M tokens
[   6.6s] wrote 38420 ids (40.0M tokens) -> /workspace/submission/selection.json
[   6.6s] budget prefix: 12026 docs, mean len 997 tokens, group shares [0.164 0.173 0.251 0.412]
[metadata] {"noOutputExpected": false}

same as tested loose2 variant: False 38420
unique: True in range: True
official packer running

[stdout]
same as tested loose2 variant: False 38420
unique: True in range: True
official packer running
[metadata] {"noOutputExpected": false}

set equal: False len 38420 38420
first order diff at 74 num diffs 1229
a[:6] [51268, 58452, 180237, 66305, 44450, 8600]
b[:6] [51268, 58452, 180237, 66305, 44450, 8600]
prefix sets equal: False 12026 12026

[stdout]
set equal: False len 38420 38420
first order diff at 74 num diffs 1229
a[:6] [51268, 58452, 180237, 66305, 44450, 8600]
b[:6] [51268, 58452, 180237, 66305, 44450, 8600]
prefix sets equal: False 12026 12026
[metadata] {"noOutputExpected": false}

packed 12000000 tokens from 38420 ids -> /tmp/final_official.npy
packed 12000000 tokens from 38420 ids -> /tmp/final_fast.npy
fast==official: True (12000000,)
verification + per-register launched
2235

[stdout]
packed 12000000 tokens from 38420 ids -> /tmp/final_official.npy
packed 12000000 tokens from 38420 ids -> /tmp/final_fast.npy
fast==official: True (12000000,)
verification + per-register launched
2235
[metadata] {"noOutputExpected": false}

patched
syntax ok

[stdout]
patched
syntax ok
[metadata] {"noOutputExpected": false}

wiki  -
news  -
qa    -
web   -
2075

[stdout]
wiki  -
news  -
qa    -
web   -
2075
[metadata] {"noOutputExpected": false}

HELDOUT_PPL 304.387  loss 5.7183  train_s 269  /tmp/final_official.npy
wiki  PPL 742.447
news  PPL 276.479
qa    PPL 148.984
web   PPL 341.582
1945

[stdout]
HELDOUT_PPL 304.387  loss 5.7183  train_s 269  /tmp/final_official.npy
wiki  PPL 742.447
news  PPL 276.479
qa    PPL 148.984
web   PPL 341.582
1945
[metadata] {"noOutputExpected": false}

[   6.1s] importance weights done
[   6.2s] stage2 candidates: 108540 docs, 121M tokens
[   6.5s] wrote 38420 ids (40.0M tokens) -> /tmp/sel_C.json
[   6.5s] budget prefix: 12026 docs, mean len 997 tokens, group shares [0.164 0.173 0.251 0.412]
[   3.7s] wrote 38420 ids (40.0M tokens) -> /tmp/sel_C2.json
[   3.7s] budget prefix: 12026 docs, mean len 997 tokens, group shares [0.164 0.173 0.251 0.412]
deterministic across runs: True
vs A: identical False | set overlap 38308 / 38420

[stdout]
[   6.1s] importance weights done
[   6.2s] stage2 candidates: 108540 docs, 121M tokens
[   6.5s] wrote 38420 ids (40.0M tokens) -> /tmp/sel_C.json
[   6.5s] budget prefix: 12026 docs, mean len 997 tokens, group shares [0.164 0.173 0.251 0.412]
[   3.7s] wrote 38420 ids (40.0M tokens) -> /tmp/sel_C2.json
[   3.7s] budget prefix: 12026 docs, mean len 997 tokens, group shares [0.164 0.173 0.251 0.412]
deterministic across runs: True
vs A: identical False | set overlap 38308 / 38420
[metadata] {"noOutputExpected": false}

packed 12000000 tokens from 38420 ids -> /tmp/tr_C.npy
launched
1905

[stdout]
packed 12000000 tokens from 38420 ids -> /tmp/tr_C.npy
launched
1905
[metadata] {"noOutputExpected": false}