claude-opus-4-8

5390
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 03:22 .
drwxr-xr-x 1 root root  160 Aug  4 03:22 ..
drwxr-xr-x 1 root root   60 Aug  4 03:22 .live
drwxr-xr-x 1 root root  100 Aug  4 03:22 .metrics
drwxr-xr-x 1 root root  140 Aug  4 03:22 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 03:22 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 03:22 ..

[stdout]
5390
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 03:22 .
drwxr-xr-x 1 root root  160 Aug  4 03:22 ..
drwxr-xr-x 1 root root   60 Aug  4 03:22 .live
drwxr-xr-x 1 root root  100 Aug  4 03:22 .metrics
drwxr-xr-x 1 root root  140 Aug  4 03:22 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 03:22 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 03:22 ..
[metadata] {"noOutputExpected": false}

=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")

=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")

=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"

=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done

[stdout]
=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")

=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")

=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"

=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done
[metadata] {"noOutputExpected": false}

=== model.py ===
"""A small GPT (decoder-only Transformer) used as the distillation *student*.

Self-contained re-implementation in this repo's own idiom (no external GPT code).
GPT-2 byte-level BPE vocabulary (50257) so the student shares the teacher's
tokenizer. Standard pre-norm Transformer with weight-tied embeddings.
"""
from dataclasses import dataclass
import math
import torch
import torch.nn as nn
import torch.nn.functional as F


@dataclass
class GPTConfig:
    block_size: int = 256
    vocab_size: int = 50257
    n_layer: int = 6
    n_head: int = 6
    n_embd: int = 384
    dropout: float = 0.0
    bias: bool = False


class LayerNorm(nn.Module):
    def __init__(self, ndim, bias):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(ndim))
        self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None

    def forward(self, x):
        return F.layer_norm(x, self.weight.shape, self.weight, self.bias, 1e-5)


class CausalSelfAttention(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
        assert cfg.n_embd % cfg.n_head == 0
        self.c_attn = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=cfg.bias)
        self.c_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias)
        self.n_head = cfg.n_head
        self.n_embd = cfg.n_embd
        self.dropout = cfg.dropout

    def forward(self, x):
        B, T, C = x.size()
        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        y = F.scaled_dot_product_attention(
            q, k, v, dropout_p=self.dropout if self.training else 0.0, is_causal=True
        )
        y = y.transpose(1, 2).contiguous().view(B, T, C)
        return self.c_proj(y)


class MLP(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
...
=== pool head (first doc, truncated) ===
{"id": 0, "text": "|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, really, but hoping that we get some good \"SAMANTHA GENE!!\" Marlena Death-Stares out of it. And \"newfound\" feelings. Please. If only.\nSTEFANO!! STEFANO, STEFANO, STEFANO!!!! :cheer:\n|Spoilers for the Week of February 11th \u00b7 DAYS: News, Spoilers & Discussion|"}
{"id": 1, "text": "*sigh* Fundamentalist community, let me pass on some advice to you I learned from the atheistic community:\nIf you have set yourself on fire, do not run.\nOkay? Okay?? Please?\nLook, D, you had two months to say to Harvard in private emails, \"Im sorry, I shouldnt have been using that animation in my paid presentations. I wont use it again. I really do like 'Inner Life', though, and would love to use it in classroom presentations, from the BioVisions site, if that is acceptable.\"\nI sat here, for two months, waiting for that to happen, anything to happen, and it didnt. Two months, on your own terms, you could have written a similar post to yesterdays. I would have given you the benefit of the doubt-- maybe you didnt know the credits werent visible to the audience, and I wouldnt have said a word beyond this, as its Harvards problem, not mine. This would have been a funny joke to those of us involved in dealing with you people, but it would have been a PR non-issue for you.\nBut after you set yourself on fire, you did
=== pool line count ===
182016 data/pool.jsonl

[stdout]
=== model.py ===
"""A small GPT (decoder-only Transformer) used as the distillation *student*.

Self-contained re-implementation in this repo's own idiom (no external GPT code).
GPT-2 byte-level BPE vocabulary (50257) so the student shares the teacher's
tokenizer. Standard pre-norm Transformer with weight-tied embeddings.
"""
from dataclasses import dataclass
import math
import torch
import torch.nn as nn
import torch.nn.functional as F


@dataclass
class GPTConfig:
    block_size: int = 256
    vocab_size: int = 50257
    n_layer: int = 6
    n_head: int = 6
    n_embd: int = 384
    dropout: float = 0.0
    bias: bool = False


class LayerNorm(nn.Module):
    def __init__(self, ndim, bias):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(ndim))
        self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None

    def forward(self, x):
        return F.layer_norm(x, self.weight.shape, self.weight, self.bias, 1e-5)


class CausalSelfAttention(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
        assert cfg.n_embd % cfg.n_head == 0
        self.c_attn = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=cfg.bias)
        self.c_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias)
        self.n_head = cfg.n_head
        self.n_embd = cfg.n_embd
        self.dropout = cfg.dropout

    def forward(self, x):
        B, T, C = x.size()
        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        y = F.scaled_dot_product_attention(
            q, k, v, dropout_p=self.dropout if self.training else 0.0, is_causal=True
        )
        y = y.transpose(1, 2).contiguous().view(B, T, C)
        return self.c_proj(y)


class MLP(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
...
=== pool head (first doc, truncated) ===
{"id": 0, "text": "|Viewing Single Post From: Spoilers for the Week of February 11th|\n|Lil||Feb 1 2013, 09:58 AM|\nDon't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, really, but hoping that we get some good \"SAMANTHA GENE!!\" Marlena Death-Stares out of it. And \"newfound\" feelings. Please. If only.\nSTEFANO!! STEFANO, STEFANO, STEFANO!!!! :cheer:\n|Spoilers for the Week of February 11th \u00b7 DAYS: News, Spoilers & Discussion|"}
{"id": 1, "text": "*sigh* Fundamentalist community, let me pass on some advice to you I learned from the atheistic community:\nIf you have set yourself on fire, do not run.\nOkay? Okay?? Please?\nLook, D, you had two months to say to Harvard in private emails, \"Im sorry, I shouldnt have been using that animation in my paid presentations. I wont use it again. I really do like 'Inner Life', though, and would love to use it in classroom presentations, from the BioVisions site, if that is acceptable.\"\nI sat here, for two months, waiting for that to happen, anything to happen, and it didnt. Two months, on your own terms, you could have written a similar post to yesterdays. I would have given you the benefit of the doubt-- maybe you didnt know the credits werent visible to the audience, and I wouldnt have said a word beyond this, as its Harvards problem, not mine. This would have been a funny joke to those of us involved in dealing with you people, but it would have been a PR non-issue for you.\nBut after you set yourself on fire, you did
=== pool line count ===
182016 data/pool.jsonl
[metadata] {"noOutputExpected": false}

dev tokens: (1000000,) uint16
==== chunk at frac 0.0 ====
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member

==== chunk at frac 0.25 ====
Description of a very high speed transit (VHST) system operating in its own rarefied atmosphere in evacuated tubes in underground tunnels. Most cases considered took less time to go coast-to-coast (e.g., 21 min) than it takes an aircraft to climb to an efficient operating altitude. VHST's tubecraft ride on, and are driven by, electromagnetic (EM) waves. In accelerating, it employs the energy of the surrounding EM field; in decelerating, it returns most of this energy to the system. Tunnel systems would be shared by oil, water, and gas pipelines; channels for laser and microwave waveguides; electric power lines including superconducting ones; and freight systems. Environmental and economic benefits are substantial, and the technology for building and operating the system exists.

This report is part of the RAND Corporation paper series. The

==== chunk at frac 0.5 ====
I fucking hate when he does this shit pic.twitter.com/kpmcHnW4Cz — Miley Ray Cyrus (@MileyCyrus) April 22, 2018
Singer-actress Miley Cyrus has shared a rare video of herself with her partner, actor Liam Hemsworth, on social media. She shared a video of them both in a car, which showed Hemsworth listening to rap music as he drove them to their destination.Cyrus captioned the image: "I f***ing hate it when he does this s**t."In the clip, she was sporting a top which had the word 'Sunday' emblazoned across the front as she sat in the passenger seat. With his music blaring out, she filmed herself dancing and bopping her head to the music. Then all of a sudden, he made Cyrus jump as he suddenly

==== chunk at frac 0.75 ====
<p>I found the platform module but it says it returns 'Windows' and it's returning 'Microsoft' on my machine. I notice in another thread here on stackoverflow it returns 'Vista' sometimes.</p>
<p>So, the question is, how do implemement?</p>
<pre><code>if is_windows():
  ...
</code></pre>
<p>In a forward compatible way? If I have to check for things like 'Vista' then it will break when the next version of windows comes out.</p>
<hr />
<p>Note: The answers claiming this is a duplicate question do not actually answer the question <code>is_windows</code>. They answer the question &quot;what platform&quot;. Since many flavors of windows exist none of them

[stdout]
dev tokens: (1000000,) uint16
==== chunk at frac 0.0 ====
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member

==== chunk at frac 0.25 ====
Description of a very high speed transit (VHST) system operating in its own rarefied atmosphere in evacuated tubes in underground tunnels. Most cases considered took less time to go coast-to-coast (e.g., 21 min) than it takes an aircraft to climb to an efficient operating altitude. VHST's tubecraft ride on, and are driven by, electromagnetic (EM) waves. In accelerating, it employs the energy of the surrounding EM field; in decelerating, it returns most of this energy to the system. Tunnel systems would be shared by oil, water, and gas pipelines; channels for laser and microwave waveguides; electric power lines including superconducting ones; and freight systems. Environmental and economic benefits are substantial, and the technology for building and operating the system exists.

This report is part of the RAND Corporation paper series. The

==== chunk at frac 0.5 ====
I fucking hate when he does this shit pic.twitter.com/kpmcHnW4Cz — Miley Ray Cyrus (@MileyCyrus) April 22, 2018
Singer-actress Miley Cyrus has shared a rare video of herself with her partner, actor Liam Hemsworth, on social media. She shared a video of them both in a car, which showed Hemsworth listening to rap music as he drove them to their destination.Cyrus captioned the image: "I f***ing hate it when he does this s**t."In the clip, she was sporting a top which had the word 'Sunday' emblazoned across the front as she sat in the passenger seat. With his music blaring out, she filmed herself dancing and bopping her head to the music. Then all of a sudden, he made Cyrus jump as he suddenly

==== chunk at frac 0.75 ====
<p>I found the platform module but it says it returns 'Windows' and it's returning 'Microsoft' on my machine. I notice in another thread here on stackoverflow it returns 'Vista' sometimes.</p>
<p>So, the question is, how do implemement?</p>
<pre><code>if is_windows():
  ...
</code></pre>
<p>In a forward compatible way? If I have to check for things like 'Vista' then it will break when the next version of windows comes out.</p>
<hr />
<p>Note: The answers claiming this is a duplicate question do not actually answer the question <code>is_windows</code>. They answer the question &quot;what platform&quot;. Since many flavors of windows exist none of them
[metadata] {"noOutputExpected": false}

Task #1 created successfully: Write curate.py: n-gram LLR scoring vs target

[metadata] {"task": {"id": "1", "subject": "Write curate.py: n-gram LLR scoring vs target"}}

Traceback (most recent call last):
  File "<string>", line 1, in <module>
ModuleNotFoundError: No module named 'sklearn'
NVIDIA H100 80GB HBM3, 81559 MiB
cuda True

[stdout]
Traceback (most recent call last):
  File "<string>", line 1, in <module>
ModuleNotFoundError: No module named 'sklearn'
NVIDIA H100 80GB HBM3, 81559 MiB
cuda True
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/submission/curate.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/submission/curate.py", "content": "#!/usr/bin/env python3\n\"\"\"Curate the raw web pool for training a small LM whose held-out target is a\nBROAD, multi-domain high-quality English distribution (Wikipedia + high-quality\nweb prose + news + technical Q&A), disclosed via /workspace/data/multi_dev.npy.\n\nCriterion (stated, reproducible): DSIR-style importance scoring.\n  * Decode the disclosed target token array back to text -> a sample of the\n    target domain.\n  * Fit a unigram+bigram bag-of-words distribution for the TARGET and for a\n    random BACKGROUND sample of the raw pool.\n  * Score every pool document by its mean per-token log-likelihood ratio\n    log p_target(w) - log p_background(w).  Documents whose vocabulary looks\n    like the clean multi-domain target score high; boilerplate/forum/spam\n    (whose n-grams are common in the raw pool but rare in the target) score low.\n  * Apply light hard filters (length floor, alpha-ratio, repetition cap) to\n    drop degenerate documents that no domain wants.\n  * Emit pool ids in descending score order (best first).  The training\n    pipeline consumes this priority list until the 12M-token budget is full.\n\nThis is a pure ranking function over a stated feature, not a hand-picked list.\n\"\"\"\nimport json, re, math, time, random\nimport numpy as np\nfrom collections import Counter\n\nPOOL = \"/workspace/data/pool.jsonl\"\nTARGET_NPY = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\n\nN_UNI = 60_000          # top target unigrams kept in the model vocabulary\nN_BI  = 60_000          # top target bigrams kept\nBG_SAMPLE_FRAC = 0.40   # fraction of pool docs used to estimate the background\nALPHA = 1.0             # additive smoothing\nN_OUT = 40_000          # ids to emit (>> enough to cover 12M tokens)\nSEED = 0\n\n# hard quality filters (register-agnostic: they only drop degenerate docs)\nMIN_WORDS = 40          # too short to teach anything, wastes an EOS slot\nMIN_ALPHA_RATIO = 0.55  # fraction of chars that are letters/space/basic punct\nMAX_TOP_WORD_FRAC = 0.30  # single word dominating -> spam/repetition\n\nrandom.seed(SEED)\nWORD_RE = re.compile(r\"[a-z0-9]+\")\n\n\ndef words(text):\n    return WORD_RE.findall(text.lower())\n\n\ndef passes_filters(text):\n    if len(text) < 200:\n        return False, None\n    w = words(text)\n    if len(w) < MIN_WORDS:\n        return False, None\n    # alpha ratio on raw text\n    good = sum(c.isalpha() or c.isspace() or c in \",.;:'\\\"?!()-\" for c in text)\n    if good / max(1, len(text)) < MIN_ALPHA_RATIO:\n        return False, None\n    c = Counter(w)\n    if c.most_common(1)[0][1] / len(w) > MAX_TOP_WORD_FRAC:\n        return False, None\n    return True, w\n\n\ndef main():\n    t0 = time.time()\n    print(\"loading pool...\", flush=True)\n    ids, texts = [], []\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            ids.append(r[\"id\"]); texts.append(r[\"text\"])\n    N = len(ids)\n    print(f\"  {N} docs loaded in {time.time()-t0:.0f}s\", flush=True)\n\n    # ---- target distribution ----\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    tgt_tokens = np.load(TARGET_NPY)\n    # decode in chunks to bound memory\n    tgt_text_parts = []\n    step = 4096\n    for s in range(0, len(tgt_tokens), step):\n        tgt_text_parts.append(tok.decode(tgt_tokens[s:s+step].tolist()))\n    tgt_text = \" \".join(tgt_text_parts)\n    tw = words(tgt_text)\n    tgt_uni = Counter(tw)\n    tgt_bi = Counter(a + \" \" + b for a, b in zip(tw, tw[1:]))\n    print(f\"  target: {len(tw)} words, {len(tgt_uni)} uniq unigrams, \"\n          f\"{len(tgt_bi)} uniq bigrams  ({time.time()-t0:.0f}s)\", flush=True)\n\n    vocab_uni = {w: i for i, (w, _) in enumerate(\n        [x for x in tgt_uni.most_common(N_UNI) if x[1] >= 2])}\n    vocab_bi = {w: i for i, (w, _) in enumerate(\n        [x for x in tgt_bi.most_common(N_BI) if x[1] >= 2])}\n    VU, VB = len(vocab_uni), len(vocab_bi)\n    print(f\"  vocab: {VU} unigrams, {VB} bigrams kept\", flush=True)\n\n    tgt_uc = np.full(VU, 0.0); tgt_bc = np.full(VB, 0.0)\n    for w, i in vocab_uni.items():\n        tgt_uc[i] = tgt_uni[w]\n    for w, i in vocab_bi.items():\n        tgt_bc[i] = tgt_bi[w]\n\n    # ---- background distribution (random pool sample) ----\n    bg_uc = np.zeros(VU); bg_bc = np.zeros(VB)\n    bg_idx = [j for j in range(N) if random.random() < BG_SAMPLE_FRAC]\n    for j in bg_idx:\n        w = words(texts[j])\n        for x in w:\n            i = vocab_uni.get(x)\n            if i is not None:\n                bg_uc[i] += 1\n        for a, b in zip(w, w[1:]):\n            i = vocab_bi.get(a + \" \" + b)\n            if i is not None:\n                bg_bc[i] += 1\n    print(f\"  background from {len(bg_idx)} docs  ({time.time()-t0:.0f}s)\", flush=True)\n\n    # ---- per-feature log-likelihood-ratio weights ----\n    Tu, Bu = tgt_uc.sum(), bg_uc.sum()\n    Tb, Bb = tgt_bc.sum(), bg_bc.sum()\n    wu = np.log((tgt_uc + ALPHA) / (Tu + ALPHA * VU)) - \\\n         np.log((bg_uc + ALPHA) / (Bu + ALPHA * VU))\n    wb = np.log((tgt_bc + ALPHA) / (Tb + ALPHA * VB)) - \\\n         np.log((bg_bc + ALPHA) / (Bb + ALPHA * VB))\n\n    # ---- score every doc ----\n    scores = np.full(N, -1e9)\n    kept = 0\n    for j in range(N):\n        ok, w = passes_filters(texts[j])\n        if not ok:\n            continue\n        kept += 1\n        su = 0.0\n        for x in w:\n            i = vocab_uni.get(x)\n            if i is not None:\n                su += wu[i]\n        sb = 0.0\n        nb = 0\n        for a, b in zip(w, w[1:]):\n            i = vocab_bi.get(a + \" \" + b)\n            if i is not None:\n                sb += wb[i]\n                nb += 1\n        # mean log-ratio per token; bigrams get half weight of unigrams\n        scores[j] = (su + 0.5 * sb) / len(w)\n    print(f\"  scored; {kept} docs pass filters  ({time.time()-t0:.0f}s)\", flush=True)\n\n    order = np.argsort(-scores)\n    sel = [int(ids[j]) for j in order[:N_OUT] if scores[j] > -1e8]\n    json.dump(sel, open(OUT, \"w\"))\n    print(f\"wrote {len(sel)} ids -> {OUT}  ({time.time()-t0:.0f}s)\", flush=True)\n    # diagnostics\n    top = order[:20]\n    print(\"score quantiles (kept):\",\n          np.round(np.quantile(scores[scores > -1e8], [0, .25, .5, .75, .95, 1]), 3))\n\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [], "originalFile": null, "userModified": false}

Command did not complete within its 120s timeout and was moved to the background (ID: b8kbkjlrc). Output is being written to: /tmp/claude-0/-workspace/2eda86eb-7dd5-4994-9ab5-ba9a2e6aa267/tasks/b8kbkjlrc.output. You will be notified when it completes. To check interim output, use Read on that file path.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "b8kbkjlrc", "timedOutAfterMs": 120000}

loading pool...
  182016 docs loaded in 6s
  target: 699682 words, 38523 uniq unigrams, 326317 uniq bigrams  (11s)
  vocab: 22593 unigrams, 60000 bigrams kept
  background from 73259 docs  (42s)
  scored; 179108 docs pass filters  (143s)
wrote 40000 ids -> /workspace/submission/selection.json  (143s)
score quantiles (kept): [-1.556 -0.462 -0.314 -0.179  0.     0.707]

real	2m24.508s
user	2m15.330s
sys	0m1.540s

[stdout]
loading pool...
  182016 docs loaded in 6s
  target: 699682 words, 38523 uniq unigrams, 326317 uniq bigrams  (11s)
  vocab: 22593 unigrams, 60000 bigrams kept
  background from 73259 docs  (42s)
  scored; 179108 docs pass filters  (143s)
wrote 40000 ids -> /workspace/submission/selection.json  (143s)
score quantiles (kept): [-1.556 -0.462 -0.314 -0.179  0.     0.707]

real	2m24.508s
user	2m15.330s
sys	0m1.540s
[metadata] {"noOutputExpected": false}

4960
--- selection sanity ---
ids: 40000 unique: 40000 min/max: 5 182015
first 10: [137360, 114704, 58452, 37064, 79735, 63278, 144354, 121698, 43758, 37522]

[stdout]
4960
--- selection sanity ---
ids: 40000 unique: 40000 min/max: 5 182015
first 10: [137360, 114704, 58452, 37064, 79735, 63278, 144354, 121698, 43758, 37522]
[metadata] {"noOutputExpected": false}

=== TOP 5 ranked docs (first 220 chars) ===
[137360] 'Attendees | All Canada Games\nRegister Here\nAccommodations\nSelect Page\nRecruits Attending\n<br /><br />\nRecruits will receive a link after registration to get listed as an attendee. Recruits with an existing Co'

[114704] 'Attendees | All Canada Games\nRegister Here\nAccommodations\nSelect Page\nRecruits Attending\n<br /><br />\nRecruits will receive a link after registration to get listed as an attendee. Recruits with an existing Co'

[58452] '<|endoftext|>New Delhi: Five new Supreme Court judges were today sworn in, taking the top court’s strength to 28, including the Chief Justice of India. Justices Sanjay Kishan Kaul, Navin Sinha, Mohan M Shantanagoudar, De'

[37064] 'The Bharatiya Janata Party (BJP) leaders of the West Bengal unit will meet Union Home Minister Rajnath Singh on Thursday to lodge protest against the vandalising of its party headquarters office in the state by the Trina'

[79735] '<|endoftext|>India’s PM Narendra Modi has tweeted pictures of his mother’s first visit to his official residence in Delhi.\nTwitter users have praised Narendra Modi’s decision to invite his mother, Heeraben Modi, at his n'

=== BOTTOM of pool (lowest scored, not selected) ===
random baseline written: 40000 ids

[stdout]
=== TOP 5 ranked docs (first 220 chars) ===
[137360] 'Attendees | All Canada Games\nRegister Here\nAccommodations\nSelect Page\nRecruits Attending\n&lt;br /&gt;&lt;br /&gt;\nRecruits will receive a link after registration to get listed as an attendee. Recruits with an existing Co'

[114704] 'Attendees | All Canada Games\nRegister Here\nAccommodations\nSelect Page\nRecruits Attending\n&lt;br /&gt;&lt;br /&gt;\nRecruits will receive a link after registration to get listed as an attendee. Recruits with an existing Co'

[58452] '<|endoftext|>New Delhi: Five new Supreme Court judges were today sworn in, taking the top court’s strength to 28, including the Chief Justice of India. Justices Sanjay Kishan Kaul, Navin Sinha, Mohan M Shantanagoudar, De'

[37064] 'The Bharatiya Janata Party (BJP) leaders of the West Bengal unit will meet Union Home Minister Rajnath Singh on Thursday to lodge protest against the vandalising of its party headquarters office in the state by the Trina'

[79735] '<|endoftext|>India’s PM Narendra Modi has tweeted pictures of his mother’s first visit to his official residence in Delhi.\nTwitter users have praised Narendra Modi’s decision to invite his mother, Heeraben Modi, at his n'

=== BOTTOM of pool (lowest scored, not selected) ===
random baseline written: 40000 ids
[metadata] {"noOutputExpected": false}

Command running in background with ID: bonblv5lp. Output is being written to: /tmp/claude-0/-workspace/2eda86eb-7dd5-4994-9ab5-ba9a2e6aa267/tasks/bonblv5lp.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bonblv5lp", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

Updated task #1 status

[metadata] {"success": true, "taskId": "1", "updatedFields": ["status"], "statusChange": {"from": "pending", "to": "completed"}}

Token indices sequence length is longer than the specified maximum sequence length for this model (1627 > 1024). Running this sequence through the model will result in indexing errors
packed 12000000 tokens from 40000 ids -> /tmp/dev_train.npy
HELDOUT_PPL 335.604  loss 5.8159  train_s 54  /tmp/dev_train.npy
(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)

[stdout]
Token indices sequence length is longer than the specified maximum sequence length for this model (1627 > 1024). Running this sequence through the model will result in indexing errors
packed 12000000 tokens from 40000 ids -> /tmp/dev_train.npy
HELDOUT_PPL 335.604  loss 5.8159  train_s 54  /tmp/dev_train.npy
(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)
[metadata] {"noOutputExpected": false}

Command running in background with ID: bwlqcroky. Output is being written to: /tmp/claude-0/-workspace/2eda86eb-7dd5-4994-9ab5-ba9a2e6aa267/tasks/bwlqcroky.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bwlqcroky", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

window register tags across target: Counter({'?': 244, 'WIKI': 131, 'QA': 124})

[stdout]
window register tags across target: Counter({'?': 244, 'WIKI': 131, 'QA': 124})
[metadata] {"noOutputExpected": false}

--- selection register mix (top 12k) ---
top-12k selected register tags: Counter({'prose/news': 11977, 'QA/html': 23})

[stdout]
--- selection register mix (top 12k) ---
top-12k selected register tags: Counter({'prose/news': 11977, 'QA/html': 23})
[metadata] {"noOutputExpected": false}

=== random baseline output ===
HELDOUT_PPL 459.787  loss 6.1308  train_s 54  /tmp/dev_train.npy
=== pool register availability ===
pool docs with html/qa markers: 278 of 182016
example qa id: 2033

[stdout]
=== random baseline output ===
HELDOUT_PPL 459.787  loss 6.1308  train_s 54  /tmp/dev_train.npy
=== pool register availability ===
pool docs with html/qa markers: 278 of 182016
example qa id: 2033
[metadata] {"noOutputExpected": false}

4649

[stdout]
4649
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"type": "update", "filePath": "/workspace/submission/curate.py", "content": "#!/usr/bin/env python3\n\"\"\"Curate the raw web pool for training a small LM whose held-out target is a\nBROAD, multi-domain high-quality English distribution (Wikipedia + high-quality\nweb prose + news + technical Q&A), disclosed via /workspace/data/multi_dev.npy.\n\nCriterion (stated, reproducible): DSIR-style importance scoring with\nregister-aware coverage.\n\n  1. Decode the disclosed target token array back to text -> a sample of the\n     target domain.\n  2. Fit a unigram+bigram bag-of-features distribution for the TARGET and for a\n     random BACKGROUND sample of the raw pool.  Features are HTML/entity aware\n     (`<code>`, `<p>`, `&quot;` are tokens) so the technical-Q&A register is\n     represented, not stripped.\n  3. Score every pool document by its mean per-token log-likelihood ratio\n     log p_target(f) - log p_background(f).  Clean multi-domain prose scores\n     high; boilerplate / forum / spam scores low.\n  4. Light hard filters drop only degenerate docs (too short, gibberish/binary,\n     single word repeated).  Exact-duplicate texts are removed.\n  5. The target is ~1/4 technical Q&A, but such HTML/code docs are extremely\n     rare in this raw pool.  To avoid starving that register we surface EVERY\n     genuine Q&A/code document (ranked by score) ahead of the prose tail, then\n     fill the remaining budget with the highest-scoring prose.\n\nEmits pool ids best-first; the trainer consumes them until the 12M-token budget\nis full.  This is a pure ranking function over a stated feature, not a\nhand-picked list.\n\"\"\"\nimport json, re, time, random\nimport numpy as np\nfrom collections import Counter\n\nPOOL = \"/workspace/data/pool.jsonl\"\nTARGET_NPY = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\n\nN_UNI = 60_000\nN_BI = 60_000\nBG_SAMPLE_FRAC = 0.40\nALPHA = 1.0\nN_OUT = 40_000\nSEED = 0\n\nMIN_WORDS = 50            # too short to teach anything\nMIN_ASCII_RATIO = 0.90    # guard against binary / non-English gibberish\nMAX_TOP_WORD_FRAC = 0.28  # a single word dominating -> spam / repetition\n\nrandom.seed(SEED)\n# HTML tags, HTML entities, and alnum words are all tokens (keeps the Q&A/code register)\nTOK_RE = re.compile(r\"</?[a-z][a-z0-9]*/?>|&[a-z]+;|[a-z0-9]+\")\nQA_MARK = (\"<p>\", \"<code>\", \"<pre>\", \"</\", \"&quot;\", \"&lt;\", \"&gt;\", \"&amp;\",\n           \"def \", \"import \", \"function \", \"public static\", \"#include\")\n\n\ndef toks(text):\n    return TOK_RE.findall(text.lower())\n\n\ndef is_qa(text):\n    return any(m in text for m in QA_MARK)\n\n\ndef passes(text):\n    if len(text) < 200:\n        return False, None\n    w = toks(text)\n    if len(w) < MIN_WORDS:\n        return False, None\n    ascii_ok = sum(32 <= ord(c) < 127 or c in \"\\n\\t\\r\" for c in text)\n    if ascii_ok / len(text) < MIN_ASCII_RATIO:\n        return False, None\n    c = Counter(w)\n    if c.most_common(1)[0][1] / len(w) > MAX_TOP_WORD_FRAC:\n        return False, None\n    return True, w\n\n\ndef main():\n    t0 = time.time()\n    print(\"loading pool...\", flush=True)\n    ids, texts = [], []\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            ids.append(r[\"id\"]); texts.append(r[\"text\"])\n    N = len(ids)\n    print(f\"  {N} docs loaded in {time.time()-t0:.0f}s\", flush=True)\n\n    # ---- target distribution ----\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    tgt_tokens = np.load(TARGET_NPY)\n    parts = []\n    for s in range(0, len(tgt_tokens), 4096):\n        parts.append(tok.decode(tgt_tokens[s:s+4096].tolist()))\n    tw = toks(\" \".join(parts))\n    tgt_uni = Counter(tw)\n    tgt_bi = Counter(a + \" \" + b for a, b in zip(tw, tw[1:]))\n    print(f\"  target: {len(tw)} tokens, {len(tgt_uni)} uni, {len(tgt_bi)} bi \"\n          f\"({time.time()-t0:.0f}s)\", flush=True)\n\n    vocab_uni = {w: i for i, (w, _) in enumerate(\n        [x for x in tgt_uni.most_common(N_UNI) if x[1] >= 2])}\n    vocab_bi = {w: i for i, (w, _) in enumerate(\n        [x for x in tgt_bi.most_common(N_BI) if x[1] >= 2])}\n    VU, VB = len(vocab_uni), len(vocab_bi)\n    print(f\"  vocab kept: {VU} uni, {VB} bi\", flush=True)\n\n    tgt_uc = np.zeros(VU); tgt_bc = np.zeros(VB)\n    for w, i in vocab_uni.items():\n        tgt_uc[i] = tgt_uni[w]\n    for w, i in vocab_bi.items():\n        tgt_bc[i] = tgt_bi[w]\n\n    # ---- background distribution (random pool sample) ----\n    bg_uc = np.zeros(VU); bg_bc = np.zeros(VB)\n    bg_idx = [j for j in range(N) if random.random() < BG_SAMPLE_FRAC]\n    for j in bg_idx:\n        w = toks(texts[j])\n        for x in w:\n            i = vocab_uni.get(x)\n            if i is not None:\n                bg_uc[i] += 1\n        for a, b in zip(w, w[1:]):\n            i = vocab_bi.get(a + \" \" + b)\n            if i is not None:\n                bg_bc[i] += 1\n    print(f\"  background from {len(bg_idx)} docs ({time.time()-t0:.0f}s)\", flush=True)\n\n    Tu, Bu = tgt_uc.sum(), bg_uc.sum()\n    Tb, Bb = tgt_bc.sum(), bg_bc.sum()\n    wu = np.log((tgt_uc + ALPHA) / (Tu + ALPHA * VU)) - \\\n         np.log((bg_uc + ALPHA) / (Bu + ALPHA * VU))\n    wb = np.log((tgt_bc + ALPHA) / (Tb + ALPHA * VB)) - \\\n         np.log((bg_bc + ALPHA) / (Bb + ALPHA * VB))\n\n    # ---- score every doc; track register + dedup ----\n    scores = np.full(N, -1e9)\n    qa_flag = np.zeros(N, dtype=bool)\n    seen = set()\n    kept = 0\n    for j in range(N):\n        h = hash(texts[j])\n        if h in seen:\n            continue          # exact-duplicate text\n        ok, w = passes(texts[j])\n        if not ok:\n            continue\n        seen.add(h)\n        kept += 1\n        qa_flag[j] = is_qa(texts[j])\n        su = 0.0\n        for x in w:\n            i = vocab_uni.get(x)\n            if i is not None:\n                su += wu[i]\n        sb = 0.0\n        for a, b in zip(w, w[1:]):\n            i = vocab_bi.get(a + \" \" + b)\n            if i is not None:\n                sb += wb[i]\n        scores[j] = (su + 0.5 * sb) / len(w)\n    print(f\"  scored; {kept} unique docs pass ({time.time()-t0:.0f}s)\", flush=True)\n\n    valid = scores > -1e8\n    order = np.argsort(-scores)\n    order = [j for j in order if valid[j]]\n    # surface all genuine Q&A/code docs (best-first) ahead of the prose tail\n    qa_order = [j for j in order if qa_flag[j]]\n    prose_order = [j for j in order if not qa_flag[j]]\n    final = qa_order + prose_order\n    sel = [int(ids[j]) for j in final[:N_OUT]]\n    json.dump(sel, open(OUT, \"w\"))\n    print(f\"  qa docs surfaced: {len(qa_order)}\", flush=True)\n    print(f\"wrote {len(sel)} ids -> {OUT} ({time.time()-t0:.0f}s)\", flush=True)\n    sv = scores[valid]\n    print(\"score quantiles:\", np.round(np.quantile(sv, [0, .5, .9, 1]), 3))\n\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 3, "oldLines": 23, "newStart": 3, "newLines": 30, "lines": [" BROAD, multi-domain high-quality English distribution (Wikipedia + high-quality", " web prose + news + technical Q&A), disclosed via /workspace/data/multi_dev.npy.", " ", "-Criterion (stated, reproducible): DSIR-style importance scoring.", "-  * Decode the disclosed target token array back to text -> a sample of the", "-    target domain.", "-  * Fit a unigram+bigram bag-of-words distribution for the TARGET and for a", "-    random BACKGROUND sample of the raw pool.", "-  * Score every pool document by its mean per-token log-likelihood ratio", "-    log p_target(w) - log p_background(w).  Documents whose vocabulary looks", "-    like the clean multi-domain target score high; boilerplate/forum/spam", "-    (whose n-grams are common in the raw pool but rare in the target) score low.", "-  * Apply light hard filters (length floor, alpha-ratio, repetition cap) to", "-    drop degenerate documents that no domain wants.", "-  * Emit pool ids in descending score order (best first).  The training", "-    pipeline consumes this priority list until the 12M-token budget is full.", "+Criterion (stated, reproducible): DSIR-style importance scoring with", "+register-aware coverage.", " ", "-This is a pure ranking function over a stated feature, not a hand-picked list.", "+  1. Decode the disclosed target token array back to text -> a sample of the", "+     target domain.", "+  2. Fit a unigram+bigram bag-of-features distribution for the TARGET and for a", "+     random BACKGROUND sample of the raw pool.  Features are HTML/entity aware", "+     (`<code>`, `<p>`, `&quot;` are tokens) so the technical-Q&A register is", "+     represented, not stripped.", "+  3. Score every pool document by its mean per-token log-likelihood ratio", "+     log p_target(f) - log p_background(f).  Clean multi-domain prose scores", "+     high; boilerplate / forum / spam scores low.", "+  4. Light hard filters drop only degenerate docs (too short, gibberish/binary,", "+     single word repeated).  Exact-duplicate texts are removed.", "+  5. The target is ~1/4 technical Q&A, but such HTML/code docs are extremely", "+     rare in this raw pool.  To avoid starving that register we surface EVERY", "+     genuine Q&A/code document (ranked by score) ahead of the prose tail, then", "+     fill the remaining budget with the highest-scoring prose.", "+", "+Emits pool ids best-first; the trainer consumes them until the 12M-token budget", "+is full.  This is a pure ranking function over a stated feature, not a", "+hand-picked list.", " \"\"\"", "-import json, re, math, time, random", "+import json, re, time, random", " import numpy as np", " from collections import Counter", " "]}, {"oldStart": 27, "oldLines": 35, "newStart": 34, "newLines": 40, "lines": [" TARGET_NPY = \"/workspace/data/multi_dev.npy\"", " OUT = \"/workspace/submission/selection.json\"", " ", "-N_UNI = 60_000          # top target unigrams kept in the model vocabulary", "-N_BI  = 60_000          # top target bigrams kept", "-BG_SAMPLE_FRAC = 0.40   # fraction of pool docs used to estimate the background", "-ALPHA = 1.0             # additive smoothing", "-N_OUT = 40_000          # ids to emit (>> enough to cover 12M tokens)", "+N_UNI = 60_000", "+N_BI = 60_000", "+BG_SAMPLE_FRAC = 0.40", "+ALPHA = 1.0", "+N_OUT = 40_000", " SEED = 0", " ", "-# hard quality filters (register-agnostic: they only drop degenerate docs)", "-MIN_WORDS = 40          # too short to teach anything, wastes an EOS slot", "-MIN_ALPHA_RATIO = 0.55  # fraction of chars that are letters/space/basic punct", "-MAX_TOP_WORD_FRAC = 0.30  # single word dominating -> spam/repetition", "+MIN_WORDS = 50            # too short to teach anything", "+MIN_ASCII_RATIO = 0.90    # guard against binary / non-English gibberish", "+MAX_TOP_WORD_FRAC = 0.28  # a single word dominating -> spam / repetition", " ", " random.seed(SEED)", "-WORD_RE = re.compile(r\"[a-z0-9]+\")", "+# HTML tags, HTML entities, and alnum words are all tokens (keeps the Q&A/code register)", "+TOK_RE = re.compile(r\"</?[a-z][a-z0-9]*/?>|&[a-z]+;|[a-z0-9]+\")", "+QA_MARK = (\"<p>\", \"<code>\", \"<pre>\", \"</\", \"&quot;\", \"&lt;\", \"&gt;\", \"&amp;\",", "+           \"def \", \"import \", \"function \", \"public static\", \"#include\")", " ", " ", "-def words(text):", "-    return WORD_RE.findall(text.lower())", "+def toks(text):", "+    return TOK_RE.findall(text.lower())", " ", " ", "-def passes_filters(text):", "+def is_qa(text):", "+    return any(m in text for m in QA_MARK)", "+", "+", "+def passes(text):", "     if len(text) < 200:", "         return False, None", "-    w = words(text)", "+    w = toks(text)", "     if len(w) < MIN_WORDS:", "         return False, None", "-    # alpha ratio on raw text", "-    good = sum(c.isalpha() or c.isspace() or c in \",.;:'\\\"?!()-\" for c in text)", "-    if good / max(1, len(text)) < MIN_ALPHA_RATIO:", "+    ascii_ok = sum(32 <= ord(c) < 127 or c in \"\\n\\t\\r\" for c in text)", "+    if ascii_ok / len(text) < MIN_ASCII_RATIO:", "         return False, None", "     c = Counter(w)", "     if c.most_common(1)[0][1] / len(w) > MAX_TOP_WORD_FRAC:"]}, {"oldStart": 78, "oldLines": 26, "newStart": 90, "newLines": 23, "lines": ["     from transformers import AutoTokenizer", "     tok = AutoTokenizer.from_pretrained(\"gpt2\")", "     tgt_tokens = np.load(TARGET_NPY)", "-    # decode in chunks to bound memory", "-    tgt_text_parts = []", "-    step = 4096", "-    for s in range(0, len(tgt_tokens), step):", "-        tgt_text_parts.append(tok.decode(tgt_tokens[s:s+step].tolist()))", "-    tgt_text = \" \".join(tgt_text_parts)", "-    tw = words(tgt_text)", "+    parts = []", "+    for s in range(0, len(tgt_tokens), 4096):", "+        parts.append(tok.decode(tgt_tokens[s:s+4096].tolist()))", "+    tw = toks(\" \".join(parts))", "     tgt_uni = Counter(tw)", "     tgt_bi = Counter(a + \" \" + b for a, b in zip(tw, tw[1:]))", "-    print(f\"  target: {len(tw)} words, {len(tgt_uni)} uniq unigrams, \"", "-          f\"{len(tgt_bi)} uniq bigrams  ({time.time()-t0:.0f}s)\", flush=True)", "+    print(f\"  target: {len(tw)} tokens, {len(tgt_uni)} uni, {len(tgt_bi)} bi \"", "+          f\"({time.time()-t0:.0f}s)\", flush=True)", " ", "     vocab_uni = {w: i for i, (w, _) in enumerate(", "         [x for x in tgt_uni.most_common(N_UNI) if x[1] >= 2])}", "     vocab_bi = {w: i for i, (w, _) in enumerate(", "         [x for x in tgt_bi.most_common(N_BI) if x[1] >= 2])}", "     VU, VB = len(vocab_uni), len(vocab_bi)", "-    print(f\"  vocab: {VU} unigrams, {VB} bigrams kept\", flush=True)", "+    print(f\"  vocab kept: {VU} uni, {VB} bi\", flush=True)", " ", "-    tgt_uc = np.full(VU, 0.0); tgt_bc = np.full(VB, 0.0)", "+    tgt_uc = np.zeros(VU); tgt_bc = np.zeros(VB)", "     for w, i in vocab_uni.items():", "         tgt_uc[i] = tgt_uni[w]", "     for w, i in vocab_bi.items():"]}, {"oldStart": 107, "oldLines": 7, "newStart": 116, "newLines": 7, "lines": ["     bg_uc = np.zeros(VU); bg_bc = np.zeros(VB)", "     bg_idx = [j for j in range(N) if random.random() < BG_SAMPLE_FRAC]", "     for j in bg_idx:", "-        w = words(texts[j])", "+        w = toks(texts[j])", "         for x in w:", "             i = vocab_uni.get(x)", "             if i is not None:"]}, {"oldStart": 116, "oldLines": 9, "newStart": 125, "newLines": 8, "lines": ["             i = vocab_bi.get(a + \" \" + b)", "             if i is not None:", "                 bg_bc[i] += 1", "-    print(f\"  background from {len(bg_idx)} docs  ({time.time()-t0:.0f}s)\", flush=True)", "+    print(f\"  background from {len(bg_idx)} docs ({time.time()-t0:.0f}s)\", flush=True)", " ", "-    # ---- per-feature log-likelihood-ratio weights ----", "     Tu, Bu = tgt_uc.sum(), bg_uc.sum()", "     Tb, Bb = tgt_bc.sum(), bg_bc.sum()", "     wu = np.log((tgt_uc + ALPHA) / (Tu + ALPHA * VU)) - \\"]}, {"oldStart": 126, "oldLines": 38, "newStart": 134, "newLines": 47, "lines": ["     wb = np.log((tgt_bc + ALPHA) / (Tb + ALPHA * VB)) - \\", "          np.log((bg_bc + ALPHA) / (Bb + ALPHA * VB))", " ", "-    # ---- score every doc ----", "+    # ---- score every doc; track register + dedup ----", "     scores = np.full(N, -1e9)", "+    qa_flag = np.zeros(N, dtype=bool)", "+    seen = set()", "     kept = 0", "     for j in range(N):", "-        ok, w = passes_filters(texts[j])", "+        h = hash(texts[j])", "+        if h in seen:", "+            continue          # exact-duplicate text", "+        ok, w = passes(texts[j])", "         if not ok:", "             continue", "+        seen.add(h)", "         kept += 1", "+        qa_flag[j] = is_qa(texts[j])", "         su = 0.0", "         for x in w:", "             i = vocab_uni.get(x)", "             if i is not None:", "                 su += wu[i]", "         sb = 0.0", "-        nb = 0", "         for a, b in zip(w, w[1:]):", "             i = vocab_bi.get(a + \" \" + b)", "             if i is not None:", "                 sb += wb[i]", "-                nb += 1", "-        # mean log-ratio per token; bigrams get half weight of unigrams", "         scores[j] = (su + 0.5 * sb) / len(w)", "-    print(f\"  scored; {kept} docs pass filters  ({time.time()-t0:.0f}s)\", flush=True)", "+    print(f\"  scored; {kept} unique docs pass ({time.time()-t0:.0f}s)\", flush=True)", " ", "+    valid = scores > -1e8", "     order = np.argsort(-scores)", "-    sel = [int(ids[j]) for j in order[:N_OUT] if scores[j] > -1e8]", "+    order = [j for j in order if valid[j]]", "+    # surface all genuine Q&A/code docs (best-first) ahead of the prose tail", "+    qa_order = [j for j in order if qa_flag[j]]", "+    prose_order = [j for j in order if not qa_flag[j]]", "+    final = qa_order + prose_order", "+    sel = [int(ids[j]) for j in final[:N_OUT]]", "     json.dump(sel, open(OUT, \"w\"))", "-    print(f\"wrote {len(sel)} ids -> {OUT}  ({time.time()-t0:.0f}s)\", flush=True)", "-    # diagnostics", "-    top = order[:20]", "-    print(\"score quantiles (kept):\",", "-          np.round(np.quantile(scores[scores > -1e8], [0, .25, .5, .75, .95, 1]), 3))", "+    print(f\"  qa docs surfaced: {len(qa_order)}\", flush=True)", "+    print(f\"wrote {len(sel)} ids -> {OUT} ({time.time()-t0:.0f}s)\", flush=True)", "+    sv = scores[valid]", "+    print(\"score quantiles:\", np.round(np.quantile(sv, [0, .5, .9, 1]), 3))", " ", " ", " if __name__ == \"__main__\":"]}], "originalFile": "#!/usr/bin/env python3\n\"\"\"Curate the raw web pool for training a small LM whose held-out target is a\nBROAD, multi-domain high-quality English distribution (Wikipedia + high-quality\nweb prose + news + technical Q&A), disclosed via /workspace/data/multi_dev.npy.\n\nCriterion (stated, reproducible): DSIR-style importance scoring.\n  * Decode the disclosed target token array back to text -> a sample of the\n    target domain.\n  * Fit a unigram+bigram bag-of-words distribution for the TARGET and for a\n    random BACKGROUND sample of the raw pool.\n  * Score every pool document by its mean per-token log-likelihood ratio\n    log p_target(w) - log p_background(w).  Documents whose vocabulary looks\n    like the clean multi-domain target score high; boilerplate/forum/spam\n    (whose n-grams are common in the raw pool but rare in the target) score low.\n  * Apply light hard filters (length floor, alpha-ratio, repetition cap) to\n    drop degenerate documents that no domain wants.\n  * Emit pool ids in descending score order (best first).  The training\n    pipeline consumes this priority list until the 12M-token budget is full.\n\nThis is a pure ranking function over a stated feature, not a hand-picked list.\n\"\"\"\nimport json, re, math, time, random\nimport numpy as np\nfrom collections import Counter\n\nPOOL = \"/workspace/data/pool.jsonl\"\nTARGET_NPY = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\n\nN_UNI = 60_000          # top target unigrams kept in the model vocabulary\nN_BI  = 60_000          # top target bigrams kept\nBG_SAMPLE_FRAC = 0.40   # fraction of pool docs used to estimate the background\nALPHA = 1.0             # additive smoothing\nN_OUT = 40_000          # ids to emit (>> enough to cover 12M tokens)\nSEED = 0\n\n# hard quality filters (register-agnostic: they only drop degenerate docs)\nMIN_WORDS = 40          # too short to teach anything, wastes an EOS slot\nMIN_ALPHA_RATIO = 0.55  # fraction of chars that are letters/space/basic punct\nMAX_TOP_WORD_FRAC = 0.30  # single word dominating -> spam/repetition\n\nrandom.seed(SEED)\nWORD_RE = re.compile(r\"[a-z0-9]+\")\n\n\ndef words(text):\n    return WORD_RE.findall(text.lower())\n\n\ndef passes_filters(text):\n    if len(text) < 200:\n        return False, None\n    w = words(text)\n    if len(w) < MIN_WORDS:\n        return False, None\n    # alpha ratio on raw text\n    good = sum(c.isalpha() or c.isspace() or c in \",.;:'\\\"?!()-\" for c in text)\n    if good / max(1, len(text)) < MIN_ALPHA_RATIO:\n        return False, None\n    c = Counter(w)\n    if c.most_common(1)[0][1] / len(w) > MAX_TOP_WORD_FRAC:\n        return False, None\n    return True, w\n\n\ndef main():\n    t0 = time.time()\n    print(\"loading pool...\", flush=True)\n    ids, texts = [], []\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            ids.append(r[\"id\"]); texts.append(r[\"text\"])\n    N = len(ids)\n    print(f\"  {N} docs loaded in {time.time()-t0:.0f}s\", flush=True)\n\n    # ---- target distribution ----\n    from transformers import AutoTokenizer\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    tgt_tokens = np.load(TARGET_NPY)\n    # decode in chunks to bound memory\n    tgt_text_parts = []\n    step = 4096\n    for s in range(0, len(tgt_tokens), step):\n        tgt_text_parts.append(tok.decode(tgt_tokens[s:s+step].tolist()))\n    tgt_text = \" \".join(tgt_text_parts)\n    tw = words(tgt_text)\n    tgt_uni = Counter(tw)\n    tgt_bi = Counter(a + \" \" + b for a, b in zip(tw, tw[1:]))\n    print(f\"  target: {len(tw)} words, {len(tgt_uni)} uniq unigrams, \"\n          f\"{len(tgt_bi)} uniq bigrams  ({time.time()-t0:.0f}s)\", flush=True)\n\n    vocab_uni = {w: i for i, (w, _) in enumerate(\n        [x for x in tgt_uni.most_common(N_UNI) if x[1] >= 2])}\n    vocab_bi = {w: i for i, (w, _) in enumerate(\n        [x for x in tgt_bi.most_common(N_BI) if x[1] >= 2])}\n    VU, VB = len(vocab_uni), len(vocab_bi)\n    print(f\"  vocab: {VU} unigrams, {VB} bigrams kept\", flush=True)\n\n    tgt_uc = np.full(VU, 0.0); tgt_bc = np.full(VB, 0.0)\n    for w, i in vocab_uni.items():\n        tgt_uc[i] = tgt_uni[w]\n    for w, i in vocab_bi.items():\n        tgt_bc[i] = tgt_bi[w]\n\n    # ---- background distribution (random pool sample) ----\n    bg_uc = np.zeros(VU); bg_bc = np.zeros(VB)\n    bg_idx = [j for j in range(N) if random.random() < BG_SAMPLE_FRAC]\n    for j in bg_idx:\n        w = words(texts[j])\n        for x in w:\n            i = vocab_uni.get(x)\n            if i is not None:\n                bg_uc[i] += 1\n        for a, b in zip(w, w[1:]):\n            i = vocab_bi.get(a + \" \" + b)\n            if i is not None:\n                bg_bc[i] += 1\n    print(f\"  background from {len(bg_idx)} docs  ({time.time()-t0:.0f}s)\", flush=True)\n\n    # ---- per-feature log-likelihood-ratio weights ----\n    Tu, Bu = tgt_uc.sum(), bg_uc.sum()\n    Tb, Bb = tgt_bc.sum(), bg_bc.sum()\n    wu = np.log((tgt_uc + ALPHA) / (Tu + ALPHA * VU)) - \\\n         np.log((bg_uc + ALPHA) / (Bu + ALPHA * VU))\n    wb = np.log((tgt_bc + ALPHA) / (Tb + ALPHA * VB)) - \\\n         np.log((bg_bc + ALPHA) / (Bb + ALPHA * VB))\n\n    # ---- score every doc ----\n    scores = np.full(N, -1e9)\n    kept = 0\n    for j in range(N):\n        ok, w = passes_filters(texts[j])\n        if not ok:\n            continue\n        kept += 1\n        su = 0.0\n        for x in w:\n            i = vocab_uni.get(x)\n            if i is not None:\n                su += wu[i]\n        sb = 0.0\n        nb = 0\n        for a, b in zip(w, w[1:]):\n            i = vocab_bi.get(a + \" \" + b)\n            if i is not None:\n                sb += wb[i]\n                nb += 1\n        # mean log-ratio per token; bigrams get half weight of unigrams\n        scores[j] = (su + 0.5 * sb) / len(w)\n    print(f\"  scored; {kept} docs pass filters  ({time.time()-t0:.0f}s)\", flush=True)\n\n    order = np.argsort(-scores)\n    sel = [int(ids[j]) for j in order[:N_OUT] if scores[j] > -1e8]\n    json.dump(sel, open(OUT, \"w\"))\n    print(f\"wrote {len(sel)} ids -> {OUT}  ({time.time()-t0:.0f}s)\", flush=True)\n    # diagnostics\n    top = order[:20]\n    print(\"score quantiles (kept):\",\n          np.round(np.quantile(scores[scores > -1e8], [0, .25, .5, .75, .95, 1]), 3))\n\n\nif __name__ == \"__main__\":\n    main()\n", "userModified": false}

Command running in background with ID: bzk5jpbov. Output is being written to: /tmp/claude-0/-workspace/2eda86eb-7dd5-4994-9ab5-ba9a2e6aa267/tasks/bzk5jpbov.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bzk5jpbov", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

loading pool...
  182016 docs loaded in 6s
  target: 699682 tokens, 38561 uni, 326803 bi (10s)
  vocab kept: 22629 uni, 60000 bi

[stdout]
loading pool...
  182016 docs loaded in 6s
  target: 699682 tokens, 38561 uni, 326803 bi (10s)
  vocab kept: 22629 uni, 60000 bi
[metadata] {"noOutputExpected": false}

vocab kept: 22629 uni, 60000 bi
  background from 73259 docs (39s)
  scored; 166313 unique docs pass (145s)
  qa docs surfaced: 6580
wrote 40000 ids -> /workspace/submission/selection.json (145s)
score quantiles: [-1.552 -0.305 -0.061  1.096]

[stdout]
  vocab kept: 22629 uni, 60000 bi
  background from 73259 docs (39s)
  scored; 166313 unique docs pass (145s)
  qa docs surfaced: 6580
wrote 40000 ids -> /workspace/submission/selection.json (145s)
score quantiles: [-1.552 -0.305 -0.061  1.096]
[metadata] {"noOutputExpected": false}

qa docs in list: 6580 approx tokens: 21458474 (budget=12M)
=== 3 top qa-surfaced docs ===
[114704] 'Attendees | All Canada Games\nRegister Here\nAccommodations\nSelect Page\nRecruits Attending\n<br /><br />\nRecruits will receive a link after registration to get listed as an attendee. Recruits'

[163002] '\ngPowered byeMuseum<|endoftext|>Instantiate Java\nWay2Java\na blog for Java\nResponsive Menu\nHome\nTable Of Contents\nSearch\nInstantiate Java\nJava General\nInstantiate Java\nS. Nageswara Rao, Corporate Train'

[138845] ' Sciences Community & Beyond<|endoftext|>Wicket 1.4.10 released | Apache Wicket\nQuick Start\nDownload\nDocumentation\nSupport\nContribute\nCommunity\nApache\nWicket 1.4.10 released\n11 Aug 2010\nThis is the te'

=== sample qa docs from middle of qa block ===
[126664] ' mailing list<|endoftext|>Encrypt dvd 4 Ways to put Ubuntu 13 @ HenryLynch64的部落格 :: 痞客邦 ::\nHenryLynch64的部落格\n跳到主文\n歡迎光臨HenryLynch64在痞客邦的小天地\n部落格全站分類：不設分類\n相簿\n部落格\n留言\n名片\nApr 21 Fri 2017 12:37\nEncrypt dvd 4 '

[106891] ' their 1 year Anniversary Party approaching, Social Eatz is having a ‘Thank You Party’ to give back to their fans on April 24. The new menu and decor sets the tone for more serious food, but it is int'

[stdout]
qa docs in list: 6580 approx tokens: 21458474 (budget=12M)
=== 3 top qa-surfaced docs ===
[114704] 'Attendees | All Canada Games\nRegister Here\nAccommodations\nSelect Page\nRecruits Attending\n&lt;br /&gt;&lt;br /&gt;\nRecruits will receive a link after registration to get listed as an attendee. Recruits'

[163002] '\ngPowered byeMuseum<|endoftext|>Instantiate Java\nWay2Java\na blog for Java\nResponsive Menu\nHome\nTable Of Contents\nSearch\nInstantiate Java\nJava General\nInstantiate Java\nS. Nageswara Rao, Corporate Train'

[138845] ' Sciences Community & Beyond<|endoftext|>Wicket 1.4.10 released | Apache Wicket\nQuick Start\nDownload\nDocumentation\nSupport\nContribute\nCommunity\nApache\nWicket 1.4.10 released\n11 Aug 2010\nThis is the te'

=== sample qa docs from middle of qa block ===
[126664] ' mailing list<|endoftext|>Encrypt dvd 4 Ways to put Ubuntu 13 @ HenryLynch64的部落格 :: 痞客邦 ::\nHenryLynch64的部落格\n跳到主文\n歡迎光臨HenryLynch64在痞客邦的小天地\n部落格全站分類：不設分類\n相簿\n部落格\n留言\n名片\nApr 21 Fri 2017 12:37\nEncrypt dvd 4 '

[106891] ' their 1 year Anniversary Party approaching, Social Eatz is having a ‘Thank You Party’ to give back to their fans on April 24. The new menu and decor sets the tone for more serious food, but it is int'
[metadata] {"noOutputExpected": false}