claude-opus-4-8

5390
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 03:22 .
drwxr-xr-x 1 root root  160 Aug  4 03:22 ..
drwxr-xr-x 1 root root   60 Aug  4 03:22 .live
drwxr-xr-x 1 root root  100 Aug  4 03:22 .metrics
drwxr-xr-x 1 root root  140 Aug  4 03:22 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 03:22 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 03:22 ..

[stdout]
5390
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 03:22 .
drwxr-xr-x 1 root root  160 Aug  4 03:22 ..
drwxr-xr-x 1 root root   60 Aug  4 03:22 .live
drwxr-xr-x 1 root root  100 Aug  4 03:22 .metrics
drwxr-xr-x 1 root root  140 Aug  4 03:22 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 03:22 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 03:22 ..
[metadata] {"noOutputExpected": false}

=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")
=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")
=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"
=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done

[stdout]
=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")
=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")
=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"
=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done
[metadata] {"noOutputExpected": false}

name, memory.total [MiB], memory.used [MiB]
NVIDIA H100 80GB HBM3, 81559 MiB, 4 MiB
=== model.py ===
"""A small GPT (decoder-only Transformer) used as the distillation *student*.

Self-contained re-implementation in this repo's own idiom (no external GPT code).
GPT-2 byte-level BPE vocabulary (50257) so the student shares the teacher's
tokenizer. Standard pre-norm Transformer with weight-tied embeddings.
"""
from dataclasses import dataclass
import math
import torch
import torch.nn as nn
import torch.nn.functional as F


@dataclass
class GPTConfig:
    block_size: int = 256
    vocab_size: int = 50257
    n_layer: int = 6
    n_head: int = 6
    n_embd: int = 384
    dropout: float = 0.0
    bias: bool = False


class LayerNorm(nn.Module):
    def __init__(self, ndim, bias):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(ndim))
        self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None

    def forward(self, x):
        return F.layer_norm(x, self.weight.shape, self.weight, self.bias, 1e-5)


class CausalSelfAttention(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
        assert cfg.n_embd % cfg.n_head == 0
        self.c_attn = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=cfg.bias)
        self.c_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias)
        self.n_head = cfg.n_head
        self.n_embd = cfg.n_embd
        self.dropout = cfg.dropout

    def forward(self, x):
        B, T, C = x.size()
        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        y = F.scaled_dot_product_attention(
            q, k, v, dropout_p=self.dropout if self.training else 0.0, is_causal=True
        )
        y = y.transpose(1, 2).contiguous().view(B, T, C)
        return self.c_proj(y)


class MLP(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()

[stdout]
name, memory.total [MiB], memory.used [MiB]
NVIDIA H100 80GB HBM3, 81559 MiB, 4 MiB
=== model.py ===
"""A small GPT (decoder-only Transformer) used as the distillation *student*.

Self-contained re-implementation in this repo's own idiom (no external GPT code).
GPT-2 byte-level BPE vocabulary (50257) so the student shares the teacher's
tokenizer. Standard pre-norm Transformer with weight-tied embeddings.
"""
from dataclasses import dataclass
import math
import torch
import torch.nn as nn
import torch.nn.functional as F


@dataclass
class GPTConfig:
    block_size: int = 256
    vocab_size: int = 50257
    n_layer: int = 6
    n_head: int = 6
    n_embd: int = 384
    dropout: float = 0.0
    bias: bool = False


class LayerNorm(nn.Module):
    def __init__(self, ndim, bias):
        super().__init__()
        self.weight = nn.Parameter(torch.ones(ndim))
        self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None

    def forward(self, x):
        return F.layer_norm(x, self.weight.shape, self.weight, self.bias, 1e-5)


class CausalSelfAttention(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
        assert cfg.n_embd % cfg.n_head == 0
        self.c_attn = nn.Linear(cfg.n_embd, 3 * cfg.n_embd, bias=cfg.bias)
        self.c_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias)
        self.n_head = cfg.n_head
        self.n_embd = cfg.n_embd
        self.dropout = cfg.dropout

    def forward(self, x):
        B, T, C = x.size()
        q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2)
        y = F.scaled_dot_product_attention(
            q, k, v, dropout_p=self.dropout if self.training else 0.0, is_causal=True
        )
        y = y.transpose(1, 2).contiguous().view(B, T, C)
        return self.c_proj(y)


class MLP(nn.Module):
    def __init__(self, cfg: GPTConfig):
        super().__init__()
[metadata] {"noOutputExpected": false}

multi_dev shape (1000000,) dtype uint16 total tokens 1000000
EOS id 50256 num EOS in dev 2704
num docs approx 2704
--- DEV DOC 0 (206 tok) ---
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , includin

--- DEV DOC 1 (125 tok) ---
 On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land 

--- DEV DOC 2 (223 tok) ---
 To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were uns

--- DEV DOC 3 (95 tok) ---
 Nimrod arrived at McMurdo Sound on 29 January , but was stopped by ice 16 miles ( 26 km ) north of Discovery 's old base at Hut Point . After considerable weather delays , Shackleton 's base was eventually established at Cape Royds , about 24 miles ( 39 km ) north of Hut Point . The party was in high spirits , despite the difficult conditions ; Shackleton 's ability to communicate with each man kept the party happy and focused .  

--- DEV DOC 4 (201 tok) ---
 The " Great Southern Journey " , as Frank Wild called it , began on 29 October 1908 . On 9 January 1909 , Shackleton and three companions ( Wild , Eric Marshall and Jameson Adams ) reached a new Farthest South latitude of 88 ° 23 ' S , a point only 112 miles ( 180 km ) from the Pole . En route the South Pole party discovered the Beardmore Glacier ( named after Shackleton 's patron ) and became the first persons to see and travel on the South Polar Plateau . Their return journey to McMurdo Sound was a race against starvation , on half @-@ rations for much of the way . At one point , Shackleton

--- DEV DOC 5 (131 tok) ---
 The expedition 's other main accomplishments included the first ascent of Mount Erebus , and the discovery of the approximate location of the South Magnetic Pole , reached on 16 January 1909 by Edgeworth David , Douglas Mawson , and Alistair Mackay . Shackleton returned to the United Kingdom as a hero , and soon afterwards published his expedition account , Heart of the Antarctic . Emily Shackleton later recorded : " The only comment he made to me about not reaching the Pole was ' a live donkey is better than a dead lion , isn 't it ? ' and I said ' Yes darling , as far as I am concerned ' "

[stdout]
multi_dev shape (1000000,) dtype uint16 total tokens 1000000
EOS id 50256 num EOS in dev 2704
num docs approx 2704
--- DEV DOC 0 (206 tok) ---
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , includin

--- DEV DOC 1 (125 tok) ---
 On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land 

--- DEV DOC 2 (223 tok) ---
 To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were uns

--- DEV DOC 3 (95 tok) ---
 Nimrod arrived at McMurdo Sound on 29 January , but was stopped by ice 16 miles ( 26 km ) north of Discovery 's old base at Hut Point . After considerable weather delays , Shackleton 's base was eventually established at Cape Royds , about 24 miles ( 39 km ) north of Hut Point . The party was in high spirits , despite the difficult conditions ; Shackleton 's ability to communicate with each man kept the party happy and focused .  

--- DEV DOC 4 (201 tok) ---
 The " Great Southern Journey " , as Frank Wild called it , began on 29 October 1908 . On 9 January 1909 , Shackleton and three companions ( Wild , Eric Marshall and Jameson Adams ) reached a new Farthest South latitude of 88 ° 23 ' S , a point only 112 miles ( 180 km ) from the Pole . En route the South Pole party discovered the Beardmore Glacier ( named after Shackleton 's patron ) and became the first persons to see and travel on the South Polar Plateau . Their return journey to McMurdo Sound was a race against starvation , on half @-@ rations for much of the way . At one point , Shackleton

--- DEV DOC 5 (131 tok) ---
 The expedition 's other main accomplishments included the first ascent of Mount Erebus , and the discovery of the approximate location of the South Magnetic Pole , reached on 16 January 1909 by Edgeworth David , Douglas Mawson , and Alistair Mackay . Shackleton returned to the United Kingdom as a hero , and soon afterwards published his expedition account , Heart of the Antarctic . Emily Shackleton later recorded : " The only comment he made to me about not reaching the Pole was ' a live donkey is better than a dead lion , isn 't it ? ' and I said ' Yes darling , as far as I am concerned ' "
[metadata] {"noOutputExpected": false}

--- DEV DOC 50 (89 tok) ---
 In addition to Barker and the regular cast , actor Jeff Bergman , voice actor Chris Cox , actor Michael Clarke Duncan , actor Keir Gilchrist , actress Beth Littleford and voice actress Rachael MacFarlane guest starred in the episode . Recurring guest voice actors Chris Sheridan , writer Danny Smith , writer Alec Sulkin and writer John Viener made minor appearances . Actor Patrick Warburton also has a guest appearance as well .  

--- DEV DOC 300 (73 tok) ---
 " Mystery Date " received steady viewership that was consistent with the ratings for the previous week . It received 2 @.@ 8 million viewers , down only from 2 @.@ 9 from " Tea Leaves " . The episode also received a 1 @.@ 0 rating in the important 18 @-@ 49 demographic , the same rating as the week before .  

--- DEV DOC 600 (159 tok) ---
 Stricklett attended Santa Clara University , where he played college baseball for the Santa Clara Broncos baseball team . He began his professional career in minor league baseball with the Topeka Colts of the Kansas State League in 1897 . In 1898 , he pitched for the Salina Blues and Atchison Huskers of the Kansas State League , before joining the Dallas Colts of the Class @-@ C Texas League later that year . He pitched for the Rock Island – Moline Islanders of the Class @-@ B Western Associati

--- DEV DOC 1000 (107 tok) ---
 In April 2006 , a team of astronomers , believing that Oval BA might converge with the GRS that year , observed the storms through the Hubble Space Telescope . The storms pass each other about every two years , but the passings of 2002 and 2004 did not produce anything exciting . Dr. Amy Simon @-@ Miller , of the Goddard Space Flight Center , predicted the storms would have their closest passing on July 4 , 2006 . On July 20 , the two storms were photographed passing each other by the Gemini Ob

--- DEV DOC 1500 (111 tok) ---
 In his 2002 State of the Union Address , Bush referred to an axis of evil including Iraq , Iran and North Korea . After the September 11 attacks on New York , Bush launched the War on Terror , in which the United States military and a small international coalition invaded Afghanistan . In 2003 , Bush then launched the invasion of Iraq , searching for Weapons of Mass Destruction , which he described as being part of the War on Terrorism . Those invasions led to the toppling of the Taliban regime

--- DEV DOC 2000 (151 tok) ---
Socialite Paris Hilton looked happy during her holiday in Formentera with her boyfriend Chris Zylka as they enjoyed a boating session together.Hilton was spotted in a lacy red dress as she held hands with the actor as they crossed the beach and took a ride in a small boat on Friday, reports dailymail.co.uk.Hilton accessorised her ensemble with a massive red-rimmed pair of sunglasses, as well as with a black bag.She was seen strolling around while resting her hand on Zylka's shoulders.Credit: @ P

--- DEV DOC 2500 (289 tok) ---
<p>in C# winforms when we display a message box it has no title in the title bar and no title in its button that is in the task bar. </p>  <p>What if i want to set title and icon for a message box.</p>  <p>one option is that create a form that appears and behaves like a message box and i show and hide it when i want. yes that can be done but i want to modify the "MessageBox"</p>  <p>Use a MessageBox.Show overload such as:</p>  <pre><code>public static DialogResult Show(     string text,     stri

--- DEV DOC 2650 (581 tok) ---
<p>We're running Java Swing and are using an SWT bridge to get ahold of the SWT browser. Everything normally works just fine, but we have a problem on a Vista test machine. It worked fine until just a week ago or so. The call stack:</p>  <pre><code>org.eclipse.swt.SWTError: No more handles     at org.eclipse.swt.SWT.error(SWT.java:3589)     at org.eclipse.swt.SWT.error(SWT.java:3481)     at org.eclipse.swt.SWT.error(SWT.java:3452)     at org.eclipse.swt.browser.IE.create(IE.java:184)     at org.

[stdout]
--- DEV DOC 50 (89 tok) ---
 In addition to Barker and the regular cast , actor Jeff Bergman , voice actor Chris Cox , actor Michael Clarke Duncan , actor Keir Gilchrist , actress Beth Littleford and voice actress Rachael MacFarlane guest starred in the episode . Recurring guest voice actors Chris Sheridan , writer Danny Smith , writer Alec Sulkin and writer John Viener made minor appearances . Actor Patrick Warburton also has a guest appearance as well .  

--- DEV DOC 300 (73 tok) ---
 " Mystery Date " received steady viewership that was consistent with the ratings for the previous week . It received 2 @.@ 8 million viewers , down only from 2 @.@ 9 from " Tea Leaves " . The episode also received a 1 @.@ 0 rating in the important 18 @-@ 49 demographic , the same rating as the week before .  

--- DEV DOC 600 (159 tok) ---
 Stricklett attended Santa Clara University , where he played college baseball for the Santa Clara Broncos baseball team . He began his professional career in minor league baseball with the Topeka Colts of the Kansas State League in 1897 . In 1898 , he pitched for the Salina Blues and Atchison Huskers of the Kansas State League , before joining the Dallas Colts of the Class @-@ C Texas League later that year . He pitched for the Rock Island – Moline Islanders of the Class @-@ B Western Associati

--- DEV DOC 1000 (107 tok) ---
 In April 2006 , a team of astronomers , believing that Oval BA might converge with the GRS that year , observed the storms through the Hubble Space Telescope . The storms pass each other about every two years , but the passings of 2002 and 2004 did not produce anything exciting . Dr. Amy Simon @-@ Miller , of the Goddard Space Flight Center , predicted the storms would have their closest passing on July 4 , 2006 . On July 20 , the two storms were photographed passing each other by the Gemini Ob

--- DEV DOC 1500 (111 tok) ---
 In his 2002 State of the Union Address , Bush referred to an axis of evil including Iraq , Iran and North Korea . After the September 11 attacks on New York , Bush launched the War on Terror , in which the United States military and a small international coalition invaded Afghanistan . In 2003 , Bush then launched the invasion of Iraq , searching for Weapons of Mass Destruction , which he described as being part of the War on Terrorism . Those invasions led to the toppling of the Taliban regime

--- DEV DOC 2000 (151 tok) ---
Socialite Paris Hilton looked happy during her holiday in Formentera with her boyfriend Chris Zylka as they enjoyed a boating session together.Hilton was spotted in a lacy red dress as she held hands with the actor as they crossed the beach and took a ride in a small boat on Friday, reports dailymail.co.uk.Hilton accessorised her ensemble with a massive red-rimmed pair of sunglasses, as well as with a black bag.She was seen strolling around while resting her hand on Zylka's shoulders.Credit: @ P

--- DEV DOC 2500 (289 tok) ---
<p>in C# winforms when we display a message box it has no title in the title bar and no title in its button that is in the task bar. </p>  <p>What if i want to set title and icon for a message box.</p>  <p>one option is that create a form that appears and behaves like a message box and i show and hide it when i want. yes that can be done but i want to modify the "MessageBox"</p>  <p>Use a MessageBox.Show overload such as:</p>  <pre><code>public static DialogResult Show(     string text,     stri

--- DEV DOC 2650 (581 tok) ---
<p>We're running Java Swing and are using an SWT bridge to get ahold of the SWT browser. Everything normally works just fine, but we have a problem on a Vista test machine. It worked fine until just a week ago or so. The call stack:</p>  <pre><code>org.eclipse.swt.SWTError: No more handles     at org.eclipse.swt.SWT.error(SWT.java:3589)     at org.eclipse.swt.SWT.error(SWT.java:3481)     at org.eclipse.swt.SWT.error(SWT.java:3452)     at org.eclipse.swt.browser.IE.create(IE.java:184)     at org.
[metadata] {"noOutputExpected": false}

=== id 37248 | len 11080 chars ===
<|endoftext|>Prince Edward County South Shore Conservancy P. O. Box 147 October 7, 2014 Premier Kathleen Wynne Gord Miller Legislative Building Environmental Commissioner of Ontario Queen’s Park 1075 Bay St Toronto, ON Ste. 605 M7A 1A1 Toronto, ON Toronto, ON M5S 2B1 (Sent via e-mail. Hard copy to follow.) Dear Premier Wynne and Mr. Miller: Re: wpd

=== id 35937 | len 4610 chars ===
How to Use the Holidays to Instill Values in your Children For most of us, the holidays are a season filled with a lot of hustle and bustle. If we can conquer our shopping list, decorate the house, get the cards in the mail and still find time to don a reindeer sweater and sing a carol or two, we’ve found success. It’s also a great time to instill 

=== id 7517 | len 3255 chars ===
Parts of Arizona's main north-south thoroughfare were shut down Wednesday, stranding motorists hundreds of miles south of Flagstaff and the Grand Canyon as a storm dumped snow on higher elevations and rare rainfall in the low desert. "As far as I can see, it's tail lights," said Abel Gurrola, who was headed north on Interstate 17 with his wife and 

=== id 3844 | len 1664 chars ===
GRAND TERRACE: Lions Club has new plan for old keys 10:00 PM PST on Saturday, November 20, 2010 Got a set of keys that would make a janitor envious? Too many keys dangling from your ignition switch as you turn your car engine on? The Grand Terrace Lions Club might have a solution to help lighten your load. The Lions are collecting unused and unwant

=== id 1255 | len 754 chars ===
4oz caster sugar 4oz Self rising flour 1 tsp baking powder 1 tsp cinnamon powder Place all ingredients into a mixing bowl ( or i use my Kenwood Kitchen Chef) and mix for 2-3 minutes. Scoop (large scoop by Pampered Chef) or spoon into muffin cases. Bake for 15-18 Min’s at 180c, until done. Allow to cool. Now the frosting: 125g cream cheese40g Butter

=== id 17757 | len 743 chars ===
Hull's Old Town Walking Tour Take a virtual tour. The city of Kingston upon Hull has played a leading part in British commercial and political life for over seven hundred years. Its early history as a royal planned town can still be seen in the streets of the Old Town and its medieval wealth can be felt in the grandeur of its parish churches. It ha

=== id 30765 | len 910 chars ===
 custom homepage is a customizable page that appears at your district’s custom domain (http://mydistrict.ed.voicethread.com). You’ll have one homepage for your entire district and one homepage per school. This is a place to display exemplary work, invite public participation in your VoiceThreads, or instruct your users in how to access their accoun

=== id 16464 | len 2583 chars ===
What is the direction of energy transfer, and in what form is the energy transferred (work or heat)? 1. A person goes swimming in cold water. 2. A truck pulls a trailer. 3. The explosion of gasoline in an engine cylinder pushes the piston and increases the temperature of the engine. 2 Answers | Add Yours The best way to remember this is that energy

=== id 35581 | len 807 chars ===
Progress on the Spartan cavalry and peltast bases has moved on slowly, very slowly to be honest. Hopefully work will resume at a slightly better pace in the near future. A small purchase from Gripping Beast is moving towards its new home - just waiting on the postman arriving. They have become a rarity in the local area - but sightings have been co

=== id 1094 | len 1019 chars ===
On Thursday the 18th June the open grade girls travelled down to Dunkirk to compete in the interzones. Our first match was against Northcote Intermediate which was a vey easy game and we won 31-0. Our next game was against Wesley and we thought we were going to lose but we played hard and we won 28-0. After that match we had a bye so we started to 

=== id 19844 | len 2633 chars ===
Caleb Moore crashes during the snowmobile freestyle finals Thursday, Jan. 24, 2013, during the first day of the X Games Aspen 2013. The Competition runs through Sunday at Aspen's Buttermilk. / Christian Murdock, AP ASPEN, Colo. â?? Snowmobiler Caleb Moore faces a brain complication after a crash during the X Games on Thursday. The 25-year-old Texan

=== id 8559 | len 1120 chars ===
Charming studio, close to transportation, shopping and restaurants. Fully furnished 1 Bedroom apartment for rent! Comes with new appliances, ironing board, kitchen utilities, and much more! Head and hot water included in rent. Voice command TVs and Optimum WiFi. Close to 2, 3 and 4 train. Tenant pays own gas and electric. $100 discount with on time

=== id 18356 | len 6879 chars ===
Search the Health Library Get the facts on diseases, conditions, tests and procedures. I Want To... Find a Doctor I Want To... Find Research Faculty Enter the last name, specialty or keyword for your search below. Michael Vincent Boland, M.D., Ph.D. Director of Information Technology, Wilmer Eye Institute Associate Professor of Ophthalmology Expert

=== id 1894 | len 1502 chars ===
It’s that time of year again, the time where we showcase the most stunning weddings that have graced the pages of SMP throughout the year as picked by you, our lovely SMP readers. It’s our third annual edition (here’s 2010 and here’s 2011), and boy is this year’s just beyond. Like beyond, beyond. We are putting the finishing touches on all the pret

[stdout]
=== id 37248 | len 11080 chars ===
<|endoftext|>Prince Edward County South Shore Conservancy P. O. Box 147 October 7, 2014 Premier Kathleen Wynne Gord Miller Legislative Building Environmental Commissioner of Ontario Queen’s Park 1075 Bay St Toronto, ON Ste. 605 M7A 1A1 Toronto, ON Toronto, ON M5S 2B1 (Sent via e-mail. Hard copy to follow.) Dear Premier Wynne and Mr. Miller: Re: wpd

=== id 35937 | len 4610 chars ===
How to Use the Holidays to Instill Values in your Children For most of us, the holidays are a season filled with a lot of hustle and bustle. If we can conquer our shopping list, decorate the house, get the cards in the mail and still find time to don a reindeer sweater and sing a carol or two, we’ve found success. It’s also a great time to instill 

=== id 7517 | len 3255 chars ===
Parts of Arizona's main north-south thoroughfare were shut down Wednesday, stranding motorists hundreds of miles south of Flagstaff and the Grand Canyon as a storm dumped snow on higher elevations and rare rainfall in the low desert. "As far as I can see, it's tail lights," said Abel Gurrola, who was headed north on Interstate 17 with his wife and 

=== id 3844 | len 1664 chars ===
GRAND TERRACE: Lions Club has new plan for old keys 10:00 PM PST on Saturday, November 20, 2010 Got a set of keys that would make a janitor envious? Too many keys dangling from your ignition switch as you turn your car engine on? The Grand Terrace Lions Club might have a solution to help lighten your load. The Lions are collecting unused and unwant

=== id 1255 | len 754 chars ===
4oz caster sugar 4oz Self rising flour 1 tsp baking powder 1 tsp cinnamon powder Place all ingredients into a mixing bowl ( or i use my Kenwood Kitchen Chef) and mix for 2-3 minutes. Scoop (large scoop by Pampered Chef) or spoon into muffin cases. Bake for 15-18 Min’s at 180c, until done. Allow to cool. Now the frosting: 125g cream cheese40g Butter

=== id 17757 | len 743 chars ===
Hull's Old Town Walking Tour Take a virtual tour. The city of Kingston upon Hull has played a leading part in British commercial and political life for over seven hundred years. Its early history as a royal planned town can still be seen in the streets of the Old Town and its medieval wealth can be felt in the grandeur of its parish churches. It ha

=== id 30765 | len 910 chars ===
 custom homepage is a customizable page that appears at your district’s custom domain (http://mydistrict.ed.voicethread.com). You’ll have one homepage for your entire district and one homepage per school. This is a place to display exemplary work, invite public participation in your VoiceThreads, or instruct your users in how to access their accoun

=== id 16464 | len 2583 chars ===
What is the direction of energy transfer, and in what form is the energy transferred (work or heat)? 1. A person goes swimming in cold water. 2. A truck pulls a trailer. 3. The explosion of gasoline in an engine cylinder pushes the piston and increases the temperature of the engine. 2 Answers | Add Yours The best way to remember this is that energy

=== id 35581 | len 807 chars ===
Progress on the Spartan cavalry and peltast bases has moved on slowly, very slowly to be honest. Hopefully work will resume at a slightly better pace in the near future. A small purchase from Gripping Beast is moving towards its new home - just waiting on the postman arriving. They have become a rarity in the local area - but sightings have been co

=== id 1094 | len 1019 chars ===
On Thursday the 18th June the open grade girls travelled down to Dunkirk to compete in the interzones. Our first match was against Northcote Intermediate which was a vey easy game and we won 31-0. Our next game was against Wesley and we thought we were going to lose but we played hard and we won 28-0. After that match we had a bye so we started to 

=== id 19844 | len 2633 chars ===
Caleb Moore crashes during the snowmobile freestyle finals Thursday, Jan. 24, 2013, during the first day of the X Games Aspen 2013. The Competition runs through Sunday at Aspen's Buttermilk. / Christian Murdock, AP ASPEN, Colo. â?? Snowmobiler Caleb Moore faces a brain complication after a crash during the X Games on Thursday. The 25-year-old Texan

=== id 8559 | len 1120 chars ===
Charming studio, close to transportation, shopping and restaurants. Fully furnished 1 Bedroom apartment for rent! Comes with new appliances, ironing board, kitchen utilities, and much more! Head and hot water included in rent. Voice command TVs and Optimum WiFi. Close to 2, 3 and 4 train. Tenant pays own gas and electric. $100 discount with on time

=== id 18356 | len 6879 chars ===
Search the Health Library Get the facts on diseases, conditions, tests and procedures. I Want To... Find a Doctor I Want To... Find Research Faculty Enter the last name, specialty or keyword for your search below. Michael Vincent Boland, M.D., Ph.D. Director of Information Technology, Wilmer Eye Institute Associate Professor of Ophthalmology Expert

=== id 1894 | len 1502 chars ===
It’s that time of year again, the time where we showcase the most stunning weddings that have graced the pages of SMP throughout the year as picked by you, our lovely SMP readers. It’s our third annual edition (here’s 2010 and here’s 2011), and boy is this year’s just beyond. Like beyond, beyond. We are putting the finishing touches on all the pret
[metadata] {"noOutputExpected": false}

total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
---dev register proportions---
wiki       tokens=  163490 ( 16.4%)  docs=966
qa         tokens=  249880 ( 25.1%)  docs=359
newsweb    tokens=  583926 ( 58.6%)  docs=1380
total tokens 997296

[stdout]
               total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
---dev register proportions---
wiki       tokens=  163490 ( 16.4%)  docs=966
qa         tokens=  249880 ( 25.1%)  docs=359
newsweb    tokens=  583926 ( 58.6%)  docs=1380
total tokens 997296
[metadata] {"noOutputExpected": false}

Task #1 created successfully: Write curate.py (classifier + balanced selection)

[metadata] {"task": {"id": "1", "subject": "Write curate.py (classifier + balanced selection)"}}

Task #2 created successfully: Run curate.py and dev_eval, iterate

[metadata] {"task": {"id": "2", "subject": "Run curate.py and dev_eval, iterate"}}

Task #3 created successfully: Write claim.md

[metadata] {"task": {"id": "3", "subject": "Write claim.md"}}

Updated task #1 status

[metadata] {"success": true, "taskId": "1", "updatedFields": ["status"], "statusChange": {"from": "pending", "to": "in_progress"}}

File created successfully at: /workspace/submission/curate.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/submission/curate.py", "content": "\"\"\"\nCurate a 12M-token pretraining selection from a raw web pool to minimize held-out\nperplexity on a BROAD high-quality English target (equal-parts encyclopedic /\nWikipedia, general web prose, news, technical Q&A).\n\nSTATED CRITERION\n----------------\nSelect the pool documents whose *distribution matches the disclosed target\ndomain*, register by register, then compose the final training set so its token\nmixture reproduces the target's own register mixture. Concretely:\n\n  1. The provided dev sample (data/multi_dev.npy) is a labelled draw from the\n     target domain. It cleanly separates into three registers by surface markers:\n       - `wiki`    : WikiText-style encyclopedic prose (contains @-@ / @,@ / @.@)\n       - `qa`      : StackOverflow-style technical Q&A (contains <p>/<code>/<pre>)\n       - `newsweb` : everything else = news + general high-quality web prose\n     Measured token mixture of the dev target: wiki~16%, qa~25%, newsweb~59%.\n\n  2. For each register we train a logistic-regression classifier\n     (hashed word 1-2 gram features) that separates that register's target text\n     from random pool text. Surface artifacts (@-@, HTML tags) are normalised\n     away before featurizing, so the classifier keys on *content vocabulary*\n     (encyclopedic style, code/question vocabulary, news prose) rather than on\n     markup the pool cannot contain. Each pool doc receives a target-likeness\n     score per register.\n\n  3. After a light quality prefilter (length / alphabetic-ratio / mojibake) and\n     exact-normalised de-duplication, we fill the priority list by\n     TOKEN-WEIGHTED ROUND ROBIN across the three registers, always extending the\n     register that is furthest behind its target token share. This front-loads a\n     clean, register-balanced set into the first 12M tokens (the budget the\n     trainer actually consumes) that mirrors the target distribution.\n\nThis is a reproducible, distribution-matching criterion (classifier + mixture\ncontrol), not a hand-picked id list. Deterministic given SEED.\n\"\"\"\nimport json, re, random, hashlib, time\nimport numpy as np\nimport scipy.sparse as sp\nfrom transformers import AutoTokenizer\nfrom sklearn.feature_extraction.text import HashingVectorizer\nfrom sklearn.linear_model import LogisticRegression\n\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED)\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\n# Target register token mixture, measured from the dev sample.\nPROP = {\"wiki\": 0.16, \"qa\": 0.25, \"newsweb\": 0.59}\nTARGET_TOKENS = 40_000_000          # produce ~3x the 12M budget as margin\nN_FEATURES    = 2 ** 19\nN_NEG         = 20000               # random pool negatives for the classifiers\n\n# ---------------------------------------------------------------- preprocessing\nTAG  = re.compile(r\"<[^>]+>\")            # strips HTML tags AND <|endoftext|>\nWART = re.compile(r\"\\s?@([-,.])@\\s?\")    # WikiText  X @-@ Y  ->  X-Y\nWS   = re.compile(r\"\\s+\")\ndef norm(t):\n    t = WART.sub(r\"\\1\", t)\n    t = TAG.sub(\" \", t)\n    t = t.lower()\n    return WS.sub(\" \", t)\n\ndef quality_ok(t):\n    L = len(t)\n    if L < 250 or L > 60000:            # drop tiny boilerplate & giant dumps\n        return False\n    head = t[:4000]\n    if sum(c.isalpha() or c.isspace() for c in head) < 0.70 * len(head):\n        return False                    # symbol/number soup, nav junk\n    if t.count(\"�\") > 3:           # mojibake / replacement chars\n        return False\n    return True\n\ndef dedup_key(t):\n    return hashlib.md5(norm(t)[:2000].encode(\"utf-8\", \"ignore\")).hexdigest()\n\n# ---------------------------------------------------------------- load pool\nt0 = time.time()\nids, texts = [], []\nwith open(POOL) as f:\n    for line in f:\n        r = json.loads(line)\n        ids.append(r[\"id\"]); texts.append(r[\"text\"])\nprint(f\"[load] {len(ids)} pool docs  {time.time()-t0:.0f}s\")\n\n# ---------------------------------------------------------------- dev registers\ntok = AutoTokenizer.from_pretrained(\"gpt2\"); EOS = tok.eos_token_id\nd = np.load(DEV)\ncut = np.where(d == EOS)[0]\nstarts = [0] + list(cut + 1); ends = list(cut) + [len(d)]\npos = {\"wiki\": [], \"qa\": [], \"newsweb\": []}\nfor s, e in zip(starts, ends):\n    seg = d[s:e]\n    if len(seg) < 5:\n        continue\n    tx = tok.decode(seg.tolist())\n    if (\"@-@\" in tx) or (\"@,@\" in tx) or (\"@.@\" in tx):\n        pos[\"wiki\"].append(tx)\n    elif (\"<p>\" in tx) or (\"<code>\" in tx) or (\"<pre>\" in tx) or (\"</p>\" in tx):\n        pos[\"qa\"].append(tx)\n    else:\n        pos[\"newsweb\"].append(tx)\nprint(\"[dev] pos docs:\", {k: len(v) for k, v in pos.items()})\n\n# ---------------------------------------------------------------- prefilter\nkeep = [i for i, t in enumerate(texts) if quality_ok(t)]\nprint(f\"[filter] {len(keep)}/{len(texts)} pass quality prefilter\")\n\n# ---------------------------------------------------------------- vectorize once\nvec = HashingVectorizer(preprocessor=norm, ngram_range=(1, 2),\n                        n_features=N_FEATURES, alternate_sign=False, norm=\"l2\")\nt0 = time.time()\nX_pool = vec.transform(texts)                       # 182k x 2^19 sparse\nprint(f\"[vec] pool vectorized {time.time()-t0:.0f}s  shape {X_pool.shape}\")\n\nneg_rows = random.sample(keep, min(N_NEG, len(keep)))\nX_neg = X_pool[neg_rows]\n\n# ---------------------------------------------------------------- per-register clf\nscores = {}\nfor reg in PROP:\n    X_pos = vec.transform(pos[reg])\n    Xtr = sp.vstack([X_pos, X_neg])\n    ytr = np.r_[np.ones(X_pos.shape[0]), np.zeros(X_neg.shape[0])]\n    clf = LogisticRegression(C=0.5, solver=\"liblinear\", max_iter=1000)\n    clf.fit(Xtr, ytr)\n    scores[reg] = clf.decision_function(X_pool)\n    print(f\"[clf] {reg:8s} trained on {X_pos.shape[0]} pos / {len(neg_rows)} neg\")\n\n# ---------------------------------------------------------------- ranked lists\norder = {reg: sorted(keep, key=lambda i: scores[reg][i], reverse=True) for reg in PROP}\n\n# ---------------------------------------------------------------- weighted round robin\nptr    = {r: 0 for r in PROP}\ntokens = {r: 0 for r in PROP}\nselection, chosen, seen = [], set(), set()\ntotal = 0\nwhile total < TARGET_TOKENS:\n    # register furthest behind its target token pace\n    reg = min(PROP, key=lambda r: (tokens[r] / PROP[r], -PROP[r]))\n    picked = None\n    while ptr[reg] < len(order[reg]):\n        i = order[reg][ptr[reg]]; ptr[reg] += 1\n        if ids[i] in chosen:\n            continue\n        k = dedup_key(texts[i])\n        if k in seen:\n            continue\n        seen.add(k); picked = i; break\n    if picked is None:\n        if all(ptr[r] >= len(order[r]) for r in PROP):\n            break\n        tokens[reg] = float(\"inf\")      # stop selecting this exhausted register\n        continue\n    est = max(1, len(texts[picked]) // 4)\n    selection.append(int(ids[picked])); chosen.add(ids[picked])\n    tokens[reg] += est; total += est\n\njson.dump(selection, open(OUT, \"w\"))\nmix = {r: round(tokens[r] / total, 3) for r in PROP if tokens[r] != float(\"inf\")}\nprint(f\"[out] {len(selection)} ids, ~{total/1e6:.1f}M est tokens, mix={mix} -> {OUT}\")\n", "structuredPatch": [], "originalFile": null, "userModified": false}

time left: 5000s
Traceback (most recent call last):
  File "/workspace/submission/curate.py", line 39, in <module>
    import scipy.sparse as sp
ModuleNotFoundError: No module named 'scipy'

real	0m0.242s
user	0m1.510s
sys	0m0.790s

[stdout]
time left: 5000s
Traceback (most recent call last):
  File "/workspace/submission/curate.py", line 39, in <module>
    import scipy.sparse as sp
ModuleNotFoundError: No module named 'scipy'

real	0m0.242s
user	0m1.510s
sys	0m0.790s
[metadata] {"noOutputExpected": false}

Command did not complete within its 120s timeout and was moved to the background (ID: blwexan02). Output is being written to: /tmp/claude-0/-workspace/010b8dbe-8f98-478c-a3eb-e2fc1f0f67e0/tasks/blwexan02.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "blwexan02", "timedOutAfterMs": 120000, "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

Traceback (most recent call last):
Traceback (most recent call last):
---checking core libs---
numpy 1.26.4 torch 2.8.0+cu128 cuda True

[stdout]
Traceback (most recent call last):
Traceback (most recent call last):
---checking core libs---
numpy 1.26.4 torch 2.8.0+cu128 cuda True
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"type": "update", "filePath": "/workspace/submission/curate.py", "content": "\"\"\"\nCurate a 12M-token pretraining selection from a raw web pool to minimize held-out\nperplexity on a BROAD high-quality English target (equal parts encyclopedic /\nWikipedia, general web prose, news, technical Q&A).\n\nSTATED CRITERION\n----------------\nSelect the pool documents whose *distribution matches the disclosed target\ndomain*, register by register, then compose the training set so its token mixture\nreproduces the target's own register mixture. Concretely:\n\n  1. The provided dev sample (data/multi_dev.npy) is a labelled draw from the\n     target domain. It separates into three registers by surface markers:\n       - `wiki`    : WikiText-style encyclopedic prose (contains @-@ / @,@ / @.@)\n       - `qa`      : StackOverflow-style technical Q&A (contains <p>/<code>/<pre>)\n       - `newsweb` : everything else = news + general high-quality web prose\n     Measured dev token mixture: wiki~16%, qa~25%, newsweb~59%.\n\n  2. For each register, train a logistic-regression classifier on hashed word\n     1-2 gram features (implemented as a torch EmbeddingBag over a hashed feature\n     space) that separates that register's target text from random pool text.\n     Surface artifacts (@-@, HTML tags) are normalised away before featurizing so\n     the classifier keys on *content vocabulary* (encyclopedic style, code /\n     question vocabulary, news prose), not on markup the pool cannot contain.\n     Every pool doc gets a target-likeness score per register.\n\n  3. After a light quality prefilter (length / alphabetic-ratio / mojibake) and\n     exact-normalised de-duplication, fill the priority list by TOKEN-WEIGHTED\n     ROUND ROBIN across registers, always extending the register furthest behind\n     its target token share. This front-loads a clean, register-balanced set into\n     the first 12M tokens (the budget the trainer consumes) that mirrors the\n     target distribution.\n\nReproducible distribution-matching criterion (classifier + mixture control),\ndeterministic given SEED — not a hand-picked id list.\n\"\"\"\nimport json, re, random, hashlib, time\nimport numpy as np\nimport torch, torch.nn as nn\n\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\nDEV_T = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nPROP          = {\"wiki\": 0.16, \"qa\": 0.25, \"newsweb\": 0.59}  # target token mixture\nTARGET_TOKENS = 40_000_000        # ~3x the 12M budget, as margin\nD             = 1 << 19           # hashed feature dimension (524288)\nMAXW          = 1500              # words/doc used for featurization\nN_NEG         = 20000             # random pool negatives\nSTEPS         = 400               # full-batch LR training steps per register\nWORD = re.compile(r\"[a-z0-9]+\")\n\n# multiplicative hash constants (uint64 wrap-around hashing)\nH1 = np.uint64(2654435761); H2 = np.uint64(2246822519); MASK = np.uint64(D - 1)\n\n# ---------------------------------------------------------------- preprocessing\nTAG  = re.compile(r\"<[^>]+>\")            # strips HTML tags AND <|endoftext|>\nWART = re.compile(r\"\\s?@([-,.])@\\s?\")    # WikiText  X @-@ Y  ->  X-Y\nWS   = re.compile(r\"\\s+\")\ndef norm(t):\n    t = WART.sub(r\"\\1\", t)\n    t = TAG.sub(\" \", t)\n    return WS.sub(\" \", t.lower())\n\ndef quality_ok(t):\n    L = len(t)\n    if L < 250 or L > 60000:\n        return False\n    head = t[:4000]\n    if sum(c.isalpha() or c.isspace() for c in head) < 0.70 * len(head):\n        return False\n    if t.count(\"�\") > 3:\n        return False\n    return True\n\ndef dedup_key(t):\n    return hashlib.md5(norm(t)[:2000].encode(\"utf-8\", \"ignore\")).hexdigest()\n\n# word -> stable int id (first-occurrence; permutation-invariant for the model)\nw2id = {}\ndef featurize(t):\n    \"\"\"normalised text -> sorted unique hashed 1-2gram feature ids (np.int64).\"\"\"\n    words = WORD.findall(norm(t))[:MAXW]\n    if not words:\n        return np.zeros(1, dtype=np.int64)\n    sd = w2id.setdefault\n    ids = np.fromiter((sd(w, len(w2id)) for w in words), dtype=np.uint64, count=len(words))\n    uni = (ids * H1) & MASK\n    if len(ids) > 1:\n        bi = ((ids[:-1] * H1) ^ (ids[1:] * H2)) & MASK\n        feat = np.concatenate([uni, bi])\n    else:\n        feat = uni\n    return np.unique(feat.astype(np.int64))\n\ndef pack(feat_list):\n    \"\"\"list of feature-id arrays -> (input, offsets) LongTensors on device.\"\"\"\n    offs = np.zeros(len(feat_list), dtype=np.int64)\n    tot = 0\n    for i, f in enumerate(feat_list):\n        offs[i] = tot; tot += len(f)\n    inp = np.concatenate(feat_list) if feat_list else np.zeros(0, np.int64)\n    return (torch.from_numpy(inp).to(DEV_T), torch.from_numpy(offs).to(DEV_T))\n\n# ---------------------------------------------------------------- load pool\nt0 = time.time()\nids, texts = [], []\nwith open(POOL) as f:\n    for line in f:\n        r = json.loads(line)\n        ids.append(r[\"id\"]); texts.append(r[\"text\"])\nprint(f\"[load] {len(ids)} pool docs  {time.time()-t0:.0f}s\")\n\n# ---------------------------------------------------------------- quality prefilter\nkeep = [i for i, t in enumerate(texts) if quality_ok(t)]\nprint(f\"[filter] {len(keep)}/{len(texts)} pass quality prefilter\")\n\n# ---------------------------------------------------------------- featurize pool\nt0 = time.time()\nkeep_feat = [featurize(texts[i]) for i in keep]\nprint(f\"[feat] pool featurized {time.time()-t0:.0f}s  |vocab|={len(w2id)}\")\npool_inp, pool_off = pack(keep_feat)\n\n# ---------------------------------------------------------------- dev registers\ntok_time = time.time()\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained(\"gpt2\"); EOS = tok.eos_token_id\nd = np.load(DEV)\ncut = np.where(d == EOS)[0]\nstarts = [0] + list(cut + 1); ends = list(cut) + [len(d)]\npos = {\"wiki\": [], \"qa\": [], \"newsweb\": []}\nfor s, e in zip(starts, ends):\n    seg = d[s:e]\n    if len(seg) < 5:\n        continue\n    tx = tok.decode(seg.tolist())\n    if (\"@-@\" in tx) or (\"@,@\" in tx) or (\"@.@\" in tx):\n        pos[\"wiki\"].append(featurize(tx))\n    elif (\"<p>\" in tx) or (\"<code>\" in tx) or (\"<pre>\" in tx) or (\"</p>\" in tx):\n        pos[\"qa\"].append(featurize(tx))\n    else:\n        pos[\"newsweb\"].append(featurize(tx))\nprint(\"[dev] pos docs:\", {k: len(v) for k, v in pos.items()}, f\"{time.time()-tok_time:.0f}s\")\n\n# ---------------------------------------------------------------- per-register LR\nneg_rows = random.sample(range(len(keep)), min(N_NEG, len(keep)))\nneg_inp, neg_off = pack([keep_feat[j] for j in neg_rows])\nneg_off_g = neg_off\n\ndef train_score(pos_feats):\n    p_inp, p_off = pack(pos_feats)\n    npos, nneg = len(pos_feats), len(neg_rows)\n    # concat pos+neg into one batch\n    inp = torch.cat([p_inp, neg_inp])\n    off = torch.cat([p_off, neg_off + len(p_inp)])\n    y = torch.cat([torch.ones(npos), torch.zeros(nneg)]).to(DEV_T)\n    emb = nn.EmbeddingBag(D, 1, mode=\"sum\").to(DEV_T)\n    emb.weight.data.zero_()\n    bias = torch.zeros(1, device=DEV_T, requires_grad=True)\n    opt = torch.optim.Adam(list(emb.parameters()) + [bias], lr=0.05, weight_decay=2e-6)\n    lossf = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(nneg / npos, device=DEV_T))\n    for _ in range(STEPS):\n        opt.zero_grad()\n        logit = emb(inp, off).squeeze(1) + bias\n        loss = lossf(logit, y)\n        loss.backward(); opt.step()\n    with torch.no_grad():\n        sc = (emb(pool_inp, pool_off).squeeze(1) + bias).cpu().numpy()\n    return sc\n\nscores = {}\nfor reg in PROP:\n    t0 = time.time()\n    scores[reg] = train_score(pos[reg])\n    print(f\"[clf] {reg:8s} trained ({len(pos[reg])} pos/{len(neg_rows)} neg) {time.time()-t0:.0f}s\")\n\n# ---------------------------------------------------------------- ranked lists (over kept docs, by kept-index j)\norder = {reg: sorted(range(len(keep)), key=lambda j: scores[reg][j], reverse=True) for reg in PROP}\n\n# ---------------------------------------------------------------- weighted round robin\nptr    = {r: 0 for r in PROP}\ntokens = {r: 0.0 for r in PROP}\nselection, chosen, seen = [], set(), set()\ntotal = 0\nwhile total < TARGET_TOKENS:\n    reg = min(PROP, key=lambda r: (tokens[r] / PROP[r], -PROP[r]))\n    picked = None\n    while ptr[reg] < len(order[reg]):\n        j = order[reg][ptr[reg]]; ptr[reg] += 1\n        gi = keep[j]\n        if ids[gi] in chosen:\n            continue\n        k = dedup_key(texts[gi])\n        if k in seen:\n            continue\n        seen.add(k); picked = gi; break\n    if picked is None:\n        if all(ptr[r] >= len(order[r]) for r in PROP):\n            break\n        tokens[reg] = float(\"inf\")\n        continue\n    est = max(1, len(texts[picked]) // 4)\n    selection.append(int(ids[picked])); chosen.add(ids[picked])\n    tokens[reg] += est; total += est\n\njson.dump(selection, open(OUT, \"w\"))\nmix = {r: round(tokens[r] / total, 3) for r in PROP if tokens[r] != float(\"inf\")}\nprint(f\"[out] {len(selection)} ids, ~{total/1e6:.1f}M est tokens, mix={mix} -> {OUT}\")\n", "structuredPatch": [{"oldStart": 1, "oldLines": 59, "newStart": 1, "newLines": 62, "lines": [" \"\"\"", " Curate a 12M-token pretraining selection from a raw web pool to minimize held-out", "-perplexity on a BROAD high-quality English target (equal-parts encyclopedic /", "+perplexity on a BROAD high-quality English target (equal parts encyclopedic /", " Wikipedia, general web prose, news, technical Q&A).", " ", " STATED CRITERION", " ----------------", " Select the pool documents whose *distribution matches the disclosed target", "-domain*, register by register, then compose the final training set so its token", "-mixture reproduces the target's own register mixture. Concretely:", "+domain*, register by register, then compose the training set so its token mixture", "+reproduces the target's own register mixture. Concretely:", " ", "   1. The provided dev sample (data/multi_dev.npy) is a labelled draw from the", "-     target domain. It cleanly separates into three registers by surface markers:", "+     target domain. It separates into three registers by surface markers:", "        - `wiki`    : WikiText-style encyclopedic prose (contains @-@ / @,@ / @.@)", "        - `qa`      : StackOverflow-style technical Q&A (contains <p>/<code>/<pre>)", "        - `newsweb` : everything else = news + general high-quality web prose", "-     Measured token mixture of the dev target: wiki~16%, qa~25%, newsweb~59%.", "+     Measured dev token mixture: wiki~16%, qa~25%, newsweb~59%.", " ", "-  2. For each register we train a logistic-regression classifier", "-     (hashed word 1-2 gram features) that separates that register's target text", "-     from random pool text. Surface artifacts (@-@, HTML tags) are normalised", "-     away before featurizing, so the classifier keys on *content vocabulary*", "-     (encyclopedic style, code/question vocabulary, news prose) rather than on", "-     markup the pool cannot contain. Each pool doc receives a target-likeness", "-     score per register.", "+  2. For each register, train a logistic-regression classifier on hashed word", "+     1-2 gram features (implemented as a torch EmbeddingBag over a hashed feature", "+     space) that separates that register's target text from random pool text.", "+     Surface artifacts (@-@, HTML tags) are normalised away before featurizing so", "+     the classifier keys on *content vocabulary* (encyclopedic style, code /", "+     question vocabulary, news prose), not on markup the pool cannot contain.", "+     Every pool doc gets a target-likeness score per register.", " ", "   3. After a light quality prefilter (length / alphabetic-ratio / mojibake) and", "-     exact-normalised de-duplication, we fill the priority list by", "-     TOKEN-WEIGHTED ROUND ROBIN across the three registers, always extending the", "-     register that is furthest behind its target token share. This front-loads a", "-     clean, register-balanced set into the first 12M tokens (the budget the", "-     trainer actually consumes) that mirrors the target distribution.", "+     exact-normalised de-duplication, fill the priority list by TOKEN-WEIGHTED", "+     ROUND ROBIN across registers, always extending the register furthest behind", "+     its target token share. This front-loads a clean, register-balanced set into", "+     the first 12M tokens (the budget the trainer consumes) that mirrors the", "+     target distribution.", " ", "-This is a reproducible, distribution-matching criterion (classifier + mixture", "-control), not a hand-picked id list. Deterministic given SEED.", "+Reproducible distribution-matching criterion (classifier + mixture control),", "+deterministic given SEED — not a hand-picked id list.", " \"\"\"", " import json, re, random, hashlib, time", " import numpy as np", "-import scipy.sparse as sp", "-from transformers import AutoTokenizer", "-from sklearn.feature_extraction.text import HashingVectorizer", "-from sklearn.linear_model import LogisticRegression", "+import torch, torch.nn as nn", " ", " SEED = 1337", "-random.seed(SEED); np.random.seed(SEED)", "+random.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)", "+DEV_T = \"cuda\" if torch.cuda.is_available() else \"cpu\"", " ", " POOL = \"/workspace/data/pool.jsonl\"", " DEV  = \"/workspace/data/multi_dev.npy\"", " OUT  = \"/workspace/submission/selection.json\"", " ", "-# Target register token mixture, measured from the dev sample.", "-PROP = {\"wiki\": 0.16, \"qa\": 0.25, \"newsweb\": 0.59}", "-TARGET_TOKENS = 40_000_000          # produce ~3x the 12M budget as margin", "-N_FEATURES    = 2 ** 19", "-N_NEG         = 20000               # random pool negatives for the classifiers", "+PROP          = {\"wiki\": 0.16, \"qa\": 0.25, \"newsweb\": 0.59}  # target token mixture", "+TARGET_TOKENS = 40_000_000        # ~3x the 12M budget, as margin", "+D             = 1 << 19           # hashed feature dimension (524288)", "+MAXW          = 1500              # words/doc used for featurization", "+N_NEG         = 20000             # random pool negatives", "+STEPS         = 400               # full-batch LR training steps per register", "+WORD = re.compile(r\"[a-z0-9]+\")", " ", "+# multiplicative hash constants (uint64 wrap-around hashing)", "+H1 = np.uint64(2654435761); H2 = np.uint64(2246822519); MASK = np.uint64(D - 1)", "+", " # ---------------------------------------------------------------- preprocessing", " TAG  = re.compile(r\"<[^>]+>\")            # strips HTML tags AND <|endoftext|>", " WART = re.compile(r\"\\s?@([-,.])@\\s?\")    # WikiText  X @-@ Y  ->  X-Y"]}, {"oldStart": 61, "oldLines": 23, "newStart": 64, "newLines": 48, "lines": [" def norm(t):", "     t = WART.sub(r\"\\1\", t)", "     t = TAG.sub(\" \", t)", "-    t = t.lower()", "-    return WS.sub(\" \", t)", "+    return WS.sub(\" \", t.lower())", " ", " def quality_ok(t):", "     L = len(t)", "-    if L < 250 or L > 60000:            # drop tiny boilerplate & giant dumps", "+    if L < 250 or L > 60000:", "         return False", "     head = t[:4000]", "     if sum(c.isalpha() or c.isspace() for c in head) < 0.70 * len(head):", "-        return False                    # symbol/number soup, nav junk", "-    if t.count(\"�\") > 3:           # mojibake / replacement chars", "         return False", "+    if t.count(\"�\") > 3:", "+        return False", "     return True", " ", " def dedup_key(t):", "     return hashlib.md5(norm(t)[:2000].encode(\"utf-8\", \"ignore\")).hexdigest()", " ", "+# word -> stable int id (first-occurrence; permutation-invariant for the model)", "+w2id = {}", "+def featurize(t):", "+    \"\"\"normalised text -> sorted unique hashed 1-2gram feature ids (np.int64).\"\"\"", "+    words = WORD.findall(norm(t))[:MAXW]", "+    if not words:", "+        return np.zeros(1, dtype=np.int64)", "+    sd = w2id.setdefault", "+    ids = np.fromiter((sd(w, len(w2id)) for w in words), dtype=np.uint64, count=len(words))", "+    uni = (ids * H1) & MASK", "+    if len(ids) > 1:", "+        bi = ((ids[:-1] * H1) ^ (ids[1:] * H2)) & MASK", "+        feat = np.concatenate([uni, bi])", "+    else:", "+        feat = uni", "+    return np.unique(feat.astype(np.int64))", "+", "+def pack(feat_list):", "+    \"\"\"list of feature-id arrays -> (input, offsets) LongTensors on device.\"\"\"", "+    offs = np.zeros(len(feat_list), dtype=np.int64)", "+    tot = 0", "+    for i, f in enumerate(feat_list):", "+        offs[i] = tot; tot += len(f)", "+    inp = np.concatenate(feat_list) if feat_list else np.zeros(0, np.int64)", "+    return (torch.from_numpy(inp).to(DEV_T), torch.from_numpy(offs).to(DEV_T))", "+", " # ---------------------------------------------------------------- load pool", " t0 = time.time()", " ids, texts = [], []"]}, {"oldStart": 87, "oldLines": 7, "newStart": 115, "newLines": 19, "lines": ["         ids.append(r[\"id\"]); texts.append(r[\"text\"])", " print(f\"[load] {len(ids)} pool docs  {time.time()-t0:.0f}s\")", " ", "+# ---------------------------------------------------------------- quality prefilter", "+keep = [i for i, t in enumerate(texts) if quality_ok(t)]", "+print(f\"[filter] {len(keep)}/{len(texts)} pass quality prefilter\")", "+", "+# ---------------------------------------------------------------- featurize pool", "+t0 = time.time()", "+keep_feat = [featurize(texts[i]) for i in keep]", "+print(f\"[feat] pool featurized {time.time()-t0:.0f}s  |vocab|={len(w2id)}\")", "+pool_inp, pool_off = pack(keep_feat)", "+", " # ---------------------------------------------------------------- dev registers", "+tok_time = time.time()", "+from transformers import AutoTokenizer", " tok = AutoTokenizer.from_pretrained(\"gpt2\"); EOS = tok.eos_token_id", " d = np.load(DEV)", " cut = np.where(d == EOS)[0]"]}, {"oldStart": 99, "oldLines": 62, "newStart": 139, "newLines": 69, "lines": ["         continue", "     tx = tok.decode(seg.tolist())", "     if (\"@-@\" in tx) or (\"@,@\" in tx) or (\"@.@\" in tx):", "-        pos[\"wiki\"].append(tx)", "+        pos[\"wiki\"].append(featurize(tx))", "     elif (\"<p>\" in tx) or (\"<code>\" in tx) or (\"<pre>\" in tx) or (\"</p>\" in tx):", "-        pos[\"qa\"].append(tx)", "+        pos[\"qa\"].append(featurize(tx))", "     else:", "-        pos[\"newsweb\"].append(tx)", "-print(\"[dev] pos docs:\", {k: len(v) for k, v in pos.items()})", "+        pos[\"newsweb\"].append(featurize(tx))", "+print(\"[dev] pos docs:\", {k: len(v) for k, v in pos.items()}, f\"{time.time()-tok_time:.0f}s\")", " ", "-# ---------------------------------------------------------------- prefilter", "-keep = [i for i, t in enumerate(texts) if quality_ok(t)]", "-print(f\"[filter] {len(keep)}/{len(texts)} pass quality prefilter\")", "+# ---------------------------------------------------------------- per-register LR", "+neg_rows = random.sample(range(len(keep)), min(N_NEG, len(keep)))", "+neg_inp, neg_off = pack([keep_feat[j] for j in neg_rows])", "+neg_off_g = neg_off", " ", "-# ---------------------------------------------------------------- vectorize once", "-vec = HashingVectorizer(preprocessor=norm, ngram_range=(1, 2),", "-                        n_features=N_FEATURES, alternate_sign=False, norm=\"l2\")", "-t0 = time.time()", "-X_pool = vec.transform(texts)                       # 182k x 2^19 sparse", "-print(f\"[vec] pool vectorized {time.time()-t0:.0f}s  shape {X_pool.shape}\")", "+def train_score(pos_feats):", "+    p_inp, p_off = pack(pos_feats)", "+    npos, nneg = len(pos_feats), len(neg_rows)", "+    # concat pos+neg into one batch", "+    inp = torch.cat([p_inp, neg_inp])", "+    off = torch.cat([p_off, neg_off + len(p_inp)])", "+    y = torch.cat([torch.ones(npos), torch.zeros(nneg)]).to(DEV_T)", "+    emb = nn.EmbeddingBag(D, 1, mode=\"sum\").to(DEV_T)", "+    emb.weight.data.zero_()", "+    bias = torch.zeros(1, device=DEV_T, requires_grad=True)", "+    opt = torch.optim.Adam(list(emb.parameters()) + [bias], lr=0.05, weight_decay=2e-6)", "+    lossf = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(nneg / npos, device=DEV_T))", "+    for _ in range(STEPS):", "+        opt.zero_grad()", "+        logit = emb(inp, off).squeeze(1) + bias", "+        loss = lossf(logit, y)", "+        loss.backward(); opt.step()", "+    with torch.no_grad():", "+        sc = (emb(pool_inp, pool_off).squeeze(1) + bias).cpu().numpy()", "+    return sc", " ", "-neg_rows = random.sample(keep, min(N_NEG, len(keep)))", "-X_neg = X_pool[neg_rows]", "-", "-# ---------------------------------------------------------------- per-register clf", " scores = {}", " for reg in PROP:", "-    X_pos = vec.transform(pos[reg])", "-    Xtr = sp.vstack([X_pos, X_neg])", "-    ytr = np.r_[np.ones(X_pos.shape[0]), np.zeros(X_neg.shape[0])]", "-    clf = LogisticRegression(C=0.5, solver=\"liblinear\", max_iter=1000)", "-    clf.fit(Xtr, ytr)", "-    scores[reg] = clf.decision_function(X_pool)", "-    print(f\"[clf] {reg:8s} trained on {X_pos.shape[0]} pos / {len(neg_rows)} neg\")", "+    t0 = time.time()", "+    scores[reg] = train_score(pos[reg])", "+    print(f\"[clf] {reg:8s} trained ({len(pos[reg])} pos/{len(neg_rows)} neg) {time.time()-t0:.0f}s\")", " ", "-# ---------------------------------------------------------------- ranked lists", "-order = {reg: sorted(keep, key=lambda i: scores[reg][i], reverse=True) for reg in PROP}", "+# ---------------------------------------------------------------- ranked lists (over kept docs, by kept-index j)", "+order = {reg: sorted(range(len(keep)), key=lambda j: scores[reg][j], reverse=True) for reg in PROP}", " ", " # ---------------------------------------------------------------- weighted round robin", " ptr    = {r: 0 for r in PROP}", "-tokens = {r: 0 for r in PROP}", "+tokens = {r: 0.0 for r in PROP}", " selection, chosen, seen = [], set(), set()", " total = 0", " while total < TARGET_TOKENS:", "-    # register furthest behind its target token pace", "     reg = min(PROP, key=lambda r: (tokens[r] / PROP[r], -PROP[r]))", "     picked = None", "     while ptr[reg] < len(order[reg]):", "-        i = order[reg][ptr[reg]]; ptr[reg] += 1", "-        if ids[i] in chosen:", "+        j = order[reg][ptr[reg]]; ptr[reg] += 1", "+        gi = keep[j]", "+        if ids[gi] in chosen:", "             continue", "-        k = dedup_key(texts[i])", "+        k = dedup_key(texts[gi])", "         if k in seen:", "             continue", "-        seen.add(k); picked = i; break", "+        seen.add(k); picked = gi; break", "     if picked is None:", "         if all(ptr[r] >= len(order[r]) for r in PROP):", "             break", "-        tokens[reg] = float(\"inf\")      # stop selecting this exhausted register", "+        tokens[reg] = float(\"inf\")", "         continue", "     est = max(1, len(texts[picked]) // 4)", "     selection.append(int(ids[picked])); chosen.add(ids[picked])"]}], "originalFile": "\"\"\"\nCurate a 12M-token pretraining selection from a raw web pool to minimize held-out\nperplexity on a BROAD high-quality English target (equal-parts encyclopedic /\nWikipedia, general web prose, news, technical Q&A).\n\nSTATED CRITERION\n----------------\nSelect the pool documents whose *distribution matches the disclosed target\ndomain*, register by register, then compose the final training set so its token\nmixture reproduces the target's own register mixture. Concretely:\n\n  1. The provided dev sample (data/multi_dev.npy) is a labelled draw from the\n     target domain. It cleanly separates into three registers by surface markers:\n       - `wiki`    : WikiText-style encyclopedic prose (contains @-@ / @,@ / @.@)\n       - `qa`      : StackOverflow-style technical Q&A (contains <p>/<code>/<pre>)\n       - `newsweb` : everything else = news + general high-quality web prose\n     Measured token mixture of the dev target: wiki~16%, qa~25%, newsweb~59%.\n\n  2. For each register we train a logistic-regression classifier\n     (hashed word 1-2 gram features) that separates that register's target text\n     from random pool text. Surface artifacts (@-@, HTML tags) are normalised\n     away before featurizing, so the classifier keys on *content vocabulary*\n     (encyclopedic style, code/question vocabulary, news prose) rather than on\n     markup the pool cannot contain. Each pool doc receives a target-likeness\n     score per register.\n\n  3. After a light quality prefilter (length / alphabetic-ratio / mojibake) and\n     exact-normalised de-duplication, we fill the priority list by\n     TOKEN-WEIGHTED ROUND ROBIN across the three registers, always extending the\n     register that is furthest behind its target token share. This front-loads a\n     clean, register-balanced set into the first 12M tokens (the budget the\n     trainer actually consumes) that mirrors the target distribution.\n\nThis is a reproducible, distribution-matching criterion (classifier + mixture\ncontrol), not a hand-picked id list. Deterministic given SEED.\n\"\"\"\nimport json, re, random, hashlib, time\nimport numpy as np\nimport scipy.sparse as sp\nfrom transformers import AutoTokenizer\nfrom sklearn.feature_extraction.text import HashingVectorizer\nfrom sklearn.linear_model import LogisticRegression\n\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED)\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\n# Target register token mixture, measured from the dev sample.\nPROP = {\"wiki\": 0.16, \"qa\": 0.25, \"newsweb\": 0.59}\nTARGET_TOKENS = 40_000_000          # produce ~3x the 12M budget as margin\nN_FEATURES    = 2 ** 19\nN_NEG         = 20000               # random pool negatives for the classifiers\n\n# ---------------------------------------------------------------- preprocessing\nTAG  = re.compile(r\"<[^>]+>\")            # strips HTML tags AND <|endoftext|>\nWART = re.compile(r\"\\s?@([-,.])@\\s?\")    # WikiText  X @-@ Y  ->  X-Y\nWS   = re.compile(r\"\\s+\")\ndef norm(t):\n    t = WART.sub(r\"\\1\", t)\n    t = TAG.sub(\" \", t)\n    t = t.lower()\n    return WS.sub(\" \", t)\n\ndef quality_ok(t):\n    L = len(t)\n    if L < 250 or L > 60000:            # drop tiny boilerplate & giant dumps\n        return False\n    head = t[:4000]\n    if sum(c.isalpha() or c.isspace() for c in head) < 0.70 * len(head):\n        return False                    # symbol/number soup, nav junk\n    if t.count(\"�\") > 3:           # mojibake / replacement chars\n        return False\n    return True\n\ndef dedup_key(t):\n    return hashlib.md5(norm(t)[:2000].encode(\"utf-8\", \"ignore\")).hexdigest()\n\n# ---------------------------------------------------------------- load pool\nt0 = time.time()\nids, texts = [], []\nwith open(POOL) as f:\n    for line in f:\n        r = json.loads(line)\n        ids.append(r[\"id\"]); texts.append(r[\"text\"])\nprint(f\"[load] {len(ids)} pool docs  {time.time()-t0:.0f}s\")\n\n# ---------------------------------------------------------------- dev registers\ntok = AutoTokenizer.from_pretrained(\"gpt2\"); EOS = tok.eos_token_id\nd = np.load(DEV)\ncut = np.where(d == EOS)[0]\nstarts = [0] + list(cut + 1); ends = list(cut) + [len(d)]\npos = {\"wiki\": [], \"qa\": [], \"newsweb\": []}\nfor s, e in zip(starts, ends):\n    seg = d[s:e]\n    if len(seg) < 5:\n        continue\n    tx = tok.decode(seg.tolist())\n    if (\"@-@\" in tx) or (\"@,@\" in tx) or (\"@.@\" in tx):\n        pos[\"wiki\"].append(tx)\n    elif (\"<p>\" in tx) or (\"<code>\" in tx) or (\"<pre>\" in tx) or (\"</p>\" in tx):\n        pos[\"qa\"].append(tx)\n    else:\n        pos[\"newsweb\"].append(tx)\nprint(\"[dev] pos docs:\", {k: len(v) for k, v in pos.items()})\n\n# ---------------------------------------------------------------- prefilter\nkeep = [i for i, t in enumerate(texts) if quality_ok(t)]\nprint(f\"[filter] {len(keep)}/{len(texts)} pass quality prefilter\")\n\n# ---------------------------------------------------------------- vectorize once\nvec = HashingVectorizer(preprocessor=norm, ngram_range=(1, 2),\n                        n_features=N_FEATURES, alternate_sign=False, norm=\"l2\")\nt0 = time.time()\nX_pool = vec.transform(texts)                       # 182k x 2^19 sparse\nprint(f\"[vec] pool vectorized {time.time()-t0:.0f}s  shape {X_pool.shape}\")\n\nneg_rows = random.sample(keep, min(N_NEG, len(keep)))\nX_neg = X_pool[neg_rows]\n\n# ---------------------------------------------------------------- per-register clf\nscores = {}\nfor reg in PROP:\n    X_pos = vec.transform(pos[reg])\n    Xtr = sp.vstack([X_pos, X_neg])\n    ytr = np.r_[np.ones(X_pos.shape[0]), np.zeros(X_neg.shape[0])]\n    clf = LogisticRegression(C=0.5, solver=\"liblinear\", max_iter=1000)\n    clf.fit(Xtr, ytr)\n    scores[reg] = clf.decision_function(X_pool)\n    print(f\"[clf] {reg:8s} trained on {X_pos.shape[0]} pos / {len(neg_rows)} neg\")\n\n# ---------------------------------------------------------------- ranked lists\norder = {reg: sorted(keep, key=lambda i: scores[reg][i], reverse=True) for reg in PROP}\n\n# ---------------------------------------------------------------- weighted round robin\nptr    = {r: 0 for r in PROP}\ntokens = {r: 0 for r in PROP}\nselection, chosen, seen = [], set(), set()\ntotal = 0\nwhile total < TARGET_TOKENS:\n    # register furthest behind its target token pace\n    reg = min(PROP, key=lambda r: (tokens[r] / PROP[r], -PROP[r]))\n    picked = None\n    while ptr[reg] < len(order[reg]):\n        i = order[reg][ptr[reg]]; ptr[reg] += 1\n        if ids[i] in chosen:\n            continue\n        k = dedup_key(texts[i])\n        if k in seen:\n            continue\n        seen.add(k); picked = i; break\n    if picked is None:\n        if all(ptr[r] >= len(order[r]) for r in PROP):\n            break\n        tokens[reg] = float(\"inf\")      # stop selecting this exhausted register\n        continue\n    est = max(1, len(texts[picked]) // 4)\n    selection.append(int(ids[picked])); chosen.add(ids[picked])\n    tokens[reg] += est; total += est\n\njson.dump(selection, open(OUT, \"w\"))\nmix = {r: round(tokens[r] / total, 3) for r in PROP if tokens[r] != float(\"inf\")}\nprint(f\"[out] {len(selection)} ids, ~{total/1e6:.1f}M est tokens, mix={mix} -> {OUT}\")\n", "userModified": false}

time left: 4720s
[load] 182016 pool docs  5s
[filter] 177507/182016 pass quality prefilter
[feat] pool featurized 63s  |vocab|=953498
[dev] pos docs: {'wiki': 966, 'qa': 359, 'newsweb': 1380} 3s
[clf] wiki     trained (966 pos/20000 neg) 3s
[clf] qa       trained (359 pos/20000 neg) 2s
[clf] newsweb  trained (1380 pos/20000 neg) 2s
[out] 94116 ids, ~40.0M est tokens, mix={'wiki': 0.16, 'qa': 0.25, 'newsweb': 0.59} -> /workspace/submission/selection.json

real	1m48.875s
user	1m41.110s
sys	0m3.950s

[stdout]
time left: 4720s
[load] 182016 pool docs  5s
[filter] 177507/182016 pass quality prefilter
[feat] pool featurized 63s  |vocab|=953498
[dev] pos docs: {'wiki': 966, 'qa': 359, 'newsweb': 1380} 3s
[clf] wiki     trained (966 pos/20000 neg) 3s
[clf] qa       trained (359 pos/20000 neg) 2s
[clf] newsweb  trained (1380 pos/20000 neg) 2s
[out] 94116 ids, ~40.0M est tokens, mix={'wiki': 0.16, 'qa': 0.25, 'newsweb': 0.59} -> /workspace/submission/selection.json

real	1m48.875s
user	1m41.110s
sys	0m3.950s
[metadata] {"noOutputExpected": false}

n ids 94116 first 10 [52768, 172882, 82269, 22765, 6827, 12246, 58907, 6138, 18553, 45266]
--- id 52768 (9570 ch) ---
As an election draws closer, politicians – both incumbent and otherwise – are expected to make themselves and their abilities known, louder than usual. While several states are scheduled to go to polls in 2022 with peopl
--- id 172882 (3663 ch) ---
 AutoComplete : How to get the selected items id?_编程问答_动力学知识库 动力学知识库 主页 编程 软件 设计 生活 游戏 作文 当前位置: 动力学知识库 > 问答 > 编程问答 > javascript - Jquery AutoComplete : How to get the selected items id? 问题描述: I have an auto complete fiel
--- id 82269 (24054 ch) ---
Hill 303 massacre |Hill 303 massacre| Bodies of massacre victims gathered near Waegwan, South Korea, many with their hands still bound |Location||Hill 303, Waegwan, South Korea| |Date||August 17, 1950 |Target||U.S. Army 
--- id 22765 (3515 ch) ---
I think that the main page is referring to modules developed with any language, but are executed as dll's, lib's, vi's, exe's, and the like. Was there a specific scripting language that you were looking at using? I belie
--- id 6827 (8026 ch) ---
Slippers were hurled at RJD leader Tejashwi Yadav during a poll rally in Bihar’s Aurangabad. The leader was sitting with party workers when the incident happened. Two slippers were hurled at the leader. One of the slippe
--- id 12246 (3877 ch) ---
Connecting to your Database¶ There are two ways to connect to a database: The “auto connect” feature will load and instantiate the database class with every page load. To enable “auto connecting”, add the word database t
--- id 58907 (11203 ch) ---
 Special Forces covered up massacre of Afghans Preparations advance for assault on Kandahar 6 April 2010 A NATO military statement issued Sunday admitted that US Special Forces commandos carried out the execution-style k
--- id 6138 (3373 ch) ---
XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the
--- id 18553 (9847 ch) ---
President Trump’s ouster of national security adviser Michael Flynn, and the circumstances leading up to it, have quickly become a major crisis for the fledgling administration, forcing the White House on the defensive a
--- id 45266 (766 ch) ---
'm interested in approaches that avoids code in the code behind. In my opinion, there are some cases where code must be placed in the code behind. For example: I have a grid with an undefined count of columns. Columns ca
--- id 92766 (4024 ch) ---
<|endoftext|>While working with Silverlight applications, we may face some scenarios where we may need to embed Silverlight as a component, like for e.g in Sharepoint Webpars or simple we can have the same with ASP.NET. 
--- id 37786 (4945 ch) ---
Prime Minister Narendra Modi’s visit to Fiji Islands would be an enormous boost for the Indian diasporic community and will underline increased warmth in Fiji’s ties with India. The scenic South Pacific island nation, wh

[stdout]
n ids 94116 first 10 [52768, 172882, 82269, 22765, 6827, 12246, 58907, 6138, 18553, 45266]
--- id 52768 (9570 ch) ---
As an election draws closer, politicians – both incumbent and otherwise – are expected to make themselves and their abilities known, louder than usual. While several states are scheduled to go to polls in 2022 with peopl
--- id 172882 (3663 ch) ---
 AutoComplete : How to get the selected items id?_编程问答_动力学知识库 动力学知识库 主页 编程 软件 设计 生活 游戏 作文 当前位置: 动力学知识库 > 问答 > 编程问答 > javascript - Jquery AutoComplete : How to get the selected items id? 问题描述: I have an auto complete fiel
--- id 82269 (24054 ch) ---
Hill 303 massacre |Hill 303 massacre| Bodies of massacre victims gathered near Waegwan, South Korea, many with their hands still bound |Location||Hill 303, Waegwan, South Korea| |Date||August 17, 1950 |Target||U.S. Army 
--- id 22765 (3515 ch) ---
I think that the main page is referring to modules developed with any language, but are executed as dll's, lib's, vi's, exe's, and the like. Was there a specific scripting language that you were looking at using? I belie
--- id 6827 (8026 ch) ---
Slippers were hurled at RJD leader Tejashwi Yadav during a poll rally in Bihar’s Aurangabad. The leader was sitting with party workers when the incident happened. Two slippers were hurled at the leader. One of the slippe
--- id 12246 (3877 ch) ---
Connecting to your Database¶ There are two ways to connect to a database: The “auto connect” feature will load and instantiate the database class with every page load. To enable “auto connecting”, add the word database t
--- id 58907 (11203 ch) ---
 Special Forces covered up massacre of Afghans Preparations advance for assault on Kandahar 6 April 2010 A NATO military statement issued Sunday admitted that US Special Forces commandos carried out the execution-style k
--- id 6138 (3373 ch) ---
XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the
--- id 18553 (9847 ch) ---
President Trump’s ouster of national security adviser Michael Flynn, and the circumstances leading up to it, have quickly become a major crisis for the fledgling administration, forcing the White House on the defensive a
--- id 45266 (766 ch) ---
'm interested in approaches that avoids code in the code behind. In my opinion, there are some cases where code must be placed in the code behind. For example: I have a grid with an undefined count of columns. Columns ca
--- id 92766 (4024 ch) ---
<|endoftext|>While working with Silverlight applications, we may face some scenarios where we may need to embed Silverlight as a component, like for e.g in Sharepoint Webpars or simple we can have the same with ASP.NET. 
--- id 37786 (4945 ch) ---
Prime Minister Narendra Modi’s visit to Fiji Islands would be an enormous boost for the Indian diasporic community and will underline increased warmth in Fiji’s ties with India. The scenic South Pacific island nation, wh
[metadata] {"noOutputExpected": false}

time left: 4579s
Token indices sequence length is longer than the specified maximum sequence length for this model (2109 > 1024). Running this sequence through the model will result in indexing errors
packed 12000000 tokens from 94116 ids -> /tmp/dev_train.npy
HELDOUT_PPL 355.663  loss 5.8740  train_s 54  /tmp/dev_train.npy
(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)

real	1m29.286s
user	1m26.160s
sys	0m2.860s

[stdout]
time left: 4579s
Token indices sequence length is longer than the specified maximum sequence length for this model (2109 > 1024). Running this sequence through the model will result in indexing errors
packed 12000000 tokens from 94116 ids -> /tmp/dev_train.npy
HELDOUT_PPL 355.663  loss 5.8740  train_s 54  /tmp/dev_train.npy
(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)

real	1m29.286s
user	1m26.160s
sys	0m2.860s
[metadata] {"noOutputExpected": false}

random baseline ids 45000
time left: 4459s
HELDOUT_PPL 457.994  loss 6.1269  train_s 54  /tmp/dev_train.npy

real	1m28.413s
user	1m25.190s
sys	0m3.060s

[stdout]
random baseline ids 45000
time left: 4459s
HELDOUT_PPL 457.994  loss 6.1269  train_s 54  /tmp/dev_train.npy

real	1m28.413s
user	1m25.190s
sys	0m3.060s
[metadata] {"noOutputExpected": false}

total 182016
p                96  (0.1%)
code            118  (0.1%)
pre              13  (0.0%)
wart              0  (0.0%)
both_pcode      215  (0.1%)

real	0m4.335s
user	0m3.940s
sys	0m0.060s

[stdout]
total 182016
p                96  (0.1%)
code            118  (0.1%)
pre              13  (0.0%)
wart              0  (0.0%)
both_pcode      215  (0.1%)

real	0m4.335s
user	0m3.940s
sys	0m0.060s
[metadata] {"noOutputExpected": false}

html docs 215
already in selection: 56
also in first 12M budget region? first 20000 ids: 13
char len: min/med/max 394 5048 115501
--- id 2033 len 665 ---
WATCH #FORUM Full Length & Free For a Limited Time videos by Lorene Voskinarian October 24th, 2012No Comments Tweet Forum is making their latest film #FORUM free to watch for a very limited time. Check it out below. Related posts:Win a Trip to the Streets at S
--- id 4313 len 21523 ---
Methods | Statistics | Clinical | Educational | Industrial | Professional items | World psychology | The Semantic Web is an evolving extension of the World Wide Web in which the semantics of information and services on the web is defined, making it possible fo
--- id 5799 len 4224 ---
What a sad sad day.<br>Progression my arse, this is just bullcrap made in China fking with indigenous landmarks! Surely the security will be costing a HUGE penny.<br><br>LAME.<br><br>TEOTIHUACAN, Mexico (Reuters) - U.S. retail powerhouse Wal-Mart refused on We
--- id 6138 len 3373 ---
XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the file when the form is created and you k

[stdout]
html docs 215
already in selection: 56
also in first 12M budget region? first 20000 ids: 13
char len: min/med/max 394 5048 115501
--- id 2033 len 665 ---
WATCH #FORUM Full Length & Free For a Limited Time videos by Lorene Voskinarian October 24th, 2012No Comments Tweet Forum is making their latest film #FORUM free to watch for a very limited time. Check it out below. Related posts:Win a Trip to the Streets at S
--- id 4313 len 21523 ---
Methods | Statistics | Clinical | Educational | Industrial | Professional items | World psychology | The Semantic Web is an evolving extension of the World Wide Web in which the semantics of information and services on the web is defined, making it possible fo
--- id 5799 len 4224 ---
What a sad sad day.<br>Progression my arse, this is just bullcrap made in China fking with indigenous landmarks! Surely the security will be costing a HUGE penny.<br><br>LAME.<br><br>TEOTIHUACAN, Mexico (Reuters) - U.S. retail powerhouse Wal-Mart refused on We
--- id 6138 len 3373 ---
XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the file when the form is created and you k
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "# ---------------------------------------------------------------- ranked lists (over kept docs, by kept-index j)\norder = {reg: sorted(range(len(keep)), key=lambda j: scores[reg][j], reverse=True) for reg in PROP}", "newString": "# ---------------------------------------------------------------- ranked lists (over kept docs, by kept-index j)\norder = {reg: sorted(range(len(keep)), key=lambda j: scores[reg][j], reverse=True) for reg in PROP}\n\n# Format-match the Q&A register: the target's technical-Q&A portion is raw\n# StackOverflow HTML (<p> ... </p>, <pre><code> ... </code></pre>). Those tag\n# tokens recur throughout ~25% of the eval target but are almost absent from the\n# pool. Front-load every pool doc that carries them (ordered by qa-likeness) so\n# the model gets exposure to the exact format tokens instead of never seeing them.\nFMT = (\"<p>\", \"<code>\", \"<pre>\", \"</p>\", \"<br>\")\nfmt_j = [j for j in range(len(keep)) if any(m in texts[keep[j]] for m in FMT)]\nfmt_j.sort(key=lambda j: scores[\"qa\"][j], reverse=True)\nfmt_set = set(fmt_j)\norder[\"qa\"] = fmt_j + [j for j in order[\"qa\"] if j not in fmt_set]\nprint(f\"[fmt] front-loaded {len(fmt_j)} HTML-format docs into qa stream\")", "originalFile": "\"\"\"\nCurate a 12M-token pretraining selection from a raw web pool to minimize held-out\nperplexity on a BROAD high-quality English target (equal parts encyclopedic /\nWikipedia, general web prose, news, technical Q&A).\n\nSTATED CRITERION\n----------------\nSelect the pool documents whose *distribution matches the disclosed target\ndomain*, register by register, then compose the training set so its token mixture\nreproduces the target's own register mixture. Concretely:\n\n  1. The provided dev sample (data/multi_dev.npy) is a labelled draw from the\n     target domain. It separates into three registers by surface markers:\n       - `wiki`    : WikiText-style encyclopedic prose (contains @-@ / @,@ / @.@)\n       - `qa`      : StackOverflow-style technical Q&A (contains <p>/<code>/<pre>)\n       - `newsweb` : everything else = news + general high-quality web prose\n     Measured dev token mixture: wiki~16%, qa~25%, newsweb~59%.\n\n  2. For each register, train a logistic-regression classifier on hashed word\n     1-2 gram features (implemented as a torch EmbeddingBag over a hashed feature\n     space) that separates that register's target text from random pool text.\n     Surface artifacts (@-@, HTML tags) are normalised away before featurizing so\n     the classifier keys on *content vocabulary* (encyclopedic style, code /\n     question vocabulary, news prose), not on markup the pool cannot contain.\n     Every pool doc gets a target-likeness score per register.\n\n  3. After a light quality prefilter (length / alphabetic-ratio / mojibake) and\n     exact-normalised de-duplication, fill the priority list by TOKEN-WEIGHTED\n     ROUND ROBIN across registers, always extending the register furthest behind\n     its target token share. This front-loads a clean, register-balanced set into\n     the first 12M tokens (the budget the trainer consumes) that mirrors the\n     target distribution.\n\nReproducible distribution-matching criterion (classifier + mixture control),\ndeterministic given SEED — not a hand-picked id list.\n\"\"\"\nimport json, re, random, hashlib, time\nimport numpy as np\nimport torch, torch.nn as nn\n\nSEED = 1337\nrandom.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED)\nDEV_T = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nPROP          = {\"wiki\": 0.16, \"qa\": 0.25, \"newsweb\": 0.59}  # target token mixture\nTARGET_TOKENS = 40_000_000        # ~3x the 12M budget, as margin\nD             = 1 << 19           # hashed feature dimension (524288)\nMAXW          = 1500              # words/doc used for featurization\nN_NEG         = 20000             # random pool negatives\nSTEPS         = 400               # full-batch LR training steps per register\nWORD = re.compile(r\"[a-z0-9]+\")\n\n# multiplicative hash constants (uint64 wrap-around hashing)\nH1 = np.uint64(2654435761); H2 = np.uint64(2246822519); MASK = np.uint64(D - 1)\n\n# ---------------------------------------------------------------- preprocessing\nTAG  = re.compile(r\"<[^>]+>\")            # strips HTML tags AND <|endoftext|>\nWART = re.compile(r\"\\s?@([-,.])@\\s?\")    # WikiText  X @-@ Y  ->  X-Y\nWS   = re.compile(r\"\\s+\")\ndef norm(t):\n    t = WART.sub(r\"\\1\", t)\n    t = TAG.sub(\" \", t)\n    return WS.sub(\" \", t.lower())\n\ndef quality_ok(t):\n    L = len(t)\n    if L < 250 or L > 60000:\n        return False\n    head = t[:4000]\n    if sum(c.isalpha() or c.isspace() for c in head) < 0.70 * len(head):\n        return False\n    if t.count(\"�\") > 3:\n        return False\n    return True\n\ndef dedup_key(t):\n    return hashlib.md5(norm(t)[:2000].encode(\"utf-8\", \"ignore\")).hexdigest()\n\n# word -> stable int id (first-occurrence; permutation-invariant for the model)\nw2id = {}\ndef featurize(t):\n    \"\"\"normalised text -> sorted unique hashed 1-2gram feature ids (np.int64).\"\"\"\n    words = WORD.findall(norm(t))[:MAXW]\n    if not words:\n        return np.zeros(1, dtype=np.int64)\n    sd = w2id.setdefault\n    ids = np.fromiter((sd(w, len(w2id)) for w in words), dtype=np.uint64, count=len(words))\n    uni = (ids * H1) & MASK\n    if len(ids) > 1:\n        bi = ((ids[:-1] * H1) ^ (ids[1:] * H2)) & MASK\n        feat = np.concatenate([uni, bi])\n    else:\n        feat = uni\n    return np.unique(feat.astype(np.int64))\n\ndef pack(feat_list):\n    \"\"\"list of feature-id arrays -> (input, offsets) LongTensors on device.\"\"\"\n    offs = np.zeros(len(feat_list), dtype=np.int64)\n    tot = 0\n    for i, f in enumerate(feat_list):\n        offs[i] = tot; tot += len(f)\n    inp = np.concatenate(feat_list) if feat_list else np.zeros(0, np.int64)\n    return (torch.from_numpy(inp).to(DEV_T), torch.from_numpy(offs).to(DEV_T))\n\n# ---------------------------------------------------------------- load pool\nt0 = time.time()\nids, texts = [], []\nwith open(POOL) as f:\n    for line in f:\n        r = json.loads(line)\n        ids.append(r[\"id\"]); texts.append(r[\"text\"])\nprint(f\"[load] {len(ids)} pool docs  {time.time()-t0:.0f}s\")\n\n# ---------------------------------------------------------------- quality prefilter\nkeep = [i for i, t in enumerate(texts) if quality_ok(t)]\nprint(f\"[filter] {len(keep)}/{len(texts)} pass quality prefilter\")\n\n# ---------------------------------------------------------------- featurize pool\nt0 = time.time()\nkeep_feat = [featurize(texts[i]) for i in keep]\nprint(f\"[feat] pool featurized {time.time()-t0:.0f}s  |vocab|={len(w2id)}\")\npool_inp, pool_off = pack(keep_feat)\n\n# ---------------------------------------------------------------- dev registers\ntok_time = time.time()\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained(\"gpt2\"); EOS = tok.eos_token_id\nd = np.load(DEV)\ncut = np.where(d == EOS)[0]\nstarts = [0] + list(cut + 1); ends = list(cut) + [len(d)]\npos = {\"wiki\": [], \"qa\": [], \"newsweb\": []}\nfor s, e in zip(starts, ends):\n    seg = d[s:e]\n    if len(seg) < 5:\n        continue\n    tx = tok.decode(seg.tolist())\n    if (\"@-@\" in tx) or (\"@,@\" in tx) or (\"@.@\" in tx):\n        pos[\"wiki\"].append(featurize(tx))\n    elif (\"<p>\" in tx) or (\"<code>\" in tx) or (\"<pre>\" in tx) or (\"</p>\" in tx):\n        pos[\"qa\"].append(featurize(tx))\n    else:\n        pos[\"newsweb\"].append(featurize(tx))\nprint(\"[dev] pos docs:\", {k: len(v) for k, v in pos.items()}, f\"{time.time()-tok_time:.0f}s\")\n\n# ---------------------------------------------------------------- per-register LR\nneg_rows = random.sample(range(len(keep)), min(N_NEG, len(keep)))\nneg_inp, neg_off = pack([keep_feat[j] for j in neg_rows])\nneg_off_g = neg_off\n\ndef train_score(pos_feats):\n    p_inp, p_off = pack(pos_feats)\n    npos, nneg = len(pos_feats), len(neg_rows)\n    # concat pos+neg into one batch\n    inp = torch.cat([p_inp, neg_inp])\n    off = torch.cat([p_off, neg_off + len(p_inp)])\n    y = torch.cat([torch.ones(npos), torch.zeros(nneg)]).to(DEV_T)\n    emb = nn.EmbeddingBag(D, 1, mode=\"sum\").to(DEV_T)\n    emb.weight.data.zero_()\n    bias = torch.zeros(1, device=DEV_T, requires_grad=True)\n    opt = torch.optim.Adam(list(emb.parameters()) + [bias], lr=0.05, weight_decay=2e-6)\n    lossf = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(nneg / npos, device=DEV_T))\n    for _ in range(STEPS):\n        opt.zero_grad()\n        logit = emb(inp, off).squeeze(1) + bias\n        loss = lossf(logit, y)\n        loss.backward(); opt.step()\n    with torch.no_grad():\n        sc = (emb(pool_inp, pool_off).squeeze(1) + bias).cpu().numpy()\n    return sc\n\nscores = {}\nfor reg in PROP:\n    t0 = time.time()\n    scores[reg] = train_score(pos[reg])\n    print(f\"[clf] {reg:8s} trained ({len(pos[reg])} pos/{len(neg_rows)} neg) {time.time()-t0:.0f}s\")\n\n# ---------------------------------------------------------------- ranked lists (over kept docs, by kept-index j)\norder = {reg: sorted(range(len(keep)), key=lambda j: scores[reg][j], reverse=True) for reg in PROP}\n\n# ---------------------------------------------------------------- weighted round robin\nptr    = {r: 0 for r in PROP}\ntokens = {r: 0.0 for r in PROP}\nselection, chosen, seen = [], set(), set()\ntotal = 0\nwhile total < TARGET_TOKENS:\n    reg = min(PROP, key=lambda r: (tokens[r] / PROP[r], -PROP[r]))\n    picked = None\n    while ptr[reg] < len(order[reg]):\n        j = order[reg][ptr[reg]]; ptr[reg] += 1\n        gi = keep[j]\n        if ids[gi] in chosen:\n            continue\n        k = dedup_key(texts[gi])\n        if k in seen:\n            continue\n        seen.add(k); picked = gi; break\n    if picked is None:\n        if all(ptr[r] >= len(order[r]) for r in PROP):\n            break\n        tokens[reg] = float(\"inf\")\n        continue\n    est = max(1, len(texts[picked]) // 4)\n    selection.append(int(ids[picked])); chosen.add(ids[picked])\n    tokens[reg] += est; total += est\n\njson.dump(selection, open(OUT, \"w\"))\nmix = {r: round(tokens[r] / total, 3) for r in PROP if tokens[r] != float(\"inf\")}\nprint(f\"[out] {len(selection)} ids, ~{total/1e6:.1f}M est tokens, mix={mix} -> {OUT}\")\n", "structuredPatch": [{"oldStart": 181, "oldLines": 6, "newStart": 181, "newLines": 18, "lines": [" # ---------------------------------------------------------------- ranked lists (over kept docs, by kept-index j)", " order = {reg: sorted(range(len(keep)), key=lambda j: scores[reg][j], reverse=True) for reg in PROP}", " ", "+# Format-match the Q&A register: the target's technical-Q&A portion is raw", "+# StackOverflow HTML (<p> ... </p>, <pre><code> ... </code></pre>). Those tag", "+# tokens recur throughout ~25% of the eval target but are almost absent from the", "+# pool. Front-load every pool doc that carries them (ordered by qa-likeness) so", "+# the model gets exposure to the exact format tokens instead of never seeing them.", "+FMT = (\"<p>\", \"<code>\", \"<pre>\", \"</p>\", \"<br>\")", "+fmt_j = [j for j in range(len(keep)) if any(m in texts[keep[j]] for m in FMT)]", "+fmt_j.sort(key=lambda j: scores[\"qa\"][j], reverse=True)", "+fmt_set = set(fmt_j)", "+order[\"qa\"] = fmt_j + [j for j in order[\"qa\"] if j not in fmt_set]", "+print(f\"[fmt] front-loaded {len(fmt_j)} HTML-format docs into qa stream\")", "+", " # ---------------------------------------------------------------- weighted round robin", " ptr    = {r: 0 for r in PROP}", " tokens = {r: 0.0 for r in PROP}"]}], "userModified": false, "replaceAll": false}

Command did not complete within its 120s timeout and was moved to the background (ID: bvt9g3lix). Output is being written to: /tmp/claude-0/-workspace/010b8dbe-8f98-478c-a3eb-e2fc1f0f67e0/tasks/bvt9g3lix.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bvt9g3lix", "timedOutAfterMs": 120000, "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}