claude-opus-4-8

5390
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 04:02 .
drwxr-xr-x 1 root root  160 Aug  4 04:02 ..
drwxr-xr-x 1 root root   60 Aug  4 04:02 .live
drwxr-xr-x 1 root root  100 Aug  4 04:02 .metrics
drwxr-xr-x 1 root root  140 Aug  4 04:02 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 04:02 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 04:02 ..

[stdout]
5390
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 04:02 .
drwxr-xr-x 1 root root  160 Aug  4 04:02 ..
drwxr-xr-x 1 root root   60 Aug  4 04:02 .live
drwxr-xr-x 1 root root  100 Aug  4 04:02 .metrics
drwxr-xr-x 1 root root  140 Aug  4 04:02 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 04:02 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 04:02 ..
[metadata] {"noOutputExpected": false}

=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")

=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")

=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"

=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done

[stdout]
=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")

=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")

=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"

=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done
[metadata] {"noOutputExpected": false}

scanned (first pass sample) lines: 182016
char len: min 2 max 522573 mean 4233 median 2246.0
======================================== id 0 len 414
|Viewing Single Post From: Spoilers for the Week of February 11th|
|Lil||Feb 1 2013, 09:58 AM|
Don't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, really, but hoping that we get some good "SAMANTHA GENE!!" Marlena Death-Stares out of it. And "newfound" feelings. Please. If only.
STEFANO!! STEFANO, STEFANO, STEFANO!!!! :cheer:
|Spoilers for the Week of February 11th · DAYS: News, Spoilers & Discussion|
======================================== id 1 len 3018
*sigh* Fundamentalist community, let me pass on some advice to you I learned from the atheistic community:
If you have set yourself on fire, do not run.
Okay? Okay?? Please?
Look, D, you had two months to say to Harvard in private emails, "Im sorry, I shouldnt have been using that animation in my paid presentations. I wont use it again. I really do like 'Inner Life', though, and would love to use it in classroom presentations, from the BioVisions site, if that is acceptable."
I sat here, for two months, waiting for that to happen, anything to happen, and it didnt. Two months, on your own terms, you could have written a similar post to yesterdays. I would have given you the benefit of the doubt-- maybe you didnt know the credits werent visible to the audience, and I wouldnt have said a word
======================================== id 2 len 2825
A novel two-step immunotherapy approach has shown clinically beneficial responses in patients with advanced ovarian cancer. Following Lifestyle Recommendations Reduces Risk of Cancer Death
People who follow the diet and lifestyle recommendations laid out by the WCRF and the AICR have a 20 percent reduced risk of dying from cancer. UCSF Launches Social Networking Site for Patients and Families with Hereditary Cancers
For Immediate Release May 14, 2013 UCSF Launches Social Networking Site for Patients and Families... Genomic Test May Help Guide Prostate Cancer Treatment
The Oncotype DX® Prostate Cancer Test strongly predicts aggressiveness of disease. Statins Linked to Lower Risk of Liver Cancer in Hepatitis C
People infected with chronic hepatitis C are less likely to develop liver cancer i

[stdout]
scanned (first pass sample) lines: 182016
char len: min 2 max 522573 mean 4233 median 2246.0
======================================== id 0 len 414
|Viewing Single Post From: Spoilers for the Week of February 11th|
|Lil||Feb 1 2013, 09:58 AM|
Don't care about Chloe/Taniel/Jen-Jen. Don't care about Sami, really, but hoping that we get some good "SAMANTHA GENE!!" Marlena Death-Stares out of it. And "newfound" feelings. Please. If only.
STEFANO!! STEFANO, STEFANO, STEFANO!!!! :cheer:
|Spoilers for the Week of February 11th · DAYS: News, Spoilers & Discussion|
======================================== id 1 len 3018
*sigh* Fundamentalist community, let me pass on some advice to you I learned from the atheistic community:
If you have set yourself on fire, do not run.
Okay? Okay?? Please?
Look, D, you had two months to say to Harvard in private emails, "Im sorry, I shouldnt have been using that animation in my paid presentations. I wont use it again. I really do like 'Inner Life', though, and would love to use it in classroom presentations, from the BioVisions site, if that is acceptable."
I sat here, for two months, waiting for that to happen, anything to happen, and it didnt. Two months, on your own terms, you could have written a similar post to yesterdays. I would have given you the benefit of the doubt-- maybe you didnt know the credits werent visible to the audience, and I wouldnt have said a word
======================================== id 2 len 2825
A novel two-step immunotherapy approach has shown clinically beneficial responses in patients with advanced ovarian cancer. Following Lifestyle Recommendations Reduces Risk of Cancer Death
People who follow the diet and lifestyle recommendations laid out by the WCRF and the AICR have a 20 percent reduced risk of dying from cancer. UCSF Launches Social Networking Site for Patients and Families with Hereditary Cancers
For Immediate Release May 14, 2013 UCSF Launches Social Networking Site for Patients and Families... Genomic Test May Help Guide Prostate Cancer Treatment
The Oncotype DX® Prostate Cancer Test strongly predicts aggressiveness of disease. Statins Linked to Lower Risk of Liver Cancer in Hepatitis C
People infected with chronic hepatitis C are less likely to develop liver cancer i
[metadata] {"noOutputExpected": false}

dev tokens: (1000000,) uint16 min 0 max 50256
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member of the Royal Victorian Order , 4th Class ( MVO ; the present @-@ day grade of Lieutenant ) . 
<|endoftext|> On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land . 
<|endoftext|> To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were unstable , precluding the establishment of a safe base there . An extended search for an anchorage at King Edward VII Land proved equally fruitless , so Shackleton was forced to break his undertaking to Scott and set sail for McMurdo Sound , a decision which , according to second officer Arthur Harbord , was " dictated by common sense " in view of the difficulties of ice pressure , coal shortage and the lack of any nearer known base . 
<|endoftext|> Nimrod arrived at McMurdo Sound on 29 January , but was stopped by ice 16 miles ( 26 km ) north of Discovery 's old base at Hut Point . After considerable weather delays , Shackleton 's base was eventually established at Cape Royds , about 24 miles ( 39 km ) north of Hut Point . The party was in high spirits , despite the difficult conditions ; Shackleton 's ability to communicate with each man kept the party happy and focused . 
<|endoftext|> The " Great Southern Journey " , as Frank Wild called it , began on 29 October 1908 . On 9 January 1909 , Shackleton and three companions ( Wild , Eric Marshall and Jameson Adams ) reached a new Farthest South latitude of 88 ° 23 ' S , a point only 112 miles ( 180 km ) from the Pole . En route the South Pole party discovered th

[stdout]
dev tokens: (1000000,) uint16 min 0 max 50256
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member of the Royal Victorian Order , 4th Class ( MVO ; the present @-@ day grade of Lieutenant ) . 
<|endoftext|> On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land . 
<|endoftext|> To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were unstable , precluding the establishment of a safe base there . An extended search for an anchorage at King Edward VII Land proved equally fruitless , so Shackleton was forced to break his undertaking to Scott and set sail for McMurdo Sound , a decision which , according to second officer Arthur Harbord , was " dictated by common sense " in view of the difficulties of ice pressure , coal shortage and the lack of any nearer known base . 
<|endoftext|> Nimrod arrived at McMurdo Sound on 29 January , but was stopped by ice 16 miles ( 26 km ) north of Discovery 's old base at Hut Point . After considerable weather delays , Shackleton 's base was eventually established at Cape Royds , about 24 miles ( 39 km ) north of Hut Point . The party was in high spirits , despite the difficult conditions ; Shackleton 's ability to communicate with each man kept the party happy and focused . 
<|endoftext|> The " Great Southern Journey " , as Frank Wild called it , began on 29 October 1908 . On 9 January 1909 , Shackleton and three companions ( Wild , Eric Marshall and Jameson Adams ) reached a new Farthest South latitude of 88 ° 23 ' S , a point only 112 miles ( 180 km ) from the Pole . En route the South Pole party discovered th
[metadata] {"noOutputExpected": false}

name, memory.total [MiB], memory.used [MiB]
NVIDIA H100 80GB HBM3, 81559 MiB, 4 MiB
---
num EOS: 2704 -> ~segments
segment token lens: n 2705 median 193 mean 370 max 23347
============================== at frac 0.0
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , includin
============================== at frac 0.25
Description of a very high speed transit (VHST) system operating in its own rarefied atmosphere in evacuated tubes in underground tunnels. Most cases considered took less time to go coast-to-coast (e.g., 21 min) than it takes an aircraft to climb to an efficient operating altitude. VHST's tubecraft ride on, and are driven by, electromagnetic (EM) waves. In accelerating, it employs the energy of the surrounding EM field; in decelerating, it returns most of this energy to the system. Tunnel systems would be shared by oil, water, and gas pipelines; channels for laser and microwave waveguides; ele
============================== at frac 0.5
I fucking hate when he does this shit pic.twitter.com/kpmcHnW4Cz — Miley Ray Cyrus (@MileyCyrus) April 22, 2018 Singer-actress Miley Cyrus has shared a rare video of herself with her partner, actor Liam Hemsworth, on social media. She shared a video of them both in a car, which showed Hemsworth listening to rap music as he drove them to their destination.Cyrus captioned the image: "I f***ing hate it when he does this s**t."In the clip, she was sporting a top which had the word 'Sunday' emblazoned across the front as she sat in the passenger seat. With his music blaring out, she filmed herself 
============================== at frac 0.75
<p>I found the platform module but it says it returns 'Windows' and it's returning 'Microsoft' on my machine. I notice in another thread here on stackoverflow it returns 'Vista' sometimes.</p> <p>So, the question is, how do implemement?</p> <pre><code>if is_windows():   ... </code></pre> <p>In a forward compatible way? If I have to check for things like 'Vista' then it will break when the next version of windows comes out.</p> <hr /> <p>Note: The answers claiming this is a duplicate question do not actually answer the question <code>is_windows</code>. They answer the question &quot;what platfo
============================== at frac 0.9
 application (not an applet) that needs to access a web service. Proxies for the web service have been generated with JAX-WS, and seem to work fine. In one scenario it needs to talk through a web proxy server (actually Squid 3.0), which is set to require NTLM authentication.</p>  <p>Running on Sun's JRE 1.6.0_14, everything works fine for accessing HTTP URLs, without requiring any changes: the built-in NTLM authenticator kicks in and it all works seemlessly. If, however, the web service URL is a HTTPS URL, the web service call fails deep inside Sun's code:</p>  <pre><code>com.sun.xml.internal.

[stdout]
name, memory.total [MiB], memory.used [MiB]
NVIDIA H100 80GB HBM3, 81559 MiB, 4 MiB
---
num EOS: 2704 -> ~segments
segment token lens: n 2705 median 193 mean 370 max 23347
============================== at frac 0.0
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , includin
============================== at frac 0.25
Description of a very high speed transit (VHST) system operating in its own rarefied atmosphere in evacuated tubes in underground tunnels. Most cases considered took less time to go coast-to-coast (e.g., 21 min) than it takes an aircraft to climb to an efficient operating altitude. VHST's tubecraft ride on, and are driven by, electromagnetic (EM) waves. In accelerating, it employs the energy of the surrounding EM field; in decelerating, it returns most of this energy to the system. Tunnel systems would be shared by oil, water, and gas pipelines; channels for laser and microwave waveguides; ele
============================== at frac 0.5
I fucking hate when he does this shit pic.twitter.com/kpmcHnW4Cz — Miley Ray Cyrus (@MileyCyrus) April 22, 2018 Singer-actress Miley Cyrus has shared a rare video of herself with her partner, actor Liam Hemsworth, on social media. She shared a video of them both in a car, which showed Hemsworth listening to rap music as he drove them to their destination.Cyrus captioned the image: "I f***ing hate it when he does this s**t."In the clip, she was sporting a top which had the word 'Sunday' emblazoned across the front as she sat in the passenger seat. With his music blaring out, she filmed herself 
============================== at frac 0.75
<p>I found the platform module but it says it returns 'Windows' and it's returning 'Microsoft' on my machine. I notice in another thread here on stackoverflow it returns 'Vista' sometimes.</p> <p>So, the question is, how do implemement?</p> <pre><code>if is_windows():   ... </code></pre> <p>In a forward compatible way? If I have to check for things like 'Vista' then it will break when the next version of windows comes out.</p> <hr /> <p>Note: The answers claiming this is a duplicate question do not actually answer the question <code>is_windows</code>. They answer the question &quot;what platfo
============================== at frac 0.9
 application (not an applet) that needs to access a web service. Proxies for the web service have been generated with JAX-WS, and seem to work fine. In one scenario it needs to talk through a web proxy server (actually Squid 3.0), which is set to require NTLM authentication.</p>  <p>Running on Sun's JRE 1.6.0_14, everything works fine for accessing HTTP URLs, without requiring any changes: the built-in NTLM authenticator kicks in and it all works seemlessly. If, however, the web service URL is a HTTPS URL, the web service call fails deep inside Sun's code:</p>  <pre><code>com.sun.xml.internal.
[metadata] {"noOutputExpected": false}

total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
---bench---
read+json 8000 docs in 0.33s
tokenize+unigram-hash 2673421 wordtoks in 0.93s -> 2869762 tok/s
extrapolate full pool ~200M wordtok(capped) est docs 182016

[stdout]
               total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
---bench---
read+json 8000 docs in 0.33s
tokenize+unigram-hash 2673421 wordtoks in 0.93s -> 2869762 tok/s
extrapolate full pool ~200M wordtok(capped) est docs 182016
[metadata] {"noOutputExpected": false}

Task #1 created successfully: Build DSIR curation script

[metadata] {"task": {"id": "1", "subject": "Build DSIR curation script"}}

Task #2 created successfully: Validate selection with dev_eval training runs

[metadata] {"task": {"id": "2", "subject": "Validate selection with dev_eval training runs"}}

Task #3 created successfully: Write claim.md and finalize deliverables

[metadata] {"task": {"id": "3", "subject": "Write claim.md and finalize deliverables"}}

Updated task #1 status

[metadata] {"success": true, "taskId": "1", "updatedFields": ["status"], "statusChange": {"from": "pending", "to": "in_progress"}}

File created successfully at: /workspace/submission/curate.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/submission/curate.py", "content": "\"\"\"\nCurate a pretraining subset from a raw web pool for a small LM, targeting a\nbroad multi-domain high-quality English target (Wikipedia + HQ web prose + news\n+ technical Q&A).\n\nCriterion (stated, reproducible):\n  Rank every pool document by a DSIR-style importance score = how much more likely\n  the document's n-grams are under the TARGET distribution than under the raw POOL\n  distribution, per token. The target distribution is estimated directly from the\n  disclosed dev target (data/multi_dev.npy, decoded to text). Documents that fail\n  cheap \"is this usable English text\" gates are dropped. The surviving documents\n  are emitted in descending score order (best first); the training pipeline packs\n  them in that priority order until the 12M-token budget is filled.\n\nSignal source: only the pool itself and the disclosed dev target. No labels, no\ninternet.\n\nOutput: /workspace/submission/selection.json  (ordered list of pool ids)\n\"\"\"\nimport json, re, hashlib, numpy as np\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nB = 1 << 20            # hashed feature buckets\nALPHA = 1.0           # add-k smoothing\nWORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 50        # drop docs too short to be useful prose\nSEED = 1337\n\nWORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")\nSTOP = set(\"the of and to a in is that it for was as with on be by at this or an \"\n           \"are from his he not but had which have you they were their\".split())\n\n# deterministic word -> bucket (reproducible regardless of PYTHONHASHSEED)\n_cache = {}\ndef wbucket(w):\n    b = _cache.get(w)\n    if b is None:\n        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & (B - 1)\n        _cache[w] = b\n    return b\n\ndef features(words):\n    \"\"\"return (unigram_buckets, bigram_buckets) as lists of ints\"\"\"\n    ub = [wbucket(w) for w in words]\n    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & (B - 1) for i in range(len(ub) - 1)]\n    return ub, bb\n\ndef toks(s):\n    return WORD_RE.findall(s.lower())[:WORD_CAP]\n\ndef quality_gate(words):\n    \"\"\"cheap gates: usable English prose, not junk/gibberish/foreign/pure-symbol.\"\"\"\n    n = len(words)\n    if n < MIN_WORDS:\n        return False\n    stop_frac = sum(1 for w in words if w in STOP) / n\n    if stop_frac < 0.06:                      # too few function words -> not English prose\n        return False\n    alpha = sum(1 for w in words if w[:1].isalpha())\n    if alpha / n < 0.5:                       # mostly symbols/numbers\n        return False\n    uniq = len(set(words)) / n\n    if uniq < 0.18:                           # extreme repetition / spam\n        return False\n    return True\n\ndef main():\n    # ---- target model from decoded dev target ----\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64)\n    dev_text = tok.decode(dev.tolist())\n    logpt_u = np.full(B, ALPHA); logpt_b = np.full(B, ALPHA)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(logpt_u, tub, 1.0)\n    np.add.at(logpt_b, tbb, 1.0)\n    logpt_u = np.log(logpt_u / logpt_u.sum())\n    logpt_b = np.log(logpt_b / logpt_b.sum())\n\n    # ---- background pool model from a stratified sample ----\n    logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)\n    with open(POOL) as f:\n        for k, line in enumerate(f):\n            if k % 7:                         # ~1/7 sample for background\n                continue\n            r = json.loads(line)\n            w = toks(r[\"text\"])\n            ub, bb = features(w)\n            np.add.at(logpp_u, ub, 1.0)\n            np.add.at(logpp_b, bb, 1.0)\n    logpp_u = np.log(logpp_u / logpp_u.sum())\n    logpp_b = np.log(logpp_b / logpp_b.sum())\n\n    ru = logpt_u - logpp_u                     # per-bucket log importance ratio\n    rb = logpt_b - logpp_b\n\n    # ---- score every doc ----\n    scored = []\n    seen = set()                               # exact/near-dup guard on normalized prefix\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            did = r[\"id\"]; text = r[\"text\"]\n            w = toks(text)\n            if not quality_gate(w):\n                continue\n            key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),\n                                  digest_size=8).digest()\n            if key in seen:\n                continue\n            seen.add(key)\n            ub, bb = features(w)\n            us = ru[ub].mean()\n            bs = rb[bb].mean() if bb else 0.0\n            score = us + bs\n            scored.append((score, did))\n\n    scored.sort(reverse=True)\n    ids = [d for _, d in scored]\n    json.dump(ids, open(OUT, \"w\"))\n    print(f\"scored {len(scored)} docs (passed gates); wrote {len(ids)} ids -> {OUT}\")\n    print(f\"score range: {scored[0][0]:.3f} .. {scored[-1][0]:.3f}\")\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [], "originalFile": null, "userModified": false}

scored 155778 docs (passed gates); wrote 155778 ids -> /workspace/submission/selection.json
score range: 1.588 .. -4.540

real	1m52.087s
user	1m44.310s
sys	0m3.580s

[stdout]
scored 155778 docs (passed gates); wrote 155778 ids -> /workspace/submission/selection.json
score range: 1.588 .. -4.540

real	1m52.087s
user	1m44.310s
sys	0m3.580s
[metadata] {"noOutputExpected": false}

TOP 6 ==========
--- id 114704 len 369
Attendees | All Canada Games Register Here Accommodations Select Page Recruits Attending <br /><br /> Recruits will receive a link after registration to get listed as an attendee. Recruits with an existing ConnectLAX profile can simply click “Attend Event” while logged in to join. Colleg
--- id 123985 len 26380
<|endoftext|>Cast range - Dota 2 Wiki Gamepedia Help Sign In Register Cast range From Dota 2 Wiki Jump to:	navigation, search Play By the forge of creation. This section is under construction. It may still lack crucial information or contain unvalidated statements. You can help by coordinating with 
--- id 6138 len 3373
XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the file when the form is created and you know that the file name will never change
--- id 116653 len 10642
<|endoftext|>Toys - AllEquipped Sign in My Account Checkout Wish List Compare English English call for support 312-978-9936 All CategoriesHomeOptics StoreCamping StoreShooters SupplyHunting StoreFishing StoreKnives & BladesBoating StoreSportsman's KitchenFlashlights and LightingWater Treatment / Tra
--- id 8221 len 3509
Follow-up and lochial Stephanus clamours his cutinization undressings unsold firstly. mesmeric Ted overtrusts, resep cara mengolah ubi jalar his czaritza enthralling eyeleting occasionally. biform Dennie bodges, her research training for social scientists a handbook for postgraduate researchers auto
--- id 66445 len 4981
Derrol tasty snoring residing agro que es six sigma en espanol inflexible. Niccolo que es el algebra de funciones cannibalize pruritic, Sheila admit their cooing que choisir juin 2015 formulaire to the outside. lave linge candy que choisir Wendall overcapitalizing cavalierly and subordinating his te

BOTTOM 4 ==========
--- id 162435 len 4527
 Level Design/Clip Brushes:ru - Valve Developer Community L4D Level Design/Clip Brushes:ru From Valve Developer Community < L4D Level Design Jump to:	navigation, search This page needs to be translated. This page either contains information that is only partially or incorrectly translated, or there 
--- id 129080 len 765
.Ok<|endoftext|>Привет всем.Меня зовут Ирина.Сижу сейчас в декрете.Ищу Mom.life. The app for Modern Moms Download for iOS or Android Привет всем.Меня зовут Ирина.Сижу сейчас в декрете.Ищу сейчас дочке кота кузю. #первыйпост Open in the app You will be able to see all photos, comment and read other p
--- id 156307 len 838
 problems of safety: intelligent transport systems and situational centers" ITS Forum - Kazan Регистрация / Вход [ • RU • EN ] Главная О конференции Информационное письмо Практические площадки Программный комитет ERTICO Выставка Гости ERTICO Хакатон История Контакты Программа конференции Итоги конфе
--- id 133651 len 833
<|endoftext|>ITS Forum - Kazan "Modern problems of safety: intelligent transport systems and situational centers" ITS Forum - Kazan Регистрация / Вход [ • RU • EN ] Главная О конференции Информационное письмо Практические площадки Программный комитет ERTICO Выставка Гости ERTICO Хакатон История Конт

~12M tokens covered by ~10661 top docs (char/4 est)

[stdout]
TOP 6 ==========
--- id 114704 len 369
Attendees | All Canada Games Register Here Accommodations Select Page Recruits Attending &lt;br /&gt;&lt;br /&gt; Recruits will receive a link after registration to get listed as an attendee. Recruits with an existing ConnectLAX profile can simply click “Attend Event” while logged in to join. Colleg
--- id 123985 len 26380
<|endoftext|>Cast range - Dota 2 Wiki Gamepedia Help Sign In Register Cast range From Dota 2 Wiki Jump to:	navigation, search Play By the forge of creation. This section is under construction. It may still lack crucial information or contain unvalidated statements. You can help by coordinating with 
--- id 6138 len 3373
XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the file when the form is created and you know that the file name will never change
--- id 116653 len 10642
<|endoftext|>Toys - AllEquipped Sign in My Account Checkout Wish List Compare English English call for support 312-978-9936 All CategoriesHomeOptics StoreCamping StoreShooters SupplyHunting StoreFishing StoreKnives & BladesBoating StoreSportsman's KitchenFlashlights and LightingWater Treatment / Tra
--- id 8221 len 3509
Follow-up and lochial Stephanus clamours his cutinization undressings unsold firstly. mesmeric Ted overtrusts, resep cara mengolah ubi jalar his czaritza enthralling eyeleting occasionally. biform Dennie bodges, her research training for social scientists a handbook for postgraduate researchers auto
--- id 66445 len 4981
Derrol tasty snoring residing agro que es six sigma en espanol inflexible. Niccolo que es el algebra de funciones cannibalize pruritic, Sheila admit their cooing que choisir juin 2015 formulaire to the outside. lave linge candy que choisir Wendall overcapitalizing cavalierly and subordinating his te

BOTTOM 4 ==========
--- id 162435 len 4527
 Level Design/Clip Brushes:ru - Valve Developer Community L4D Level Design/Clip Brushes:ru From Valve Developer Community < L4D Level Design Jump to:	navigation, search This page needs to be translated. This page either contains information that is only partially or incorrectly translated, or there 
--- id 129080 len 765
.Ok<|endoftext|>Привет всем.Меня зовут Ирина.Сижу сейчас в декрете.Ищу Mom.life. The app for Modern Moms Download for iOS or Android Привет всем.Меня зовут Ирина.Сижу сейчас в декрете.Ищу сейчас дочке кота кузю. #первыйпост Open in the app You will be able to see all photos, comment and read other p
--- id 156307 len 838
 problems of safety: intelligent transport systems and situational centers" ITS Forum - Kazan Регистрация / Вход [ • RU • EN ] Главная О конференции Информационное письмо Практические площадки Программный комитет ERTICO Выставка Гости ERTICO Хакатон История Контакты Программа конференции Итоги конфе
--- id 133651 len 833
<|endoftext|>ITS Forum - Kazan "Modern problems of safety: intelligent transport systems and situational centers" ITS Forum - Kazan Регистрация / Вход [ • RU • EN ] Главная О конференции Информационное письмо Практические площадки Программный комитет ERTICO Выставка Гости ERTICO Хакатон История Конт

~12M tokens covered by ~10661 top docs (char/4 est)
[metadata] {"noOutputExpected": false}

target uniq unigrams 38739 uniq bigrams 321884
id  ucov  bhit  label
   8221 0.481 0.059  SPAM
  66445 0.542 0.109  SPAM
 116653 0.407 0.050  ecom-boiler
 114704 0.988 0.585  nav-boiler
 123985 0.932 0.441  wiki-ok
   6138 0.937 0.701  tech-ok
      2 0.918 0.453  news-ok
      1 0.953 0.661  blog-ok

[stdout]
target uniq unigrams 38739 uniq bigrams 321884
id  ucov  bhit  label
   8221 0.481 0.059  SPAM
  66445 0.542 0.109  SPAM
 116653 0.407 0.050  ecom-boiler
 114704 0.988 0.585  nav-boiler
 123985 0.932 0.441  wiki-ok
   6138 0.937 0.701  tech-ok
      2 0.918 0.453  news-ok
      1 0.953 0.661  blog-ok
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "WORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 50        # drop docs too short to be useful prose\nSEED = 1337", "newString": "WORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 80        # drop docs too short to be useful prose\nMIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)\nMIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)\nBI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density\nSEED = 1337", "originalFile": "\"\"\"\nCurate a pretraining subset from a raw web pool for a small LM, targeting a\nbroad multi-domain high-quality English target (Wikipedia + HQ web prose + news\n+ technical Q&A).\n\nCriterion (stated, reproducible):\n  Rank every pool document by a DSIR-style importance score = how much more likely\n  the document's n-grams are under the TARGET distribution than under the raw POOL\n  distribution, per token. The target distribution is estimated directly from the\n  disclosed dev target (data/multi_dev.npy, decoded to text). Documents that fail\n  cheap \"is this usable English text\" gates are dropped. The surviving documents\n  are emitted in descending score order (best first); the training pipeline packs\n  them in that priority order until the 12M-token budget is filled.\n\nSignal source: only the pool itself and the disclosed dev target. No labels, no\ninternet.\n\nOutput: /workspace/submission/selection.json  (ordered list of pool ids)\n\"\"\"\nimport json, re, hashlib, numpy as np\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nB = 1 << 20            # hashed feature buckets\nALPHA = 1.0           # add-k smoothing\nWORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 50        # drop docs too short to be useful prose\nSEED = 1337\n\nWORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")\nSTOP = set(\"the of and to a in is that it for was as with on be by at this or an \"\n           \"are from his he not but had which have you they were their\".split())\n\n# deterministic word -> bucket (reproducible regardless of PYTHONHASHSEED)\n_cache = {}\ndef wbucket(w):\n    b = _cache.get(w)\n    if b is None:\n        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & (B - 1)\n        _cache[w] = b\n    return b\n\ndef features(words):\n    \"\"\"return (unigram_buckets, bigram_buckets) as lists of ints\"\"\"\n    ub = [wbucket(w) for w in words]\n    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & (B - 1) for i in range(len(ub) - 1)]\n    return ub, bb\n\ndef toks(s):\n    return WORD_RE.findall(s.lower())[:WORD_CAP]\n\ndef quality_gate(words):\n    \"\"\"cheap gates: usable English prose, not junk/gibberish/foreign/pure-symbol.\"\"\"\n    n = len(words)\n    if n < MIN_WORDS:\n        return False\n    stop_frac = sum(1 for w in words if w in STOP) / n\n    if stop_frac < 0.06:                      # too few function words -> not English prose\n        return False\n    alpha = sum(1 for w in words if w[:1].isalpha())\n    if alpha / n < 0.5:                       # mostly symbols/numbers\n        return False\n    uniq = len(set(words)) / n\n    if uniq < 0.18:                           # extreme repetition / spam\n        return False\n    return True\n\ndef main():\n    # ---- target model from decoded dev target ----\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64)\n    dev_text = tok.decode(dev.tolist())\n    logpt_u = np.full(B, ALPHA); logpt_b = np.full(B, ALPHA)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(logpt_u, tub, 1.0)\n    np.add.at(logpt_b, tbb, 1.0)\n    logpt_u = np.log(logpt_u / logpt_u.sum())\n    logpt_b = np.log(logpt_b / logpt_b.sum())\n\n    # ---- background pool model from a stratified sample ----\n    logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)\n    with open(POOL) as f:\n        for k, line in enumerate(f):\n            if k % 7:                         # ~1/7 sample for background\n                continue\n            r = json.loads(line)\n            w = toks(r[\"text\"])\n            ub, bb = features(w)\n            np.add.at(logpp_u, ub, 1.0)\n            np.add.at(logpp_b, bb, 1.0)\n    logpp_u = np.log(logpp_u / logpp_u.sum())\n    logpp_b = np.log(logpp_b / logpp_b.sum())\n\n    ru = logpt_u - logpp_u                     # per-bucket log importance ratio\n    rb = logpt_b - logpp_b\n\n    # ---- score every doc ----\n    scored = []\n    seen = set()                               # exact/near-dup guard on normalized prefix\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            did = r[\"id\"]; text = r[\"text\"]\n            w = toks(text)\n            if not quality_gate(w):\n                continue\n            key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),\n                                  digest_size=8).digest()\n            if key in seen:\n                continue\n            seen.add(key)\n            ub, bb = features(w)\n            us = ru[ub].mean()\n            bs = rb[bb].mean() if bb else 0.0\n            score = us + bs\n            scored.append((score, did))\n\n    scored.sort(reverse=True)\n    ids = [d for _, d in scored]\n    json.dump(ids, open(OUT, \"w\"))\n    print(f\"scored {len(scored)} docs (passed gates); wrote {len(ids)} ids -> {OUT}\")\n    print(f\"score range: {scored[0][0]:.3f} .. {scored[-1][0]:.3f}\")\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 27, "oldLines": 7, "newStart": 27, "newLines": 10, "lines": [" B = 1 << 20            # hashed feature buckets", " ALPHA = 1.0           # add-k smoothing", " WORD_CAP = 600        # max words scored per doc (domain signal is stationary)", "-MIN_WORDS = 50        # drop docs too short to be useful prose", "+MIN_WORDS = 80        # drop docs too short to be useful prose", "+MIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)", "+MIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)", "+BI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density", " SEED = 1337", " ", " WORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")"]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    logpt_u = np.full(B, ALPHA); logpt_b = np.full(B, ALPHA)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(logpt_u, tub, 1.0)\n    np.add.at(logpt_b, tbb, 1.0)\n    logpt_u = np.log(logpt_u / logpt_u.sum())\n    logpt_b = np.log(logpt_b / logpt_b.sum())", "newString": "    cnt_u = np.zeros(B); cnt_b = np.zeros(B)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(cnt_u, tub, 1.0)\n    np.add.at(cnt_b, tbb, 1.0)\n    attested_u = cnt_u > 0                      # vocab/bigram presence for gates\n    attested_b = cnt_b > 0\n    logpt_u = np.log((cnt_u + ALPHA) / (cnt_u.sum() + ALPHA * B))\n    logpt_b = np.log((cnt_b + ALPHA) / (cnt_b.sum() + ALPHA * B))", "originalFile": "\"\"\"\nCurate a pretraining subset from a raw web pool for a small LM, targeting a\nbroad multi-domain high-quality English target (Wikipedia + HQ web prose + news\n+ technical Q&A).\n\nCriterion (stated, reproducible):\n  Rank every pool document by a DSIR-style importance score = how much more likely\n  the document's n-grams are under the TARGET distribution than under the raw POOL\n  distribution, per token. The target distribution is estimated directly from the\n  disclosed dev target (data/multi_dev.npy, decoded to text). Documents that fail\n  cheap \"is this usable English text\" gates are dropped. The surviving documents\n  are emitted in descending score order (best first); the training pipeline packs\n  them in that priority order until the 12M-token budget is filled.\n\nSignal source: only the pool itself and the disclosed dev target. No labels, no\ninternet.\n\nOutput: /workspace/submission/selection.json  (ordered list of pool ids)\n\"\"\"\nimport json, re, hashlib, numpy as np\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nB = 1 << 20            # hashed feature buckets\nALPHA = 1.0           # add-k smoothing\nWORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 80        # drop docs too short to be useful prose\nMIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)\nMIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)\nBI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density\nSEED = 1337\n\nWORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")\nSTOP = set(\"the of and to a in is that it for was as with on be by at this or an \"\n           \"are from his he not but had which have you they were their\".split())\n\n# deterministic word -> bucket (reproducible regardless of PYTHONHASHSEED)\n_cache = {}\ndef wbucket(w):\n    b = _cache.get(w)\n    if b is None:\n        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & (B - 1)\n        _cache[w] = b\n    return b\n\ndef features(words):\n    \"\"\"return (unigram_buckets, bigram_buckets) as lists of ints\"\"\"\n    ub = [wbucket(w) for w in words]\n    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & (B - 1) for i in range(len(ub) - 1)]\n    return ub, bb\n\ndef toks(s):\n    return WORD_RE.findall(s.lower())[:WORD_CAP]\n\ndef quality_gate(words):\n    \"\"\"cheap gates: usable English prose, not junk/gibberish/foreign/pure-symbol.\"\"\"\n    n = len(words)\n    if n < MIN_WORDS:\n        return False\n    stop_frac = sum(1 for w in words if w in STOP) / n\n    if stop_frac < 0.06:                      # too few function words -> not English prose\n        return False\n    alpha = sum(1 for w in words if w[:1].isalpha())\n    if alpha / n < 0.5:                       # mostly symbols/numbers\n        return False\n    uniq = len(set(words)) / n\n    if uniq < 0.18:                           # extreme repetition / spam\n        return False\n    return True\n\ndef main():\n    # ---- target model from decoded dev target ----\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64)\n    dev_text = tok.decode(dev.tolist())\n    logpt_u = np.full(B, ALPHA); logpt_b = np.full(B, ALPHA)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(logpt_u, tub, 1.0)\n    np.add.at(logpt_b, tbb, 1.0)\n    logpt_u = np.log(logpt_u / logpt_u.sum())\n    logpt_b = np.log(logpt_b / logpt_b.sum())\n\n    # ---- background pool model from a stratified sample ----\n    logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)\n    with open(POOL) as f:\n        for k, line in enumerate(f):\n            if k % 7:                         # ~1/7 sample for background\n                continue\n            r = json.loads(line)\n            w = toks(r[\"text\"])\n            ub, bb = features(w)\n            np.add.at(logpp_u, ub, 1.0)\n            np.add.at(logpp_b, bb, 1.0)\n    logpp_u = np.log(logpp_u / logpp_u.sum())\n    logpp_b = np.log(logpp_b / logpp_b.sum())\n\n    ru = logpt_u - logpp_u                     # per-bucket log importance ratio\n    rb = logpt_b - logpp_b\n\n    # ---- score every doc ----\n    scored = []\n    seen = set()                               # exact/near-dup guard on normalized prefix\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            did = r[\"id\"]; text = r[\"text\"]\n            w = toks(text)\n            if not quality_gate(w):\n                continue\n            key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),\n                                  digest_size=8).digest()\n            if key in seen:\n                continue\n            seen.add(key)\n            ub, bb = features(w)\n            us = ru[ub].mean()\n            bs = rb[bb].mean() if bb else 0.0\n            score = us + bs\n            scored.append((score, did))\n\n    scored.sort(reverse=True)\n    ids = [d for _, d in scored]\n    json.dump(ids, open(OUT, \"w\"))\n    print(f\"scored {len(scored)} docs (passed gates); wrote {len(ids)} ids -> {OUT}\")\n    print(f\"score range: {scored[0][0]:.3f} .. {scored[-1][0]:.3f}\")\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 76, "oldLines": 13, "newStart": 76, "newLines": 15, "lines": ["     tok = AutoTokenizer.from_pretrained(\"gpt2\")", "     dev = np.load(DEV).astype(np.int64)", "     dev_text = tok.decode(dev.tolist())", "-    logpt_u = np.full(B, ALPHA); logpt_b = np.full(B, ALPHA)", "+    cnt_u = np.zeros(B); cnt_b = np.zeros(B)", "     tw = WORD_RE.findall(dev_text.lower())", "     tub, tbb = features(tw)", "-    np.add.at(logpt_u, tub, 1.0)", "-    np.add.at(logpt_b, tbb, 1.0)", "-    logpt_u = np.log(logpt_u / logpt_u.sum())", "-    logpt_b = np.log(logpt_b / logpt_b.sum())", "+    np.add.at(cnt_u, tub, 1.0)", "+    np.add.at(cnt_b, tbb, 1.0)", "+    attested_u = cnt_u > 0                      # vocab/bigram presence for gates", "+    attested_b = cnt_b > 0", "+    logpt_u = np.log((cnt_u + ALPHA) / (cnt_u.sum() + ALPHA * B))", "+    logpt_b = np.log((cnt_b + ALPHA) / (cnt_b.sum() + ALPHA * B))", " ", "     # ---- background pool model from a stratified sample ----", "     logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)"]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "            ub, bb = features(w)\n            us = ru[ub].mean()\n            bs = rb[bb].mean() if bb else 0.0\n            score = us + bs\n            scored.append((score, did))", "newString": "            ub, bb = features(w)\n            # fluency/vocab gates against the target\n            if attested_u[ub].mean() < MIN_UCOV:\n                continue\n            if not bb or attested_b[bb].mean() < MIN_BHIT:\n                continue\n            us = ru[ub].mean()\n            bs = rb[bb].mean()\n            score = us + BI_WEIGHT * bs\n            scored.append((score, did))", "originalFile": "\"\"\"\nCurate a pretraining subset from a raw web pool for a small LM, targeting a\nbroad multi-domain high-quality English target (Wikipedia + HQ web prose + news\n+ technical Q&A).\n\nCriterion (stated, reproducible):\n  Rank every pool document by a DSIR-style importance score = how much more likely\n  the document's n-grams are under the TARGET distribution than under the raw POOL\n  distribution, per token. The target distribution is estimated directly from the\n  disclosed dev target (data/multi_dev.npy, decoded to text). Documents that fail\n  cheap \"is this usable English text\" gates are dropped. The surviving documents\n  are emitted in descending score order (best first); the training pipeline packs\n  them in that priority order until the 12M-token budget is filled.\n\nSignal source: only the pool itself and the disclosed dev target. No labels, no\ninternet.\n\nOutput: /workspace/submission/selection.json  (ordered list of pool ids)\n\"\"\"\nimport json, re, hashlib, numpy as np\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nB = 1 << 20            # hashed feature buckets\nALPHA = 1.0           # add-k smoothing\nWORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 80        # drop docs too short to be useful prose\nMIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)\nMIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)\nBI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density\nSEED = 1337\n\nWORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")\nSTOP = set(\"the of and to a in is that it for was as with on be by at this or an \"\n           \"are from his he not but had which have you they were their\".split())\n\n# deterministic word -> bucket (reproducible regardless of PYTHONHASHSEED)\n_cache = {}\ndef wbucket(w):\n    b = _cache.get(w)\n    if b is None:\n        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & (B - 1)\n        _cache[w] = b\n    return b\n\ndef features(words):\n    \"\"\"return (unigram_buckets, bigram_buckets) as lists of ints\"\"\"\n    ub = [wbucket(w) for w in words]\n    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & (B - 1) for i in range(len(ub) - 1)]\n    return ub, bb\n\ndef toks(s):\n    return WORD_RE.findall(s.lower())[:WORD_CAP]\n\ndef quality_gate(words):\n    \"\"\"cheap gates: usable English prose, not junk/gibberish/foreign/pure-symbol.\"\"\"\n    n = len(words)\n    if n < MIN_WORDS:\n        return False\n    stop_frac = sum(1 for w in words if w in STOP) / n\n    if stop_frac < 0.06:                      # too few function words -> not English prose\n        return False\n    alpha = sum(1 for w in words if w[:1].isalpha())\n    if alpha / n < 0.5:                       # mostly symbols/numbers\n        return False\n    uniq = len(set(words)) / n\n    if uniq < 0.18:                           # extreme repetition / spam\n        return False\n    return True\n\ndef main():\n    # ---- target model from decoded dev target ----\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64)\n    dev_text = tok.decode(dev.tolist())\n    cnt_u = np.zeros(B); cnt_b = np.zeros(B)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(cnt_u, tub, 1.0)\n    np.add.at(cnt_b, tbb, 1.0)\n    attested_u = cnt_u > 0                      # vocab/bigram presence for gates\n    attested_b = cnt_b > 0\n    logpt_u = np.log((cnt_u + ALPHA) / (cnt_u.sum() + ALPHA * B))\n    logpt_b = np.log((cnt_b + ALPHA) / (cnt_b.sum() + ALPHA * B))\n\n    # ---- background pool model from a stratified sample ----\n    logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)\n    with open(POOL) as f:\n        for k, line in enumerate(f):\n            if k % 7:                         # ~1/7 sample for background\n                continue\n            r = json.loads(line)\n            w = toks(r[\"text\"])\n            ub, bb = features(w)\n            np.add.at(logpp_u, ub, 1.0)\n            np.add.at(logpp_b, bb, 1.0)\n    logpp_u = np.log(logpp_u / logpp_u.sum())\n    logpp_b = np.log(logpp_b / logpp_b.sum())\n\n    ru = logpt_u - logpp_u                     # per-bucket log importance ratio\n    rb = logpt_b - logpp_b\n\n    # ---- score every doc ----\n    scored = []\n    seen = set()                               # exact/near-dup guard on normalized prefix\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            did = r[\"id\"]; text = r[\"text\"]\n            w = toks(text)\n            if not quality_gate(w):\n                continue\n            key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),\n                                  digest_size=8).digest()\n            if key in seen:\n                continue\n            seen.add(key)\n            ub, bb = features(w)\n            us = ru[ub].mean()\n            bs = rb[bb].mean() if bb else 0.0\n            score = us + bs\n            scored.append((score, did))\n\n    scored.sort(reverse=True)\n    ids = [d for _, d in scored]\n    json.dump(ids, open(OUT, \"w\"))\n    print(f\"scored {len(scored)} docs (passed gates); wrote {len(ids)} ids -> {OUT}\")\n    print(f\"score range: {scored[0][0]:.3f} .. {scored[-1][0]:.3f}\")\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 119, "oldLines": 9, "newStart": 119, "newLines": 14, "lines": ["                 continue", "             seen.add(key)", "             ub, bb = features(w)", "+            # fluency/vocab gates against the target", "+            if attested_u[ub].mean() < MIN_UCOV:", "+                continue", "+            if not bb or attested_b[bb].mean() < MIN_BHIT:", "+                continue", "             us = ru[ub].mean()", "-            bs = rb[bb].mean() if bb else 0.0", "-            score = us + bs", "+            bs = rb[bb].mean()", "+            score = us + BI_WEIGHT * bs", "             scored.append((score, did))", " ", "     scored.sort(reverse=True)"]}], "userModified": false, "replaceAll": false}

scored 151224 docs (passed gates); wrote 151224 ids -> /workspace/submission/selection.json
score range: 3.303 .. -5.796

real	1m56.340s
user	1m48.880s
sys	0m2.800s

[stdout]
scored 151224 docs (passed gates); wrote 151224 ids -> /workspace/submission/selection.json
score range: 3.303 .. -5.796

real	1m56.340s
user	1m48.880s
sys	0m2.800s
[metadata] {"noOutputExpected": false}

spam 8221 still selected? False
spam 66445 still selected? False
spam 116653 still selected? False
spam 114704 still selected? True

TOP 8 ==========
--- id 114704 len 369: Attendees | All Canada Games Register Here Accommodations Select Page Recruits Attending <br /><br /> Recruits will receive a link after registration to get listed as an attendee. Recruits with an existing Co
--- id 123985 len 26380: <|endoftext|>Cast range - Dota 2 Wiki Gamepedia Help Sign In Register Cast range From Dota 2 Wiki Jump to:	navigation, search Play By the forge of creation. This section is under construction. It may still lack crucial i
--- id 6138 len 3373: XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the
--- id 13746 len 4110: Scaling the Windows Stack George Beech @GABeech PICC ‘12. out of 23 Post on 27-Dec-2015 Embed Size (px) <p>PowerPoint Presentation</p> <p>Scaling the Windows StackGeorge Beech @GABeechPICC 12AgendaWhat is Stack Exchange?
--- id 9544 len 1605: I'm not sure if I worded my topic title properly, which is probably why I haven't been able to search for the answer to my problem just yet (oh, have I tried, just can't find the right keywords I suppose). Anyways, I am 
--- id 12743 len 531: ZF-5830: Zend_Db_Table_Select doesn't allow use of $select->columns('..') Zend_Db_Table_Select doesn't allow use of $select->columns('..') code fragment: $tbl = new Category_Table(); $select = $tbl->select()->columns('id
--- id 117870 len 3022: Transvision Beta Main Views Home 3 locales Glossary TMX Download QA Views Access Keys Keyboard Shortcuts Check Variables Empty Strings Unchanged Strings Unlocalized Words Translation Consistency Health Status Overview Pr
--- id 148963 len 8821:  dropdown list dynamically in symfony? (select the cities of the region) (Ajax) - Codedump.io CodeDump Add Browse Sign up Sign in Select language ActionScript Ajax Android AngularJS Apache Configuration AppleScript ASP.N

rank position of known-good ids:
123985 rank 1
6138 rank 2
2 rank 65881
1 rank 40469

[stdout]
spam 8221 still selected? False
spam 66445 still selected? False
spam 116653 still selected? False
spam 114704 still selected? True

TOP 8 ==========
--- id 114704 len 369: Attendees | All Canada Games Register Here Accommodations Select Page Recruits Attending &lt;br /&gt;&lt;br /&gt; Recruits will receive a link after registration to get listed as an attendee. Recruits with an existing Co
--- id 123985 len 26380: <|endoftext|>Cast range - Dota 2 Wiki Gamepedia Help Sign In Register Cast range From Dota 2 Wiki Jump to:	navigation, search Play By the forge of creation. This section is under construction. It may still lack crucial i
--- id 6138 len 3373: XForms/Read and write with get and put Sometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the
--- id 13746 len 4110: Scaling the Windows Stack George Beech @GABeech PICC ‘12. out of 23 Post on 27-Dec-2015 Embed Size (px) <p>PowerPoint Presentation</p> <p>Scaling the Windows StackGeorge Beech @GABeechPICC 12AgendaWhat is Stack Exchange?
--- id 9544 len 1605: I'm not sure if I worded my topic title properly, which is probably why I haven't been able to search for the answer to my problem just yet (oh, have I tried, just can't find the right keywords I suppose). Anyways, I am 
--- id 12743 len 531: ZF-5830: Zend_Db_Table_Select doesn't allow use of $select->columns('..') Zend_Db_Table_Select doesn't allow use of $select->columns('..') code fragment: $tbl = new Category_Table(); $select = $tbl->select()->columns('id
--- id 117870 len 3022: Transvision Beta Main Views Home 3 locales Glossary TMX Download QA Views Access Keys Keyboard Shortcuts Check Variables Empty Strings Unchanged Strings Unlocalized Words Translation Consistency Health Status Overview Pr
--- id 148963 len 8821:  dropdown list dynamically in symfony? (select the cities of the region) (Ajax) - Codedump.io CodeDump Add Browse Sign up Sign in Select language ActionScript Ajax Android AngularJS Apache Configuration AppleScript ASP.N

rank position of known-good ids:
123985 rank 1
6138 rank 2
2 rank 65881
1 rank 40469
[metadata] {"noOutputExpected": false}

random sel 40000 ids
launching baseline training in background

[stdout]
random sel 40000 ids
launching baseline training in background
[metadata] {"noOutputExpected": false}

Command running in background with ID: blpzuz32l. Output is being written to: /tmp/claude-0/-workspace/8d3f2e42-7a38-4920-b1b1-3dc7521e7d7f/tasks/blpzuz32l.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "blpzuz32l", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

saved v1
Token indices sequence length is longer than the specified maximum sequence length for this model (2521 > 1024). Running this sequence through the model will result in indexing errors
packed 12000000 tokens from 40000 ids -> /tmp/rand_train.npy

[stdout]
saved v1
Token indices sequence length is longer than the specified maximum sequence length for this model (2521 > 1024). Running this sequence through the model will result in indexing errors
packed 12000000 tokens from 40000 ids -> /tmp/rand_train.npy
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"type": "update", "filePath": "/workspace/submission/curate.py", "content": "\"\"\"\nCurate a pretraining subset from a raw web pool for a small LM, targeting a\nBROAD multi-domain high-quality English target that is disclosed to be *equal\nparts* four registers: encyclopedic (Wikipedia), general HQ web prose, news, and\ntechnical Q&A.\n\nStated, reproducible criterion\n------------------------------\n1. QUALITY GATE. Drop documents that are not usable English prose: too short,\n   too few function words, mostly symbols/numbers, extreme repetition, low\n   overlap with the target's vocabulary, or incoherent (low fraction of bigrams\n   attested in the target). This removes foreign text, navigation/boilerplate,\n   and multilingual keyword-spam.\n2. REGISTER MATCHING (DSIR-style, per register). The disclosed dev target\n   (data/multi_dev.npy) is decoded to text, split into its documents, and\n   clustered into K=4 registers. For each register c we build an n-gram model\n   and score every surviving pool doc by its per-token log importance ratio\n   log p_c(doc) - log p_pool(doc): how much more the register explains the doc\n   than the raw pool does. Each doc is assigned to its best-matching register.\n3. BALANCED FILL. Because the target is *equal parts* the four registers and\n   held-out perplexity is dominated by the worst-modelled register, we emit\n   documents so that any prefix of the priority list holds ~equal TOKENS from\n   each register: greedily append the next-best unused doc from whichever\n   register currently has the fewest accumulated tokens. The training pipeline\n   packs this prefix until the 12M-token budget is met -> a balanced training mix.\n\nSignal source: only the pool itself and the disclosed dev target. No labels, no\ninternet. Deterministic (fixed seed).\n\"\"\"\nimport json, re, hashlib, numpy as np\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nB  = 1 << 20          # hashed feature buckets for scoring models\nB2 = 1 << 14          # hashed feature buckets for clustering\nK  = 4                # registers (disclosed: wiki / web prose / news / tech Q&A)\nALPHA = 1.0           # add-k smoothing\nWORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 80        # drop docs too short to be useful prose\nMIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)\nMIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)\nBI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density\nBUDGET = 12_000_000\nSEED = 1337\n\nWORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")\nSTOP = set(\"the of and to a in is that it for was as with on be by at this or an \"\n           \"are from his he not but had which have you they were their\".split())\n\n_cache = {}\ndef wbucket(w, mask):\n    key = (w, mask)\n    b = _cache.get(key)\n    if b is None:\n        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & mask\n        _cache[key] = b\n    return b\n\ndef features(words, B_):\n    m = B_ - 1\n    ub = [wbucket(w, m) for w in words]\n    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & m for i in range(len(ub) - 1)]\n    return ub, bb\n\ndef toks(s):\n    return WORD_RE.findall(s.lower())[:WORD_CAP]\n\ndef quality_gate(words):\n    n = len(words)\n    if n < MIN_WORDS:\n        return False\n    if sum(1 for w in words if w in STOP) / n < 0.06:\n        return False\n    if sum(1 for w in words if w[:1].isalpha()) / n < 0.5:\n        return False\n    if len(set(words)) / n < 0.18:\n        return False\n    return True\n\ndef kmeans(X, k, iters=30, seed=SEED):\n    rng = np.random.default_rng(seed)\n    # kmeans++-ish: random distinct starts\n    cen = X[rng.choice(len(X), k, replace=False)].copy()\n    assign = np.zeros(len(X), dtype=np.int64)\n    for _ in range(iters):\n        sim = X @ cen.T                       # rows already L2-normalised\n        new = sim.argmax(1)\n        if (new == assign).all():\n            assign = new; break\n        assign = new\n        for j in range(k):\n            m = X[assign == j]\n            if len(m):\n                v = m.sum(0); nrm = np.linalg.norm(v)\n                if nrm > 0: cen[j] = v / nrm\n    return assign\n\ndef main():\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64)\n\n    # ---- split dev target into documents ----\n    EOS = 50256\n    cut = np.where(dev == EOS)[0]\n    segs = [s for s in np.split(dev, cut) if len(s) > 30]\n    seg_text = [tok.decode(s.tolist()) for s in segs]\n\n    # ---- cluster dev docs into K registers (cosine k-means on hashed n-grams) ----\n    Xc = np.zeros((len(seg_text), B2), dtype=np.float32)\n    for i, t in enumerate(seg_text):\n        w = WORD_RE.findall(t.lower())[:WORD_CAP]\n        ub, bb = features(w, B2)\n        for h in ub: Xc[i, h] += 1.0\n        for h in bb: Xc[i, h] += 1.0\n    Xc = np.log1p(Xc)\n    nrm = np.linalg.norm(Xc, axis=1, keepdims=True); nrm[nrm == 0] = 1\n    Xc /= nrm\n    lab = kmeans(Xc, K)\n    sizes = [int((lab == j).sum()) for j in range(K)]\n    print(\"register cluster sizes:\", sizes)\n\n    # ---- per-register target models + global attested sets (for gates) ----\n    cnt_u = np.zeros(B); cnt_b = np.zeros(B)          # global target (gates)\n    logpt_u = [np.full(B, ALPHA) for _ in range(K)]   # per-register unigram\n    logpt_b = [np.full(B, ALPHA) for _ in range(K)]   # per-register bigram\n    for t, c in zip(seg_text, lab):\n        w = WORD_RE.findall(t.lower())\n        ub, bb = features(w, B)\n        np.add.at(cnt_u, ub, 1.0); np.add.at(cnt_b, bb, 1.0)\n        np.add.at(logpt_u[c], ub, 1.0); np.add.at(logpt_b[c], bb, 1.0)\n    attested_u = cnt_u > 0; attested_b = cnt_b > 0\n    for c in range(K):\n        logpt_u[c] = np.log(logpt_u[c] / logpt_u[c].sum())\n        logpt_b[c] = np.log(logpt_b[c] / logpt_b[c].sum())\n\n    # ---- background pool model (1/7 stratified sample) ----\n    logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)\n    with open(POOL) as f:\n        for k, line in enumerate(f):\n            if k % 7:\n                continue\n            w = toks(json.loads(line)[\"text\"])\n            ub, bb = features(w, B)\n            np.add.at(logpp_u, ub, 1.0); np.add.at(logpp_b, bb, 1.0)\n    logpp_u = np.log(logpp_u / logpp_u.sum())\n    logpp_b = np.log(logpp_b / logpp_b.sum())\n\n    ru = [logpt_u[c] - logpp_u for c in range(K)]     # per-register importance ratio\n    rb = [logpt_b[c] - logpp_b for c in range(K)]\n\n    # ---- score every doc, assign to best register ----\n    buckets = [[] for _ in range(K)]                  # (score, est_tokens, id) per register\n    seen = set()\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line); did = r[\"id\"]; text = r[\"text\"]\n            w = toks(text)\n            if not quality_gate(w):\n                continue\n            ub, bb = features(w, B)\n            if attested_u[ub].mean() < MIN_UCOV: continue\n            if not bb or attested_b[bb].mean() < MIN_BHIT: continue\n            key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),\n                                  digest_size=8).digest()\n            if key in seen: continue\n            seen.add(key)\n            ubi = np.asarray(ub); bbi = np.asarray(bb)\n            best_c, best_s = 0, -1e9\n            for c in range(K):\n                s = ru[c][ubi].mean() + BI_WEIGHT * rb[c][bbi].mean()\n                if s > best_s: best_s, best_c = s, c\n            est = max(1, len(text) // 4)              # ~GPT-2 tokens\n            buckets[best_c].append((best_s, est, did))\n\n    for c in range(K):\n        buckets[c].sort(reverse=True)\n    print(\"docs per register:\", [len(b) for b in buckets])\n\n    # ---- balanced fill: keep any prefix ~equal in tokens across registers ----\n    ptr = [0] * K\n    acc = [0] * K\n    order = []\n    total_needed = BUDGET * 3                          # emit well past budget\n    total = 0\n    while total < total_needed:\n        # choose register with fewest accumulated tokens that still has docs\n        cand = [c for c in range(K) if ptr[c] < len(buckets[c])]\n        if not cand:\n            break\n        c = min(cand, key=lambda c: acc[c])\n        s, est, did = buckets[c][ptr[c]]; ptr[c] += 1\n        order.append(did); acc[c] += est; total += est\n\n    json.dump(order, open(OUT, \"w\"))\n    print(f\"wrote {len(order)} ids -> {OUT}\")\n    print(\"token share by register (first {}M):\".format(total_needed // 1_000_000), acc)\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [{"oldStart": 1, "oldLines": 21, "newStart": 1, "newLines": 31, "lines": [" \"\"\"", " Curate a pretraining subset from a raw web pool for a small LM, targeting a", "-broad multi-domain high-quality English target (Wikipedia + HQ web prose + news", "-+ technical Q&A).", "+BROAD multi-domain high-quality English target that is disclosed to be *equal", "+parts* four registers: encyclopedic (Wikipedia), general HQ web prose, news, and", "+technical Q&A.", " ", "-Criterion (stated, reproducible):", "-  Rank every pool document by a DSIR-style importance score = how much more likely", "-  the document's n-grams are under the TARGET distribution than under the raw POOL", "-  distribution, per token. The target distribution is estimated directly from the", "-  disclosed dev target (data/multi_dev.npy, decoded to text). Documents that fail", "-  cheap \"is this usable English text\" gates are dropped. The surviving documents", "-  are emitted in descending score order (best first); the training pipeline packs", "-  them in that priority order until the 12M-token budget is filled.", "+Stated, reproducible criterion", "+------------------------------", "+1. QUALITY GATE. Drop documents that are not usable English prose: too short,", "+   too few function words, mostly symbols/numbers, extreme repetition, low", "+   overlap with the target's vocabulary, or incoherent (low fraction of bigrams", "+   attested in the target). This removes foreign text, navigation/boilerplate,", "+   and multilingual keyword-spam.", "+2. REGISTER MATCHING (DSIR-style, per register). The disclosed dev target", "+   (data/multi_dev.npy) is decoded to text, split into its documents, and", "+   clustered into K=4 registers. For each register c we build an n-gram model", "+   and score every surviving pool doc by its per-token log importance ratio", "+   log p_c(doc) - log p_pool(doc): how much more the register explains the doc", "+   than the raw pool does. Each doc is assigned to its best-matching register.", "+3. BALANCED FILL. Because the target is *equal parts* the four registers and", "+   held-out perplexity is dominated by the worst-modelled register, we emit", "+   documents so that any prefix of the priority list holds ~equal TOKENS from", "+   each register: greedily append the next-best unused doc from whichever", "+   register currently has the fewest accumulated tokens. The training pipeline", "+   packs this prefix until the 12M-token budget is met -> a balanced training mix.", " ", " Signal source: only the pool itself and the disclosed dev target. No labels, no", "-internet.", "-", "-Output: /workspace/submission/selection.json  (ordered list of pool ids)", "+internet. Deterministic (fixed seed).", " \"\"\"", " import json, re, hashlib, numpy as np", " from transformers import AutoTokenizer"]}, {"oldStart": 24, "oldLines": 116, "newStart": 34, "newLines": 169, "lines": [" DEV  = \"/workspace/data/multi_dev.npy\"", " OUT  = \"/workspace/submission/selection.json\"", " ", "-B = 1 << 20            # hashed feature buckets", "+B  = 1 << 20          # hashed feature buckets for scoring models", "+B2 = 1 << 14          # hashed feature buckets for clustering", "+K  = 4                # registers (disclosed: wiki / web prose / news / tech Q&A)", " ALPHA = 1.0           # add-k smoothing", " WORD_CAP = 600        # max words scored per doc (domain signal is stationary)", " MIN_WORDS = 80        # drop docs too short to be useful prose", " MIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)", " MIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)", " BI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density", "+BUDGET = 12_000_000", " SEED = 1337", " ", " WORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")", " STOP = set(\"the of and to a in is that it for was as with on be by at this or an \"", "            \"are from his he not but had which have you they were their\".split())", " ", "-# deterministic word -> bucket (reproducible regardless of PYTHONHASHSEED)", " _cache = {}", "-def wbucket(w):", "-    b = _cache.get(w)", "+def wbucket(w, mask):", "+    key = (w, mask)", "+    b = _cache.get(key)", "     if b is None:", "-        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & (B - 1)", "-        _cache[w] = b", "+        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & mask", "+        _cache[key] = b", "     return b", " ", "-def features(words):", "-    \"\"\"return (unigram_buckets, bigram_buckets) as lists of ints\"\"\"", "-    ub = [wbucket(w) for w in words]", "-    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & (B - 1) for i in range(len(ub) - 1)]", "+def features(words, B_):", "+    m = B_ - 1", "+    ub = [wbucket(w, m) for w in words]", "+    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & m for i in range(len(ub) - 1)]", "     return ub, bb", " ", " def toks(s):", "     return WORD_RE.findall(s.lower())[:WORD_CAP]", " ", " def quality_gate(words):", "-    \"\"\"cheap gates: usable English prose, not junk/gibberish/foreign/pure-symbol.\"\"\"", "     n = len(words)", "     if n < MIN_WORDS:", "         return False", "-    stop_frac = sum(1 for w in words if w in STOP) / n", "-    if stop_frac < 0.06:                      # too few function words -> not English prose", "+    if sum(1 for w in words if w in STOP) / n < 0.06:", "         return False", "-    alpha = sum(1 for w in words if w[:1].isalpha())", "-    if alpha / n < 0.5:                       # mostly symbols/numbers", "+    if sum(1 for w in words if w[:1].isalpha()) / n < 0.5:", "         return False", "-    uniq = len(set(words)) / n", "-    if uniq < 0.18:                           # extreme repetition / spam", "+    if len(set(words)) / n < 0.18:", "         return False", "     return True", " ", "+def kmeans(X, k, iters=30, seed=SEED):", "+    rng = np.random.default_rng(seed)", "+    # kmeans++-ish: random distinct starts", "+    cen = X[rng.choice(len(X), k, replace=False)].copy()", "+    assign = np.zeros(len(X), dtype=np.int64)", "+    for _ in range(iters):", "+        sim = X @ cen.T                       # rows already L2-normalised", "+        new = sim.argmax(1)", "+        if (new == assign).all():", "+            assign = new; break", "+        assign = new", "+        for j in range(k):", "+            m = X[assign == j]", "+            if len(m):", "+                v = m.sum(0); nrm = np.linalg.norm(v)", "+                if nrm > 0: cen[j] = v / nrm", "+    return assign", "+", " def main():", "-    # ---- target model from decoded dev target ----", "     tok = AutoTokenizer.from_pretrained(\"gpt2\")", "     dev = np.load(DEV).astype(np.int64)", "-    dev_text = tok.decode(dev.tolist())", "-    cnt_u = np.zeros(B); cnt_b = np.zeros(B)", "-    tw = WORD_RE.findall(dev_text.lower())", "-    tub, tbb = features(tw)", "-    np.add.at(cnt_u, tub, 1.0)", "-    np.add.at(cnt_b, tbb, 1.0)", "-    attested_u = cnt_u > 0                      # vocab/bigram presence for gates", "-    attested_b = cnt_b > 0", "-    logpt_u = np.log((cnt_u + ALPHA) / (cnt_u.sum() + ALPHA * B))", "-    logpt_b = np.log((cnt_b + ALPHA) / (cnt_b.sum() + ALPHA * B))", " ", "-    # ---- background pool model from a stratified sample ----", "+    # ---- split dev target into documents ----", "+    EOS = 50256", "+    cut = np.where(dev == EOS)[0]", "+    segs = [s for s in np.split(dev, cut) if len(s) > 30]", "+    seg_text = [tok.decode(s.tolist()) for s in segs]", "+", "+    # ---- cluster dev docs into K registers (cosine k-means on hashed n-grams) ----", "+    Xc = np.zeros((len(seg_text), B2), dtype=np.float32)", "+    for i, t in enumerate(seg_text):", "+        w = WORD_RE.findall(t.lower())[:WORD_CAP]", "+        ub, bb = features(w, B2)", "+        for h in ub: Xc[i, h] += 1.0", "+        for h in bb: Xc[i, h] += 1.0", "+    Xc = np.log1p(Xc)", "+    nrm = np.linalg.norm(Xc, axis=1, keepdims=True); nrm[nrm == 0] = 1", "+    Xc /= nrm", "+    lab = kmeans(Xc, K)", "+    sizes = [int((lab == j).sum()) for j in range(K)]", "+    print(\"register cluster sizes:\", sizes)", "+", "+    # ---- per-register target models + global attested sets (for gates) ----", "+    cnt_u = np.zeros(B); cnt_b = np.zeros(B)          # global target (gates)", "+    logpt_u = [np.full(B, ALPHA) for _ in range(K)]   # per-register unigram", "+    logpt_b = [np.full(B, ALPHA) for _ in range(K)]   # per-register bigram", "+    for t, c in zip(seg_text, lab):", "+        w = WORD_RE.findall(t.lower())", "+        ub, bb = features(w, B)", "+        np.add.at(cnt_u, ub, 1.0); np.add.at(cnt_b, bb, 1.0)", "+        np.add.at(logpt_u[c], ub, 1.0); np.add.at(logpt_b[c], bb, 1.0)", "+    attested_u = cnt_u > 0; attested_b = cnt_b > 0", "+    for c in range(K):", "+        logpt_u[c] = np.log(logpt_u[c] / logpt_u[c].sum())", "+        logpt_b[c] = np.log(logpt_b[c] / logpt_b[c].sum())", "+", "+    # ---- background pool model (1/7 stratified sample) ----", "     logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)", "     with open(POOL) as f:", "         for k, line in enumerate(f):", "-            if k % 7:                         # ~1/7 sample for background", "+            if k % 7:", "                 continue", "-            r = json.loads(line)", "-            w = toks(r[\"text\"])", "-            ub, bb = features(w)", "-            np.add.at(logpp_u, ub, 1.0)", "-            np.add.at(logpp_b, bb, 1.0)", "+            w = toks(json.loads(line)[\"text\"])", "+            ub, bb = features(w, B)", "+            np.add.at(logpp_u, ub, 1.0); np.add.at(logpp_b, bb, 1.0)", "     logpp_u = np.log(logpp_u / logpp_u.sum())", "     logpp_b = np.log(logpp_b / logpp_b.sum())", " ", "-    ru = logpt_u - logpp_u                     # per-bucket log importance ratio", "-    rb = logpt_b - logpp_b", "+    ru = [logpt_u[c] - logpp_u for c in range(K)]     # per-register importance ratio", "+    rb = [logpt_b[c] - logpp_b for c in range(K)]", " ", "-    # ---- score every doc ----", "-    scored = []", "-    seen = set()                               # exact/near-dup guard on normalized prefix", "+    # ---- score every doc, assign to best register ----", "+    buckets = [[] for _ in range(K)]                  # (score, est_tokens, id) per register", "+    seen = set()", "     with open(POOL) as f:", "         for line in f:", "-            r = json.loads(line)", "-            did = r[\"id\"]; text = r[\"text\"]", "+            r = json.loads(line); did = r[\"id\"]; text = r[\"text\"]", "             w = toks(text)", "             if not quality_gate(w):", "                 continue", "+            ub, bb = features(w, B)", "+            if attested_u[ub].mean() < MIN_UCOV: continue", "+            if not bb or attested_b[bb].mean() < MIN_BHIT: continue", "             key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),", "                                   digest_size=8).digest()", "-            if key in seen:", "-                continue", "+            if key in seen: continue", "             seen.add(key)", "-            ub, bb = features(w)", "-            # fluency/vocab gates against the target", "-            if attested_u[ub].mean() < MIN_UCOV:", "-                continue", "-            if not bb or attested_b[bb].mean() < MIN_BHIT:", "-                continue", "-            us = ru[ub].mean()", "-            bs = rb[bb].mean()", "-            score = us + BI_WEIGHT * bs", "-            scored.append((score, did))", "+            ubi = np.asarray(ub); bbi = np.asarray(bb)", "+            best_c, best_s = 0, -1e9", "+            for c in range(K):", "+                s = ru[c][ubi].mean() + BI_WEIGHT * rb[c][bbi].mean()", "+                if s > best_s: best_s, best_c = s, c", "+            est = max(1, len(text) // 4)              # ~GPT-2 tokens", "+            buckets[best_c].append((best_s, est, did))", " ", "-    scored.sort(reverse=True)", "-    ids = [d for _, d in scored]", "-    json.dump(ids, open(OUT, \"w\"))", "-    print(f\"scored {len(scored)} docs (passed gates); wrote {len(ids)} ids -> {OUT}\")", "-    print(f\"score range: {scored[0][0]:.3f} .. {scored[-1][0]:.3f}\")", "+    for c in range(K):", "+        buckets[c].sort(reverse=True)", "+    print(\"docs per register:\", [len(b) for b in buckets])", " ", "+    # ---- balanced fill: keep any prefix ~equal in tokens across registers ----", "+    ptr = [0] * K", "+    acc = [0] * K", "+    order = []", "+    total_needed = BUDGET * 3                          # emit well past budget", "+    total = 0", "+    while total < total_needed:", "+        # choose register with fewest accumulated tokens that still has docs", "+        cand = [c for c in range(K) if ptr[c] < len(buckets[c])]", "+        if not cand:", "+            break", "+        c = min(cand, key=lambda c: acc[c])", "+        s, est, did = buckets[c][ptr[c]]; ptr[c] += 1", "+        order.append(did); acc[c] += est; total += est", "+", "+    json.dump(order, open(OUT, \"w\"))", "+    print(f\"wrote {len(order)} ids -> {OUT}\")", "+    print(\"token share by register (first {}M):\".format(total_needed // 1_000_000), acc)", "+", " if __name__ == \"__main__\":", "     main()"]}], "originalFile": "\"\"\"\nCurate a pretraining subset from a raw web pool for a small LM, targeting a\nbroad multi-domain high-quality English target (Wikipedia + HQ web prose + news\n+ technical Q&A).\n\nCriterion (stated, reproducible):\n  Rank every pool document by a DSIR-style importance score = how much more likely\n  the document's n-grams are under the TARGET distribution than under the raw POOL\n  distribution, per token. The target distribution is estimated directly from the\n  disclosed dev target (data/multi_dev.npy, decoded to text). Documents that fail\n  cheap \"is this usable English text\" gates are dropped. The surviving documents\n  are emitted in descending score order (best first); the training pipeline packs\n  them in that priority order until the 12M-token budget is filled.\n\nSignal source: only the pool itself and the disclosed dev target. No labels, no\ninternet.\n\nOutput: /workspace/submission/selection.json  (ordered list of pool ids)\n\"\"\"\nimport json, re, hashlib, numpy as np\nfrom transformers import AutoTokenizer\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV  = \"/workspace/data/multi_dev.npy\"\nOUT  = \"/workspace/submission/selection.json\"\n\nB = 1 << 20            # hashed feature buckets\nALPHA = 1.0           # add-k smoothing\nWORD_CAP = 600        # max words scored per doc (domain signal is stationary)\nMIN_WORDS = 80        # drop docs too short to be useful prose\nMIN_UCOV = 0.65       # frac of words attested in target vocab (kills foreign/salad)\nMIN_BHIT = 0.25       # frac of bigrams attested in target (kills incoherent spam)\nBI_WEIGHT = 2.0       # emphasise fluency/domain bigrams over raw word density\nSEED = 1337\n\nWORD_RE = re.compile(r\"[a-z0-9]+|[^\\sa-z0-9]\")\nSTOP = set(\"the of and to a in is that it for was as with on be by at this or an \"\n           \"are from his he not but had which have you they were their\".split())\n\n# deterministic word -> bucket (reproducible regardless of PYTHONHASHSEED)\n_cache = {}\ndef wbucket(w):\n    b = _cache.get(w)\n    if b is None:\n        b = int.from_bytes(hashlib.blake2b(w.encode(), digest_size=8).digest(), \"little\") & (B - 1)\n        _cache[w] = b\n    return b\n\ndef features(words):\n    \"\"\"return (unigram_buckets, bigram_buckets) as lists of ints\"\"\"\n    ub = [wbucket(w) for w in words]\n    bb = [((ub[i] * 1000003) ^ ub[i + 1]) & (B - 1) for i in range(len(ub) - 1)]\n    return ub, bb\n\ndef toks(s):\n    return WORD_RE.findall(s.lower())[:WORD_CAP]\n\ndef quality_gate(words):\n    \"\"\"cheap gates: usable English prose, not junk/gibberish/foreign/pure-symbol.\"\"\"\n    n = len(words)\n    if n < MIN_WORDS:\n        return False\n    stop_frac = sum(1 for w in words if w in STOP) / n\n    if stop_frac < 0.06:                      # too few function words -> not English prose\n        return False\n    alpha = sum(1 for w in words if w[:1].isalpha())\n    if alpha / n < 0.5:                       # mostly symbols/numbers\n        return False\n    uniq = len(set(words)) / n\n    if uniq < 0.18:                           # extreme repetition / spam\n        return False\n    return True\n\ndef main():\n    # ---- target model from decoded dev target ----\n    tok = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV).astype(np.int64)\n    dev_text = tok.decode(dev.tolist())\n    cnt_u = np.zeros(B); cnt_b = np.zeros(B)\n    tw = WORD_RE.findall(dev_text.lower())\n    tub, tbb = features(tw)\n    np.add.at(cnt_u, tub, 1.0)\n    np.add.at(cnt_b, tbb, 1.0)\n    attested_u = cnt_u > 0                      # vocab/bigram presence for gates\n    attested_b = cnt_b > 0\n    logpt_u = np.log((cnt_u + ALPHA) / (cnt_u.sum() + ALPHA * B))\n    logpt_b = np.log((cnt_b + ALPHA) / (cnt_b.sum() + ALPHA * B))\n\n    # ---- background pool model from a stratified sample ----\n    logpp_u = np.full(B, ALPHA); logpp_b = np.full(B, ALPHA)\n    with open(POOL) as f:\n        for k, line in enumerate(f):\n            if k % 7:                         # ~1/7 sample for background\n                continue\n            r = json.loads(line)\n            w = toks(r[\"text\"])\n            ub, bb = features(w)\n            np.add.at(logpp_u, ub, 1.0)\n            np.add.at(logpp_b, bb, 1.0)\n    logpp_u = np.log(logpp_u / logpp_u.sum())\n    logpp_b = np.log(logpp_b / logpp_b.sum())\n\n    ru = logpt_u - logpp_u                     # per-bucket log importance ratio\n    rb = logpt_b - logpp_b\n\n    # ---- score every doc ----\n    scored = []\n    seen = set()                               # exact/near-dup guard on normalized prefix\n    with open(POOL) as f:\n        for line in f:\n            r = json.loads(line)\n            did = r[\"id\"]; text = r[\"text\"]\n            w = toks(text)\n            if not quality_gate(w):\n                continue\n            key = hashlib.blake2b(re.sub(r\"\\s+\", \" \", text[:200].lower()).encode(),\n                                  digest_size=8).digest()\n            if key in seen:\n                continue\n            seen.add(key)\n            ub, bb = features(w)\n            # fluency/vocab gates against the target\n            if attested_u[ub].mean() < MIN_UCOV:\n                continue\n            if not bb or attested_b[bb].mean() < MIN_BHIT:\n                continue\n            us = ru[ub].mean()\n            bs = rb[bb].mean()\n            score = us + BI_WEIGHT * bs\n            scored.append((score, did))\n\n    scored.sort(reverse=True)\n    ids = [d for _, d in scored]\n    json.dump(ids, open(OUT, \"w\"))\n    print(f\"scored {len(scored)} docs (passed gates); wrote {len(ids)} ids -> {OUT}\")\n    print(f\"score range: {scored[0][0]:.3f} .. {scored[-1][0]:.3f}\")\n\nif __name__ == \"__main__\":\n    main()\n", "userModified": false}

Command did not complete within its 120s timeout and was moved to the background (ID: bw8vdkj9z). Output is being written to: /tmp/claude-0/-workspace/8d3f2e42-7a38-4920-b1b1-3dc7521e7d7f/tasks/bw8vdkj9z.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bw8vdkj9z", "timedOutAfterMs": 120000, "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}