claude-opus-4-8

5390
---TIMER---
total 14
drwxr-xr-x 1 root root  100 Aug  4 03:22 .
drwxr-xr-x 1 root root  160 Aug  4 03:22 ..
drwxr-xr-x 1 root root   60 Aug  4 03:22 .live
drwxr-xr-x 1 root root  100 Aug  4 03:22 .metrics
drwxr-xr-x 1 root root  140 Aug  4 03:22 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---DATA---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 03:22 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---SUBMISSION---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 03:22 ..
---GPU---
name, memory.total [MiB], memory.used [MiB]
NVIDIA H100 80GB HBM3, 81559 MiB, 4 MiB

[stdout]
5390
---TIMER---
total 14
drwxr-xr-x 1 root root  100 Aug  4 03:22 .
drwxr-xr-x 1 root root  160 Aug  4 03:22 ..
drwxr-xr-x 1 root root   60 Aug  4 03:22 .live
drwxr-xr-x 1 root root  100 Aug  4 03:22 .metrics
drwxr-xr-x 1 root root  140 Aug  4 03:22 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---DATA---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 03:22 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---SUBMISSION---
total 1
drwxr-xr-x 1 root root  10 Aug  3 22:37 .
drwxr-xr-x 1 root root 100 Aug  4 03:22 ..
---GPU---
name, memory.total [MiB], memory.used [MiB]
NVIDIA H100 80GB HBM3, 81559 MiB, 4 MiB
[metadata] {"noOutputExpected": false}

=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")

=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")

=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"

=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done

[stdout]
=== train_nano.py ===
"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
Frozen recipe — the ONLY thing that varies between runs is the training token set."""
import argparse, math, time, numpy as np, torch
from model import GPT, GPTConfig

ap = argparse.ArgumentParser()
ap.add_argument("--train_npy", required=True)
ap.add_argument("--heldout_npy", required=True)
ap.add_argument("--out_json", required=True)
ap.add_argument("--max_iters", type=int, default=3000)
ap.add_argument("--warmup", type=int, default=150)
ap.add_argument("--batch", type=int, default=32)
ap.add_argument("--block", type=int, default=256)
ap.add_argument("--lr", type=float, default=6e-4)
ap.add_argument("--seed", type=int, default=1337)
a = ap.parse_args()

torch.manual_seed(a.seed); np.random.seed(a.seed)
dev = "cuda"
tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
rng = np.random.default_rng(a.seed)

model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
                      n_embd=384, dropout=0.0, bias=False)).to(dev)
opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")

def lr_at(it):
    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)

def get_batch(src):
    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
    return x, y

t0 = time.time()
model.train()
for it in range(a.max_iters):
    for g in opt.param_groups: g["lr"] = lr_at(it)
    x, y = get_batch(tr)
    with torch.autocast("cuda", dtype=torch.bfloat16):
        _, loss = model(x, y)
    opt.zero_grad(set_to_none=True); loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
train_s = time.time() - t0

# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
import json
model.eval()
nwin = (len(ho) - 1) // a.block
starts = [j * a.block for j in range(nwin)]
sum_loss, nseen = 0.0, 0
with torch.no_grad():
    for i in range(0, nwin, a.batch):
        bs = starts[i:i+a.batch]
        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
        with torch.autocast("cuda", dtype=torch.bfloat16):
            _, l = model(x, y)
        sum_loss += l.item() * len(bs); nseen += len(bs)
mean_loss = sum_loss / max(1, nseen)
ppl = math.exp(mean_loss)
out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
json.dump(out, open(a.out_json, "w"))
print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")

=== pack_selection.py ===
"""Solver-side: pack a selection.json (ordered ids) into a 12M-token train array from pool.jsonl."""
import json, sys, numpy as np
from transformers import AutoTokenizer

POOL = "/workspace/data/pool.jsonl"
SEL = sys.argv[1] if len(sys.argv) > 1 else "/workspace/submission/selection.json"
OUT = sys.argv[2] if len(sys.argv) > 2 else "/tmp/dev_train.npy"
BUDGET = 12_000_000

tok = AutoTokenizer.from_pretrained("gpt2"); EOS = tok.eos_token_id
text = {}
for line in open(POOL):
    r = json.loads(line); text[r["id"]] = r["text"]
sel = json.load(open(SEL))
parts, tot = [], 0
for i in sel:
    ids = tok(text[i], add_special_tokens=False).input_ids
    parts.extend(ids); parts.append(EOS); tot += len(ids) + 1
    if tot >= BUDGET: break
arr = np.array(parts[:BUDGET], dtype=np.uint16)
np.save(OUT, arr)
print(f"packed {len(arr)} tokens from {len(sel)} ids -> {OUT}")

=== dev_eval.sh ===
#!/bin/bash
# Diagnostic: train the frozen 30M GPT on your selection and report DEV perplexity.
# Usage: bash dev_eval.sh [selection.json]   (default /workspace/submission/selection.json)
# NOTE: the official target is a DIFFERENT (hidden) sample from the same HQ domain.
set -euo pipefail
export TOKENIZERS_PARALLELISM=false
SEL="${1:-/workspace/submission/selection.json}"
cd /workspace
python3 pack_selection.py "$SEL" /tmp/dev_train.npy
python3 train_nano.py --train_npy /tmp/dev_train.npy --heldout_npy /workspace/data/multi_dev.npy \
    --out_json /tmp/dev.json --max_iters 3000 --seed 1337 2>&1 | grep HELDOUT_PPL
echo "(dev PPL on multi_dev; official uses a hidden disjoint sample of the same HQ domain)"

=== timer.sh ===
#!/usr/bin/env bash
# Budget timer daemon — writes /workspace/.timer/remaining_secs every 10s so the
# solver can poll time left instead of doing wall-clock arithmetic.
#
# Budget (seconds) comes from $TASK_BUDGET_SECS (set by the runner to the agent
# timeout). Idempotent via a PID file. On ScaleTrain the trial runner starts its
# own timer; this file is the fallback for the Modal/local harbor backends, which
# bootstrap /workspace/timer.sh on environment start. The fallback default below
# (7200 = 2 h) matches this task's [agent] timeout_sec and is only used if
# $TASK_BUDGET_SECS is unset.
set -u
TIMER_DIR="/workspace/.timer"
PID_FILE="$TIMER_DIR/timer.pid"
mkdir -p "$TIMER_DIR"

if [ -s "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE" 2>/dev/null)" 2>/dev/null; then
    exit 0   # already running
fi
echo $$ > "$PID_FILE"
trap 'rm -f "$PID_FILE"' EXIT INT TERM

START=$(date +%s)
BUDGET="${TASK_BUDGET_SECS:-7200}"
echo "$START" > "$TIMER_DIR/start_epoch"
echo "$BUDGET" > "$TIMER_DIR/budget_secs"

while true; do
    NOW=$(date +%s)
    ELAPSED=$((NOW - START))
    REMAINING=$((BUDGET - ELAPSED))
    [ "$REMAINING" -lt 0 ] && REMAINING=0
    echo "$REMAINING" > "$TIMER_DIR/remaining_secs"
    echo "$ELAPSED" > "$TIMER_DIR/elapsed_secs"
    [ "$REMAINING" -le 1800 ] && [ ! -f "$TIMER_DIR/alert_30min" ] && touch "$TIMER_DIR/alert_30min"
    [ "$REMAINING" -le 600 ]  && [ ! -f "$TIMER_DIR/alert_10min" ] && touch "$TIMER_DIR/alert_10min"
    [ "$REMAINING" -le 0 ] && break
    sleep 10
done
[metadata] {"noOutputExpected": false}

Command did not complete within its 120s timeout and was moved to the background (ID: bsxngu63b). Output is being written to: /tmp/claude-0/-workspace/fd821a6f-a260-47da-a909-e4a257c63ee5/tasks/bsxngu63b.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bsxngu63b", "timedOutAfterMs": 120000, "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

(Bash completed with no output)

[metadata] {"noOutputExpected": false}

---STATUS---
total 2
drwxr-xr-x 1 root root 25 Aug  3 22:37 .
drwxr-xr-x 1 root root 33 Aug  3 22:37 ..
drwxr-xr-x 1 root root 52 Aug  3 22:37 hub
HF_HOME None

[stdout]
---STATUS---
total 2
drwxr-xr-x 1 root root 25 Aug  3 22:37 .
drwxr-xr-x 1 root root 33 Aug  3 22:37 ..
drwxr-xr-x 1 root root 52 Aug  3 22:37 hub
HF_HOME None
[metadata] {"noOutputExpected": false}

<retrieval_status>success</retrieval_status>

<task_id>bsxngu63b</task_id>

<task_type>local_bash</task_type>

<status>completed</status>

<exit_code>0</exit_code>

<output>
dev target dtype/shape: uint16 (1000000,) total tokens: 1000000
num EOS (doc boundaries): 2704
--- DEV SEG 0 (len 206) ---
 Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , includin

--- DEV SEG 1 (len 125) ---
 On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land 

--- DEV SEG 2 (len 223) ---
 To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were uns

--- DEV SEG 3 (len 95) ---
 Nimrod arrived at McMurdo Sound on 29 January , but was stopped by ice 16 miles ( 26 km ) north of Discovery 's old base at Hut Point . After considerable weather delays , Shackleton 's base was eventually established at Cape Royds , about 24 miles ( 39 km ) north of Hut Point . The party was in high spirits , despite the difficult conditions ; Shackleton 's ability to communicate with each man kept the party happy and focused .  

--- DEV SEG 4 (len 201) ---
 The " Great Southern Journey " , as Frank Wild called it , began on 29 October 1908 . On 9 January 1909 , Shackleton and three companions ( Wild , Eric Marshall and Jameson Adams ) reached a new Farthest South latitude of 88 ° 23 ' S , a point only 112 miles ( 180 km ) from the Pole . En route the South Pole party discovered the Beardmore Glacier ( named after Shackleton 's patron ) and became the first persons to see and travel on the South Polar Plateau . Their return journey to McMurdo Sound was a race against starvation , on half @-@ rations for much of the way . At one point , Shackleton

--- DEV SEG 5 (len 131) ---
 The expedition 's other main accomplishments included the first ascent of Mount Erebus , and the discovery of the approximate location of the South Magnetic Pole , reached on 16 January 1909 by Edgeworth David , Douglas Mawson , and Alistair Mackay . Shackleton returned to the United Kingdom as a hero , and soon afterwards published his expedition account , Heart of the Antarctic . Emily Shackleton later recorded : " The only comment he made to me about not reaching the Pole was ' a live donkey is better than a dead lion , isn 't it ? ' and I said ' Yes darling , as far as I am concerned ' " 

--- DEV SEG 6 (len 23) ---
 In 1910 , Shackleton made a series of three recordings describing the expedition using an Edison Phonograph .  

--- DEV SEG 7 (len 69) ---
 Several mostly intact cases of whisky and brandy left behind in 1909 were recovered in 2010 , for analysis by a distilling company . A revival of the vintage ( and since lost ) formula for the particular brands found has been offered for sale with a portion of the proceeds to benefit the New Zealand Antarctic Heritage Trust which discovered the lost spirits .
</output>

[metadata] {"retrieval_status": "success", "task": {"task_id": "bsxngu63b", "task_type": "local_bash", "status": "completed", "description": "Decode dev target to understand domain", "output": "dev target dtype/shape: uint16 (1000000,) total tokens: 1000000\nnum EOS (doc boundaries): 2704\n--- DEV SEG 0 (len 206) ---\n Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , includin\n\n--- DEV SEG 1 (len 125) ---\n On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land \n\n--- DEV SEG 2 (len 223) ---\n To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were uns\n\n--- DEV SEG 3 (len 95) ---\n Nimrod arrived at McMurdo Sound on 29 January , but was stopped by ice 16 miles ( 26 km ) north of Discovery 's old base at Hut Point . After considerable weather delays , Shackleton 's base was eventually established at Cape Royds , about 24 miles ( 39 km ) north of Hut Point . The party was in high spirits , despite the difficult conditions ; Shackleton 's ability to communicate with each man kept the party happy and focused .  \n\n--- DEV SEG 4 (len 201) ---\n The \" Great Southern Journey \" , as Frank Wild called it , began on 29 October 1908 . On 9 January 1909 , Shackleton and three companions ( Wild , Eric Marshall and Jameson Adams ) reached a new Farthest South latitude of 88 ° 23 ' S , a point only 112 miles ( 180 km ) from the Pole . En route the South Pole party discovered the Beardmore Glacier ( named after Shackleton 's patron ) and became the first persons to see and travel on the South Polar Plateau . Their return journey to McMurdo Sound was a race against starvation , on half @-@ rations for much of the way . At one point , Shackleton\n\n--- DEV SEG 5 (len 131) ---\n The expedition 's other main accomplishments included the first ascent of Mount Erebus , and the discovery of the approximate location of the South Magnetic Pole , reached on 16 January 1909 by Edgeworth David , Douglas Mawson , and Alistair Mackay . Shackleton returned to the United Kingdom as a hero , and soon afterwards published his expedition account , Heart of the Antarctic . Emily Shackleton later recorded : \" The only comment he made to me about not reaching the Pole was ' a live donkey is better than a dead lion , isn 't it ? ' and I said ' Yes darling , as far as I am concerned ' \" \n\n--- DEV SEG 6 (len 23) ---\n In 1910 , Shackleton made a series of three recordings describing the expedition using an Edison Phonograph .  \n\n--- DEV SEG 7 (len 69) ---\n Several mostly intact cases of whisky and brandy left behind in 1909 were recovered in 2010 , for analysis by a distilling company . A revival of the vintage ( and since lost ) formula for the particular brands found has been offered for sale with a portion of the proceeds to benefit the New Zealand Antarctic Heritage Trust which discovered the lost spirits .  \n\n", "exitCode": 0}}

File created successfully at: /workspace/explore.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/explore.py", "content": "import json, numpy as np, random, re, time, collections\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained('gpt2')\n\n# --- dev target: sample segments across the whole array to gauge diversity ---\nho = np.load('/workspace/data/multi_dev.npy'); eos=50256\nidx = np.where(ho==eos)[0]\nprint('DEV tokens', ho.size, 'docs', len(idx))\nprev=0; segs=[]\nfor j in idx:\n    segs.append((prev,j)); prev=j+1\nrandom.seed(0)\npick = sorted(random.sample(range(len(segs)), 10))\nfor k in pick:\n    a,b = segs[k]; txt = tok.decode(ho[a:b].tolist())\n    print(f'[dev {k} len{b-a}] ', txt[:220].replace(chr(10),' '))\nprint('='*80)\n\n# --- pool: stream, gather stats, sample ---\nt0=time.time()\nlens=[]; nwords=[]; samples=[]; ids=[]\nn=0\nWORD=re.compile(r\"\\w+\")\nwith open('/workspace/data/pool.jsonl') as f:\n    for line in f:\n        r=json.loads(line); t=r['text']; ids.append(r['id'])\n        lens.append(len(t)); nwords.append(t.count(' ')+1)\n        if n<200000 and random.random()<0.0002: samples.append((r['id'],t))\n        n+=1\nprint('pool docs', n, 'read_s', round(time.time()-t0,1))\nlens=np.array(lens); nwords=np.array(nwords)\nprint('char len: min/med/mean/max', lens.min(), int(np.median(lens)), int(lens.mean()), lens.max())\nprint('word cnt: min/med/mean/max', nwords.min(), int(np.median(nwords)), int(nwords.mean()), nwords.max())\nprint('est GPT2 tokens total (~1.33*words):', int(1.33*nwords.sum()/1e6),'M  (~0.27*chars):', int(0.27*lens.sum()/1e6),'M')\nprint('id range', min(ids), max(ids), 'unique', len(set(ids)))\nprint('--- random pool samples ---')\nfor i,(did,t) in enumerate(samples[:14]):\n    print(f'[pool {did} chars{len(t)}] ', t[:240].replace(chr(10),' '))\n", "structuredPatch": [], "originalFile": null, "userModified": false}

Command running in background with ID: bu6z21mrx. Output is being written to: /tmp/claude-0/-workspace/fd821a6f-a260-47da-a909-e4a257c63ee5/tasks/bu6z21mrx.output. You will be notified when it completes. To check interim output, use Read on that file path.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "bu6z21mrx"}

(Bash completed with no output)

[metadata] {"noOutputExpected": false}

(Bash completed with no output)

[metadata] {"noOutputExpected": false}

===TIMER===
4961

[stdout]
===TIMER===
4961
[metadata] {"noOutputExpected": false}

[end]

[stdout]
[end]
[metadata] {"noOutputExpected": false}

<retrieval_status>success</retrieval_status>

<task_id>bu6z21mrx</task_id>

<task_type>local_bash</task_type>

<status>completed</status>

<exit_code>0</exit_code>

<output>
DEV tokens 1000000 docs 2704
[dev 165 len78]   On 23 March 1942 , Hitler issued Führer Directive No. 40 , which called for the creation of an " Atlantic Wall " . He ordered naval and submarine bases to be heavily defended . Fortifications remained concentrated aroun
[dev 1060 len220]   In 1963 , Wilder was cast in a leading role in Mother Courage and Her Children , a production starring Anne Bancroft , who introduced Wilder to her boyfriend Mel Brooks . A few months later , Brooks mentioned that he wa
[dev 1242 len190]   The bluntnose stingray or Say 's stingray ( Dasyatis say , often misspelled sayi ) is a species of stingray in the family Dasyatidae , native to the coastal waters of the western Atlantic Ocean from the U.S. state of Ma
[dev 1466 len31]   During the 2001 to 2008 years , GDP grew at an average annual rate of 2 @.@ 125 percent , less than for past business cycles .  
[dev 1577 len44]   After serving as president , Bush began painting as a hobby . Subjects have included dogs and still life . He has also painted self @-@ portraits and portraits of world leaders , including Vladimir Putin and Tony Blair 
[dev 1658 len39]   In addition , most solar panels on spacecraft are also made of high efficient multi @-@ junction photovoltaic cells to derive electricity from sunlight when operating in the inner Solar System .  
[dev 1722 len896]  Today I’m sharing a blackened blue smokey eyes tutorial for hooded eyes. I’m using Black Moon Asteroid as the main eyeshadow for this look. I haven’t done a Teal Thursday in forever, so I thought you’d enjoy this. This s
[dev 1952 len816]  India and the US cannot be "fighting everyday on trade issues while cooperating on defence, former US Ambassador to India Richard Verma has said, highlighting the need to "fire on all cylinders" to move ahead their strat
[dev 1990 len248]  A 20-year-old woman was allegedly raped and then thrown off the fourth floor of an under-construction building in Begumpur area in Rohini in a semi-nude condition by a man who was later arrested, the police said.The cond
[dev 2094 len216]  National Award-winning actress Vidya Balan has skipped the ongoing Indian Film Festival of Melbourne due to her prior commitments and to celebrate her niece and nephew Ira and Ruhaan's sixth birthday on Sunday.Vidya, who
================================================================================
Traceback (most recent call last):
  File "/workspace/explore.py", line 25, in <module>
    for line in f:
OSError: [Errno 22] Invalid argument
</output>

[metadata] {"retrieval_status": "success", "task": {"task_id": "bu6z21mrx", "task_type": "local_bash", "status": "completed", "description": "Run pool exploration", "output": "DEV tokens 1000000 docs 2704\n[dev 165 len78]   On 23 March 1942 , Hitler issued Führer Directive No. 40 , which called for the creation of an \" Atlantic Wall \" . He ordered naval and submarine bases to be heavily defended . Fortifications remained concentrated aroun\n[dev 1060 len220]   In 1963 , Wilder was cast in a leading role in Mother Courage and Her Children , a production starring Anne Bancroft , who introduced Wilder to her boyfriend Mel Brooks . A few months later , Brooks mentioned that he wa\n[dev 1242 len190]   The bluntnose stingray or Say 's stingray ( Dasyatis say , often misspelled sayi ) is a species of stingray in the family Dasyatidae , native to the coastal waters of the western Atlantic Ocean from the U.S. state of Ma\n[dev 1466 len31]   During the 2001 to 2008 years , GDP grew at an average annual rate of 2 @.@ 125 percent , less than for past business cycles .  \n[dev 1577 len44]   After serving as president , Bush began painting as a hobby . Subjects have included dogs and still life . He has also painted self @-@ portraits and portraits of world leaders , including Vladimir Putin and Tony Blair \n[dev 1658 len39]   In addition , most solar panels on spacecraft are also made of high efficient multi @-@ junction photovoltaic cells to derive electricity from sunlight when operating in the inner Solar System .  \n[dev 1722 len896]  Today I’m sharing a blackened blue smokey eyes tutorial for hooded eyes. I’m using Black Moon Asteroid as the main eyeshadow for this look. I haven’t done a Teal Thursday in forever, so I thought you’d enjoy this. This s\n[dev 1952 len816]  India and the US cannot be \"fighting everyday on trade issues while cooperating on defence, former US Ambassador to India Richard Verma has said, highlighting the need to \"fire on all cylinders\" to move ahead their strat\n[dev 1990 len248]  A 20-year-old woman was allegedly raped and then thrown off the fourth floor of an under-construction building in Begumpur area in Rohini in a semi-nude condition by a man who was later arrested, the police said.The cond\n[dev 2094 len216]  National Award-winning actress Vidya Balan has skipped the ongoing Indian Film Festival of Melbourne due to her prior commitments and to celebrate her niece and nephew Ira and Ruhaan's sixth birthday on Sunday.Vidya, who\n================================================================================\nTraceback (most recent call last):\n  File \"/workspace/explore.py\", line 25, in <module>\n    for line in f:\nOSError: [Errno 22] Invalid argument\n", "exitCode": 0}}

binary-iter lines 182016 bytes 809679384 s 1.8

[stdout]
binary-iter lines 182016 bytes 809679384 s 1.8
[metadata] {"noOutputExpected": false}

total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
---
parse-all json lines 182016 s 3.8 sample_words/doc 511

[stdout]
               total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
---
parse-all json lines 182016 s 3.8 sample_words/doc 511
[metadata] {"noOutputExpected": false}

Token indices sequence length is longer than the specified maximum sequence length for this model (1048 > 1024). Running this sequence through the model will result in indexing errors
tok 4000 docs in 4.24 s -> 943 docs/s ; est full pass 193 s
tokens in 4000 docs 2768019 mean/doc 692

[stdout]
Token indices sequence length is longer than the specified maximum sequence length for this model (1048 > 1024). Running this sequence through the model will result in indexing errors
tok 4000 docs in 4.24 s -> 943 docs/s ; est full pass 193 s
tokens in 4000 docs 2768019 mean/doc 692
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/tokenize_pool.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/tokenize_pool.py", "content": "import json, time, re, numpy as np\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained('gpt2')\n\nSTOP = set(\"the a an of to in and or for on at by with from as is are was were be been being this that these those it its he she they we you i his her their our your not no but if then so than into over under out up down about after before\".split())\nWORD = re.compile(r\"[A-Za-z']+\")\nt0 = time.time()\nids=[]; texts=[]\nwith open('/workspace/data/pool.jsonl','rb') as f:\n    for line in f:\n        r = json.loads(line); ids.append(r['id']); texts.append(r['text'])\nn = len(texts); print('loaded', n, 'docs', round(time.time()-t0,1),'s', flush=True)\n\n# --- per-doc raw-text quality features ---\nnchars=np.zeros(n,np.int32); nwords=np.zeros(n,np.int32)\nmean_wlen=np.zeros(n,np.float32); alpha_frac=np.zeros(n,np.float32)\nstop_frac=np.zeros(n,np.float32); symbol_frac=np.zeros(n,np.float32)\ndigit_frac=np.zeros(n,np.float32); uniq_ratio=np.zeros(n,np.float32)\nnonlatin_frac=np.zeros(n,np.float32); upper_frac=np.zeros(n,np.float32)\nfor i,t in enumerate(texts):\n    L=len(t); nchars[i]=L\n    if L==0: continue\n    letters=sum(c.isalpha() for c in t)\n    digits=sum(c.isdigit() for c in t)\n    nonlatin=sum(1 for c in t if ord(c)>0x2fff)  # CJK/other scripts\n    uppers=sum(1 for c in t if c.isupper())\n    words=WORD.findall(t); nw=len(words); nwords[i]=nw\n    if nw:\n        mean_wlen[i]=sum(len(w) for w in words)/nw\n        lw=[w.lower() for w in words]\n        stop_frac[i]=sum(1 for w in lw if w in STOP)/nw\n        head=lw[:500]; uniq_ratio[i]=len(set(head))/len(head)\n        upper_frac[i]=uppers/max(1,letters)\n    alpha_frac[i]=letters/L\n    digit_frac[i]=digits/L\n    nonlatin_frac[i]=nonlatin/L\n    # symbol frac = non-alnum, non-space, non-basic-punct\n    sym=sum(1 for c in t if not (c.isalnum() or c.isspace() or c in \".,;:'\\\"!?()-—–%$&/\"))\n    symbol_frac[i]=sym/L\nprint('features done', round(time.time()-t0,1),'s', flush=True)\n\n# --- GPT-2 tokenization, cached as concatenated uint16 + offsets ---\noffsets=np.zeros(n+1,np.int64)\nchunks=[]; B=8000\nfor s in range(0,n,B):\n    enc=tok(texts[s:s+B], add_special_tokens=False).input_ids\n    for j,e in enumerate(enc):\n        offsets[s+j+1]=offsets[s+j]+len(e)\n        chunks.append(np.asarray(e,dtype=np.uint16))\n    if s % 40000==0: print('tok',s,round(time.time()-t0,1),'s',flush=True)\nalltok=np.concatenate(chunks) if chunks else np.zeros(0,np.uint16)\nprint('total tokens', alltok.size, round(time.time()-t0,1),'s', flush=True)\n\nnp.save('/tmp/pool_ids.npy', np.asarray(ids,np.int64))\nnp.save('/tmp/pool_offsets.npy', offsets)\nnp.save('/tmp/pool_tokens.npy', alltok)\nnp.savez('/tmp/pool_feats.npz', nchars=nchars, nwords=nwords, mean_wlen=mean_wlen,\n         alpha_frac=alpha_frac, stop_frac=stop_frac, symbol_frac=symbol_frac,\n         digit_frac=digit_frac, uniq_ratio=uniq_ratio, nonlatin_frac=nonlatin_frac,\n         upper_frac=upper_frac, ntok=(offsets[1:]-offsets[:-1]).astype(np.int32))\nprint('CACHED OK', round(time.time()-t0,1),'s', flush=True)\n", "structuredPatch": [], "originalFile": null, "userModified": false}

launched pid 1120

[stdout]
launched pid 1120
[metadata] {"returnCodeInterpretation": "No matches found", "noOutputExpected": false}

loaded 182016 docs 6.3 s

[stdout]
loaded 182016 docs 6.3 s
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/score_select.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/score_select.py", "content": "\"\"\"Score pool docs by (quality gate) x (DSIR importance vs disclosed target) and\nselect highest-scoring docs in priority order until the token budget is covered.\"\"\"\nimport json, numpy as np, sys, time\nt0=time.time()\nBUDGET=12_000_000\nTARGET_COVER=16_000_000   # emit enough ids to safely fill budget\n\nids=np.load('/tmp/pool_ids.npy')\noff=np.load('/tmp/pool_offsets.npy')\ntokd=np.load('/tmp/pool_tokens.npy')          # concatenated uint16\nF=np.load('/tmp/pool_feats.npz')\nntok=F['ntok'].astype(np.int64)\nn=len(ids); V=50257; EOS=50256\nprint('loaded cache', n, 'docs', tokd.size,'tokens', round(time.time()-t0,1),'s')\n\n# ---- target unigram from dev (disclosed domain) ----\nho=np.load('/workspace/data/multi_dev.npy').astype(np.int64)\nho=ho[ho!=EOS]\nCt=np.bincount(ho, minlength=V).astype(np.float64)\n# ---- background unigram from full pool ----\nCb=np.bincount(tokd.astype(np.int64), minlength=V).astype(np.float64)\na=1.0\npt=(Ct+a)/(Ct.sum()+a*V); pb=(Cb+a)/(Cb.sum()+a*V)\nL=(np.log(pt)-np.log(pb)).astype(np.float32)   # per-token log importance ratio\n\n# ---- per-doc mean log-ratio (DSIR-lite, length-normalized) ----\nLtok=L[tokd.astype(np.int64)]\ndoc_sum=np.add.reduceat(Ltok, off[:-1].astype(np.int64))\nempty=(ntok==0)\ndoc_mean=np.where(empty,-1e9,doc_sum/np.maximum(ntok,1))\n\n# ---- quality gate features ----\nmean_wlen=F['mean_wlen']; alpha=F['alpha_frac']; stop=F['stop_frac']\nsym=F['symbol_frac']; uniq=F['uniq_ratio']; nonlat=F['nonlatin_frac']; nwords=F['nwords']\n\ndef pct(x,name):\n    q=np.percentile(x,[1,5,25,50,75,95,99])\n    print(f'{name:11s}',' '.join(f'{v:7.3f}' for v in q))\nprint('feature percentiles [1,5,25,50,75,95,99]:')\nfor nm,x in [('mean_wlen',mean_wlen),('alpha',alpha),('stop',stop),('symbol',sym),\n             ('uniq',uniq),('nonlatin',nonlat),('ntok',ntok.astype(float)),('doc_mean',doc_mean)]:\n    pct(x,nm)\n\ngate=( (ntok>=50) & (ntok<=4096) & (nonlat<0.10) & (alpha>0.60) & (sym<0.08)\n     & (mean_wlen>=3.0) & (mean_wlen<=8.5) & (stop>=0.10) & (uniq>=0.35) & (~empty) )\nprint('gate keeps', int(gate.sum()), '/', n, '=', round(100*gate.sum()/n,1),'%')\n\n# ---- rank survivors by DSIR mean, dedup by 64-token prefix hash, fill budget ----\norder=np.argsort(-doc_mean)\nsel=[]; tot=0; seen=set()\nfor i in order:\n    if not gate[i]: continue\n    s=off[i]; e=off[i+1]\n    key=hash(tokd[s:min(e,s+64)].tobytes())\n    if key in seen: continue\n    seen.add(key)\n    sel.append(int(ids[i])); tot+=int(ntok[i])+1\n    if tot>=TARGET_COVER: break\nprint('selected', len(sel), 'ids ~', tot,'tokens (budget', BUDGET,')', round(time.time()-t0,1),'s')\njson.dump(sel, open('/workspace/submission/selection.json','w'))\n# also dump the top order (gated) more fully for potential re-use\nnp.save('/tmp/gated_order.npy', np.array([i for i in order if gate[i]],dtype=np.int64))\nprint('wrote /workspace/submission/selection.json')\n", "structuredPatch": [], "originalFile": null, "userModified": false}

loaded 182016 docs 6.3 s
===TIMER===
4601

[stdout]
loaded 182016 docs 6.3 s
===TIMER===
4601
[metadata] {"noOutputExpected": false}

The file /workspace/tokenize_pool.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"type": "update", "filePath": "/workspace/tokenize_pool.py", "content": "import json, time, re, numpy as np\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained('gpt2')\n\nSTOPB = set(b\"the a an of to in and or for on at by with from as is are was were be been being this that these those it its he she they we you his her their our your not no but if then so than into over under out up down about after before will would can could may\".split())\n# byte deletion tables (C-level counting via translate)\nALL=bytes(range(256))\nLETTERS=bytes(c for c in range(256) if chr(c).isalpha())\nDIGITS=bytes(c for c in range(256) if chr(c).isdigit())\nDEL_NONLETTER=bytes(c for c in range(256) if c not in LETTERS)\nDEL_NONDIGIT=bytes(c for c in range(256) if c not in DIGITS)\nGOOD=set(LETTERS)|set(DIGITS)|set(b\" \\t\\n\\r.,;:'\\\"!?()-%$&/\")  # \"acceptable\" chars\nDEL_GOOD=bytes(c for c in range(256) if c in GOOD)  # deleting these leaves 'symbol' chars\nWORDB=re.compile(rb\"[A-Za-z']+\")\n\nt0=time.time()\nids=[]; texts=[]\nwith open('/workspace/data/pool.jsonl','rb') as f:\n    for line in f:\n        r=json.loads(line); ids.append(r['id']); texts.append(r['text'])\nn=len(texts); print('loaded',n,'docs',round(time.time()-t0,1),'s',flush=True)\n\nnchars=np.zeros(n,np.int32); nwords=np.zeros(n,np.int32)\nmean_wlen=np.zeros(n,np.float32); alpha_frac=np.zeros(n,np.float32)\nstop_frac=np.zeros(n,np.float32); symbol_frac=np.zeros(n,np.float32)\ndigit_frac=np.zeros(n,np.float32); uniq_ratio=np.zeros(n,np.float32)\nnonlatin_frac=np.zeros(n,np.float32)\nfor i,t in enumerate(texts):\n    L=len(t); nchars[i]=L\n    if L==0: continue\n    b=t.encode('utf-8','ignore')\n    bascii=t.encode('ascii','ignore')           # drops non-ascii\n    nonlatin_frac[i]=1.0-len(bascii)/max(1,len(b))\n    letters=len(bascii)-len(bascii.translate(None,DEL_NONLETTER))\n    digits=len(bascii)-len(bascii.translate(None,DEL_NONDIGIT))\n    symbols=len(bascii.translate(None,DEL_GOOD)) # chars NOT in GOOD set\n    alpha_frac[i]=letters/L\n    digit_frac[i]=digits/L\n    symbol_frac[i]=symbols/L\n    words=WORDB.findall(bascii); nw=len(words); nwords[i]=nw\n    if nw:\n        mean_wlen[i]=letters/nw\n        head=words[:400]; low=[w.lower() for w in head]\n        stop_frac[i]=sum(1 for w in low if w in STOPB)/len(low)\n        uniq_ratio[i]=len(set(low))/len(low)\n    if i%40000==0: print('feat',i,round(time.time()-t0,1),'s',flush=True)\nprint('features done',round(time.time()-t0,1),'s',flush=True)\n\noffsets=np.zeros(n+1,np.int64); chunks=[]; B=8000\nfor s in range(0,n,B):\n    enc=tok(texts[s:s+B], add_special_tokens=False).input_ids\n    for j,e in enumerate(enc):\n        offsets[s+j+1]=offsets[s+j]+len(e); chunks.append(np.asarray(e,dtype=np.uint16))\n    if s%40000==0: print('tok',s,round(time.time()-t0,1),'s',flush=True)\nalltok=np.concatenate(chunks) if chunks else np.zeros(0,np.uint16)\nprint('total tokens',alltok.size,round(time.time()-t0,1),'s',flush=True)\n\nnp.save('/tmp/pool_ids.npy',np.asarray(ids,np.int64))\nnp.save('/tmp/pool_offsets.npy',offsets)\nnp.save('/tmp/pool_tokens.npy',alltok)\nnp.savez('/tmp/pool_feats.npz',nchars=nchars,nwords=nwords,mean_wlen=mean_wlen,\n         alpha_frac=alpha_frac,stop_frac=stop_frac,symbol_frac=symbol_frac,\n         digit_frac=digit_frac,uniq_ratio=uniq_ratio,nonlatin_frac=nonlatin_frac,\n         ntok=(offsets[1:]-offsets[:-1]).astype(np.int32))\nprint('CACHED OK',round(time.time()-t0,1),'s',flush=True)\n", "structuredPatch": [{"oldStart": 2, "oldLines": 60, "newStart": 2, "newLines": 64, "lines": [" from transformers import AutoTokenizer", " tok = AutoTokenizer.from_pretrained('gpt2')", " ", "-STOP = set(\"the a an of to in and or for on at by with from as is are was were be been being this that these those it its he she they we you i his her their our your not no but if then so than into over under out up down about after before\".split())", "-WORD = re.compile(r\"[A-Za-z']+\")", "-t0 = time.time()", "+STOPB = set(b\"the a an of to in and or for on at by with from as is are was were be been being this that these those it its he she they we you his her their our your not no but if then so than into over under out up down about after before will would can could may\".split())", "+# byte deletion tables (C-level counting via translate)", "+ALL=bytes(range(256))", "+LETTERS=bytes(c for c in range(256) if chr(c).isalpha())", "+DIGITS=bytes(c for c in range(256) if chr(c).isdigit())", "+DEL_NONLETTER=bytes(c for c in range(256) if c not in LETTERS)", "+DEL_NONDIGIT=bytes(c for c in range(256) if c not in DIGITS)", "+GOOD=set(LETTERS)|set(DIGITS)|set(b\" \\t\\n\\r.,;:'\\\"!?()-%$&/\")  # \"acceptable\" chars", "+DEL_GOOD=bytes(c for c in range(256) if c in GOOD)  # deleting these leaves 'symbol' chars", "+WORDB=re.compile(rb\"[A-Za-z']+\")", "+", "+t0=time.time()", " ids=[]; texts=[]", " with open('/workspace/data/pool.jsonl','rb') as f:", "     for line in f:", "-        r = json.loads(line); ids.append(r['id']); texts.append(r['text'])", "-n = len(texts); print('loaded', n, 'docs', round(time.time()-t0,1),'s', flush=True)", "+        r=json.loads(line); ids.append(r['id']); texts.append(r['text'])", "+n=len(texts); print('loaded',n,'docs',round(time.time()-t0,1),'s',flush=True)", " ", "-# --- per-doc raw-text quality features ---", " nchars=np.zeros(n,np.int32); nwords=np.zeros(n,np.int32)", " mean_wlen=np.zeros(n,np.float32); alpha_frac=np.zeros(n,np.float32)", " stop_frac=np.zeros(n,np.float32); symbol_frac=np.zeros(n,np.float32)", " digit_frac=np.zeros(n,np.float32); uniq_ratio=np.zeros(n,np.float32)", "-nonlatin_frac=np.zeros(n,np.float32); upper_frac=np.zeros(n,np.float32)", "+nonlatin_frac=np.zeros(n,np.float32)", " for i,t in enumerate(texts):", "     L=len(t); nchars[i]=L", "     if L==0: continue", "-    letters=sum(c.isalpha() for c in t)", "-    digits=sum(c.isdigit() for c in t)", "-    nonlatin=sum(1 for c in t if ord(c)>0x2fff)  # CJK/other scripts", "-    uppers=sum(1 for c in t if c.isupper())", "-    words=WORD.findall(t); nw=len(words); nwords[i]=nw", "-    if nw:", "-        mean_wlen[i]=sum(len(w) for w in words)/nw", "-        lw=[w.lower() for w in words]", "-        stop_frac[i]=sum(1 for w in lw if w in STOP)/nw", "-        head=lw[:500]; uniq_ratio[i]=len(set(head))/len(head)", "-        upper_frac[i]=uppers/max(1,letters)", "+    b=t.encode('utf-8','ignore')", "+    bascii=t.encode('ascii','ignore')           # drops non-ascii", "+    nonlatin_frac[i]=1.0-len(bascii)/max(1,len(b))", "+    letters=len(bascii)-len(bascii.translate(None,DEL_NONLETTER))", "+    digits=len(bascii)-len(bascii.translate(None,DEL_NONDIGIT))", "+    symbols=len(bascii.translate(None,DEL_GOOD)) # chars NOT in GOOD set", "     alpha_frac[i]=letters/L", "     digit_frac[i]=digits/L", "-    nonlatin_frac[i]=nonlatin/L", "-    # symbol frac = non-alnum, non-space, non-basic-punct", "-    sym=sum(1 for c in t if not (c.isalnum() or c.isspace() or c in \".,;:'\\\"!?()-—–%$&/\"))", "-    symbol_frac[i]=sym/L", "-print('features done', round(time.time()-t0,1),'s', flush=True)", "+    symbol_frac[i]=symbols/L", "+    words=WORDB.findall(bascii); nw=len(words); nwords[i]=nw", "+    if nw:", "+        mean_wlen[i]=letters/nw", "+        head=words[:400]; low=[w.lower() for w in head]", "+        stop_frac[i]=sum(1 for w in low if w in STOPB)/len(low)", "+        uniq_ratio[i]=len(set(low))/len(low)", "+    if i%40000==0: print('feat',i,round(time.time()-t0,1),'s',flush=True)", "+print('features done',round(time.time()-t0,1),'s',flush=True)", " ", "-# --- GPT-2 tokenization, cached as concatenated uint16 + offsets ---", "-offsets=np.zeros(n+1,np.int64)", "-chunks=[]; B=8000", "+offsets=np.zeros(n+1,np.int64); chunks=[]; B=8000", " for s in range(0,n,B):", "     enc=tok(texts[s:s+B], add_special_tokens=False).input_ids", "     for j,e in enumerate(enc):", "-        offsets[s+j+1]=offsets[s+j]+len(e)", "-        chunks.append(np.asarray(e,dtype=np.uint16))", "-    if s % 40000==0: print('tok',s,round(time.time()-t0,1),'s',flush=True)", "+        offsets[s+j+1]=offsets[s+j]+len(e); chunks.append(np.asarray(e,dtype=np.uint16))", "+    if s%40000==0: print('tok',s,round(time.time()-t0,1),'s',flush=True)", " alltok=np.concatenate(chunks) if chunks else np.zeros(0,np.uint16)", "-print('total tokens', alltok.size, round(time.time()-t0,1),'s', flush=True)", "+print('total tokens',alltok.size,round(time.time()-t0,1),'s',flush=True)", " ", "-np.save('/tmp/pool_ids.npy', np.asarray(ids,np.int64))", "-np.save('/tmp/pool_offsets.npy', offsets)", "-np.save('/tmp/pool_tokens.npy', alltok)", "-np.savez('/tmp/pool_feats.npz', nchars=nchars, nwords=nwords, mean_wlen=mean_wlen,", "-         alpha_frac=alpha_frac, stop_frac=stop_frac, symbol_frac=symbol_frac,", "-         digit_frac=digit_frac, uniq_ratio=uniq_ratio, nonlatin_frac=nonlatin_frac,", "-         upper_frac=upper_frac, ntok=(offsets[1:]-offsets[:-1]).astype(np.int32))", "-print('CACHED OK', round(time.time()-t0,1),'s', flush=True)", "+np.save('/tmp/pool_ids.npy',np.asarray(ids,np.int64))", "+np.save('/tmp/pool_offsets.npy',offsets)", "+np.save('/tmp/pool_tokens.npy',alltok)", "+np.savez('/tmp/pool_feats.npz',nchars=nchars,nwords=nwords,mean_wlen=mean_wlen,", "+         alpha_frac=alpha_frac,stop_frac=stop_frac,symbol_frac=symbol_frac,", "+         digit_frac=digit_frac,uniq_ratio=uniq_ratio,nonlatin_frac=nonlatin_frac,", "+         ntok=(offsets[1:]-offsets[:-1]).astype(np.int32))", "+print('CACHED OK',round(time.time()-t0,1),'s',flush=True)"]}], "originalFile": "import json, time, re, numpy as np\nfrom transformers import AutoTokenizer\ntok = AutoTokenizer.from_pretrained('gpt2')\n\nSTOP = set(\"the a an of to in and or for on at by with from as is are was were be been being this that these those it its he she they we you i his her their our your not no but if then so than into over under out up down about after before\".split())\nWORD = re.compile(r\"[A-Za-z']+\")\nt0 = time.time()\nids=[]; texts=[]\nwith open('/workspace/data/pool.jsonl','rb') as f:\n    for line in f:\n        r = json.loads(line); ids.append(r['id']); texts.append(r['text'])\nn = len(texts); print('loaded', n, 'docs', round(time.time()-t0,1),'s', flush=True)\n\n# --- per-doc raw-text quality features ---\nnchars=np.zeros(n,np.int32); nwords=np.zeros(n,np.int32)\nmean_wlen=np.zeros(n,np.float32); alpha_frac=np.zeros(n,np.float32)\nstop_frac=np.zeros(n,np.float32); symbol_frac=np.zeros(n,np.float32)\ndigit_frac=np.zeros(n,np.float32); uniq_ratio=np.zeros(n,np.float32)\nnonlatin_frac=np.zeros(n,np.float32); upper_frac=np.zeros(n,np.float32)\nfor i,t in enumerate(texts):\n    L=len(t); nchars[i]=L\n    if L==0: continue\n    letters=sum(c.isalpha() for c in t)\n    digits=sum(c.isdigit() for c in t)\n    nonlatin=sum(1 for c in t if ord(c)>0x2fff)  # CJK/other scripts\n    uppers=sum(1 for c in t if c.isupper())\n    words=WORD.findall(t); nw=len(words); nwords[i]=nw\n    if nw:\n        mean_wlen[i]=sum(len(w) for w in words)/nw\n        lw=[w.lower() for w in words]\n        stop_frac[i]=sum(1 for w in lw if w in STOP)/nw\n        head=lw[:500]; uniq_ratio[i]=len(set(head))/len(head)\n        upper_frac[i]=uppers/max(1,letters)\n    alpha_frac[i]=letters/L\n    digit_frac[i]=digits/L\n    nonlatin_frac[i]=nonlatin/L\n    # symbol frac = non-alnum, non-space, non-basic-punct\n    sym=sum(1 for c in t if not (c.isalnum() or c.isspace() or c in \".,;:'\\\"!?()-—–%$&/\"))\n    symbol_frac[i]=sym/L\nprint('features done', round(time.time()-t0,1),'s', flush=True)\n\n# --- GPT-2 tokenization, cached as concatenated uint16 + offsets ---\noffsets=np.zeros(n+1,np.int64)\nchunks=[]; B=8000\nfor s in range(0,n,B):\n    enc=tok(texts[s:s+B], add_special_tokens=False).input_ids\n    for j,e in enumerate(enc):\n        offsets[s+j+1]=offsets[s+j]+len(e)\n        chunks.append(np.asarray(e,dtype=np.uint16))\n    if s % 40000==0: print('tok',s,round(time.time()-t0,1),'s',flush=True)\nalltok=np.concatenate(chunks) if chunks else np.zeros(0,np.uint16)\nprint('total tokens', alltok.size, round(time.time()-t0,1),'s', flush=True)\n\nnp.save('/tmp/pool_ids.npy', np.asarray(ids,np.int64))\nnp.save('/tmp/pool_offsets.npy', offsets)\nnp.save('/tmp/pool_tokens.npy', alltok)\nnp.savez('/tmp/pool_feats.npz', nchars=nchars, nwords=nwords, mean_wlen=mean_wlen,\n         alpha_frac=alpha_frac, stop_frac=stop_frac, symbol_frac=symbol_frac,\n         digit_frac=digit_frac, uniq_ratio=uniq_ratio, nonlatin_frac=nonlatin_frac,\n         upper_frac=upper_frac, ntok=(offsets[1:]-offsets[:-1]).astype(np.int32))\nprint('CACHED OK', round(time.time()-t0,1),'s', flush=True)\n", "userModified": false}

pid 1399
loaded 182016 docs 7.0 s
feat 0 7.0 s
feat 40000 11.0 s
feat 80000 15.2 s
feat 120000 19.8 s

[stdout]
pid 1399
loaded 182016 docs 7.0 s
feat 0 7.0 s
feat 40000 11.0 s
feat 80000 15.2 s
feat 120000 19.8 s
[metadata] {"noOutputExpected": false}