claude-opus-5

5400
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 00:56 .
drwxr-xr-x 1 root root  160 Aug  4 00:56 ..
drwxr-xr-x 1 root root   60 Aug  4 00:56 .live
drwxr-xr-x 1 root root  100 Aug  4 00:56 .metrics
drwxr-xr-x 1 root root  140 Aug  4 00:56 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 00:56 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
Tue Aug  4 00:56:14 2026       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 580.95.05              Driver Version: 580.95.05      CUDA Version: 13.0     |
+-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 80GB HBM3          On  |   00000000:04:00.0 Off |                    0 |
| N/A   30C    P0             69W /  700W |       4MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|  No running processes found                                                             |
+-----------------------------------------------------------------------------------------+

[stdout]
5400
---
total 14
drwxr-xr-x 1 root root  100 Aug  4 00:56 .
drwxr-xr-x 1 root root  160 Aug  4 00:56 ..
drwxr-xr-x 1 root root   60 Aug  4 00:56 .live
drwxr-xr-x 1 root root  100 Aug  4 00:56 .metrics
drwxr-xr-x 1 root root  140 Aug  4 00:56 .timer
drwxr-xr-x 1 root root   57 Aug  3 22:37 data
-rwxrwxr-x 1 root root  714 Aug  3 22:34 dev_eval.sh
-rw-rw-r-- 1 root root 5706 Aug  3 22:34 model.py
-rw-rw-r-- 1 root root  869 Aug  3 22:34 pack_selection.py
drwxr-xr-x 1 root root   10 Aug  3 22:37 submission
-rwxrwxr-x 1 root root 1509 Aug  3 22:34 timer.sh
-rw-rw-r-- 1 root root 2962 Aug  3 22:34 train_nano.py
---
total 792657
drwxr-xr-x 1 root root        57 Aug  3 22:37 .
drwxr-xr-x 1 root root       100 Aug  4 00:56 ..
-r--r--r-- 1 root root   2000128 Aug  3 22:34 multi_dev.npy
-r--r--r-- 1 root root 809679384 Aug  3 22:35 pool.jsonl
---
Tue Aug  4 00:56:14 2026       
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 580.95.05              Driver Version: 580.95.05      CUDA Version: 13.0     |
+-----------------------------------------+------------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id          Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |           Memory-Usage | GPU-Util  Compute M. |
|                                         |                        |               MIG M. |
|=========================================+========================+======================|
|   0  NVIDIA H100 80GB HBM3          On  |   00000000:04:00.0 Off |                    0 |
| N/A   30C    P0             69W /  700W |       4MiB /  81559MiB |      0%      Default |
|                                         |                        |             Disabled |
+-----------------------------------------+------------------------+----------------------+

+-----------------------------------------------------------------------------------------+
| Processes:                                                                              |
|  GPU   GI   CI              PID   Type   Process name                        GPU Memory |
|        ID   ID                                                               Usage      |
|=========================================================================================|
|  No running processes found                                                             |
+-----------------------------------------------------------------------------------------+
[metadata] {"noOutputExpected": false}

1	"""From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.
2	Frozen recipe — the ONLY thing that varies between runs is the training token set."""
3	import argparse, math, time, numpy as np, torch
4	from model import GPT, GPTConfig
5	
6	ap = argparse.ArgumentParser()
7	ap.add_argument("--train_npy", required=True)
8	ap.add_argument("--heldout_npy", required=True)
9	ap.add_argument("--out_json", required=True)
10	ap.add_argument("--max_iters", type=int, default=3000)
11	ap.add_argument("--warmup", type=int, default=150)
12	ap.add_argument("--batch", type=int, default=32)
13	ap.add_argument("--block", type=int, default=256)
14	ap.add_argument("--lr", type=float, default=6e-4)
15	ap.add_argument("--seed", type=int, default=1337)
16	a = ap.parse_args()
17	
18	torch.manual_seed(a.seed); np.random.seed(a.seed)
19	dev = "cuda"
20	tr = np.load(a.train_npy); ho = np.load(a.heldout_npy)
21	tr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))
22	rng = np.random.default_rng(a.seed)
23	
24	model = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,
25	                      n_embd=384, dropout=0.0, bias=False)).to(dev)
26	opt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), "cuda")
27	
28	def lr_at(it):
29	    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)
30	    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)
31	    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)
32	
33	def get_batch(src):
34	    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)
35	    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)
36	    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)
37	    return x, y
38	
39	t0 = time.time()
40	model.train()
41	for it in range(a.max_iters):
42	    for g in opt.param_groups: g["lr"] = lr_at(it)
43	    x, y = get_batch(tr)
44	    with torch.autocast("cuda", dtype=torch.bfloat16):
45	        _, loss = model(x, y)
46	    opt.zero_grad(set_to_none=True); loss.backward()
47	    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()
48	train_s = time.time() - t0
49	
50	# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)
51	import json
52	model.eval()
53	nwin = (len(ho) - 1) // a.block
54	starts = [j * a.block for j in range(nwin)]
55	sum_loss, nseen = 0.0, 0
56	with torch.no_grad():
57	    for i in range(0, nwin, a.batch):
58	        bs = starts[i:i+a.batch]
59	        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)
60	        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)
61	        with torch.autocast("cuda", dtype=torch.bfloat16):
62	            _, l = model(x, y)
63	        sum_loss += l.item() * len(bs); nseen += len(bs)
64	mean_loss = sum_loss / max(1, nseen)
65	ppl = math.exp(mean_loss)
66	out = {"train_npy": a.train_npy, "heldout_ppl": ppl, "mean_loss": mean_loss,
67	       "max_iters": a.max_iters, "seed": a.seed, "train_s": round(train_s, 1)}
68	json.dump(out, open(a.out_json, "w"))
69	print(f"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}")
70

[metadata] {"type": "text", "file": {"filePath": "/workspace/train_nano.py", "content": "\"\"\"From-scratch nano-GPT (30M) LM training on a 1D token .npy; report held-out perplexity.\nFrozen recipe — the ONLY thing that varies between runs is the training token set.\"\"\"\nimport argparse, math, time, numpy as np, torch\nfrom model import GPT, GPTConfig\n\nap = argparse.ArgumentParser()\nap.add_argument(\"--train_npy\", required=True)\nap.add_argument(\"--heldout_npy\", required=True)\nap.add_argument(\"--out_json\", required=True)\nap.add_argument(\"--max_iters\", type=int, default=3000)\nap.add_argument(\"--warmup\", type=int, default=150)\nap.add_argument(\"--batch\", type=int, default=32)\nap.add_argument(\"--block\", type=int, default=256)\nap.add_argument(\"--lr\", type=float, default=6e-4)\nap.add_argument(\"--seed\", type=int, default=1337)\na = ap.parse_args()\n\ntorch.manual_seed(a.seed); np.random.seed(a.seed)\ndev = \"cuda\"\ntr = np.load(a.train_npy); ho = np.load(a.heldout_npy)\ntr = torch.from_numpy(tr.astype(np.int64)); ho = torch.from_numpy(ho.astype(np.int64))\nrng = np.random.default_rng(a.seed)\n\nmodel = GPT(GPTConfig(block_size=a.block, vocab_size=50257, n_layer=6, n_head=6,\n                      n_embd=384, dropout=0.0, bias=False)).to(dev)\nopt = model.configure_optimizers(0.1, a.lr, (0.9, 0.95), \"cuda\")\n\ndef lr_at(it):\n    if it < a.warmup: return a.lr * (it + 1) / (a.warmup + 1)\n    r = (it - a.warmup) / max(1, a.max_iters - a.warmup)\n    return 0.1 * a.lr + 0.5 * (1 + math.cos(math.pi * r)) * (a.lr - 0.1 * a.lr)\n\ndef get_batch(src):\n    ix = rng.integers(0, len(src) - a.block - 1, size=a.batch)\n    x = torch.stack([src[i:i+a.block] for i in ix]).to(dev)\n    y = torch.stack([src[i+1:i+1+a.block] for i in ix]).to(dev)\n    return x, y\n\nt0 = time.time()\nmodel.train()\nfor it in range(a.max_iters):\n    for g in opt.param_groups: g[\"lr\"] = lr_at(it)\n    x, y = get_batch(tr)\n    with torch.autocast(\"cuda\", dtype=torch.bfloat16):\n        _, loss = model(x, y)\n    opt.zero_grad(set_to_none=True); loss.backward()\n    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0); opt.step()\ntrain_s = time.time() - t0\n\n# held-out perplexity: mean next-token CE over non-overlapping windows (window-weighted)\nimport json\nmodel.eval()\nnwin = (len(ho) - 1) // a.block\nstarts = [j * a.block for j in range(nwin)]\nsum_loss, nseen = 0.0, 0\nwith torch.no_grad():\n    for i in range(0, nwin, a.batch):\n        bs = starts[i:i+a.batch]\n        x = torch.stack([ho[s:s+a.block] for s in bs]).to(dev)\n        y = torch.stack([ho[s+1:s+1+a.block] for s in bs]).to(dev)\n        with torch.autocast(\"cuda\", dtype=torch.bfloat16):\n            _, l = model(x, y)\n        sum_loss += l.item() * len(bs); nseen += len(bs)\nmean_loss = sum_loss / max(1, nseen)\nppl = math.exp(mean_loss)\nout = {\"train_npy\": a.train_npy, \"heldout_ppl\": ppl, \"mean_loss\": mean_loss,\n       \"max_iters\": a.max_iters, \"seed\": a.seed, \"train_s\": round(train_s, 1)}\njson.dump(out, open(a.out_json, \"w\"))\nprint(f\"HELDOUT_PPL {ppl:.3f}  loss {mean_loss:.4f}  train_s {train_s:.0f}  {a.train_npy}\")\n", "numLines": 70, "startLine": 1, "totalLines": 70}}

uint16 (1000000,) 0 50256
" Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member of the Royal Victorian Order , 4th Class ( MVO ; the present @-@ day grade of Lieutenant ) . \n<|endoftext|> On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land . \n<|endoftext|> To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were unstable , precluding the establishment of a safe base there . An extended search for an anchorage at King Edward VII Land proved equally fruitless , so Shackleton was forced to break his undertaking to Scott and set sail for McMur"

[stdout]
uint16 (1000000,) 0 50256
" Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society 's newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member of the Royal Victorian Order , 4th Class ( MVO ; the present @-@ day grade of Lieutenant ) . \n<|endoftext|> On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton 's original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott that he would not base himself in the McMurdo area , which Scott was claiming as his own field of work . Shackleton reluctantly agreed to look for winter quarters at either the Barrier Inlet ( which Discovery had briefly visited in 1902 ) or King Edward VII Land . \n<|endoftext|> To conserve coal , the ship was towed 1 @,@ 650 miles ( 2 @,@ 655 km ) by the steamer Koonya to the Antarctic ice , after Shackleton had persuaded the New Zealand government and the Union Steamship Company to share the cost . In accordance with Shackleton 's promise to Scott , the ship headed for the eastern sector of the Great Ice Barrier , arriving there on 21 January 1908 . They found that the Barrier Inlet had expanded to form a large bay , in which were hundreds of whales , which led to the immediate christening of the area as the Bay of Whales . It was noted that ice conditions were unstable , precluding the establishment of a safe base there . An extended search for an anchorage at King Edward VII Land proved equally fruitless , so Shackleton was forced to break his undertaking to Scott and set sail for McMur"
[metadata] {"noOutputExpected": false}

=== OFFSET 0
' Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society \'s newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member of the Royal Victorian Order , 4th Class ( MVO ; the present @-@ day grade of Lieutenant ) . \n<|endoftext|> On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton \'s original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott tha

=== OFFSET 200000
' it followed the southern end of the Lodge Freeway . By the middle of 1961 , the Watervliet – Paw Paw and Jackson – Ann Arbor freeway gaps were completed , and the freeway was extended westward to Stevensville ; By the end of the year , I @-@ 94 / US 12 extended all the way to New Buffalo . In January 1962 , the state made the biggest rerouting change of all to US 12 : the designation was removed from the I @-@ 94 freeway from New Buffalo to Detroit and shifted to completely replace US 112 . \n<|endoftext|> In 1925 , US 112 was originally proposed to run from Oshkosh to Fremont , Wisconsin , on what later became U.S. Route 110 . When it was initially designated in November 1926 , US 112 made a sharp turn to the southwest to connect to US 20 in Elkhart , Indiana . In 1931 , a new trunkline highway was designated between M @-@ 60 at Niles and US 112 at Union . This highway was numbered M @-@ 151 . In 1933 , the section of US 112 from Union to Elkhart was renumbered US 112S . M @-@ 151 and US 112S each lasted until 1935 when US 112 was extended to replace M @-@ 151 . US 112 was also extended to run concurrently with M @-@ 60 to New Buffalo , and US 112S was renumbered M @-@ 205 . \n<|endoftext|> In 1936 , the section of US 112 along Michigan Avenue east of Ypsilanti was expanded into a " super highway " . In 1955 , a realignment of US 127 in southern Jackson County removed a shor

=== OFFSET 400000
' bite out of Walker\'s counting stats. He retired with 2,160 hits, 383 home runs and 230 stolen bases and that\'s very impressive, but those totals don\'t make you think "Hall of Famer" at first glance. Had he stayed healthy, Walker might have been able to eclipse 2,500 hits, 450 homers and 300 steals. Now those numbers grab your attention. The injuries hurt Walker\'s bulk production. No doubt about it.\n\nCoors Field: Walker played most of his career with the Rockies, which means he benefited from hitter friendly Coors Field. He was a career .381/.462/.710 hitter at Coors Field (!) and a career .282/.375/.501 hitter away from Coors Field. That\'s still really good! But clearly Walker\'s offensive stats were inflated by the thin mountain air.\n\nIt\'s important to keep in mind only 2,501 of Walker\'s 8,030 career plate appearances came at Coors Field, or 31.1 percent. Nearly 70 percent of his career plate appearances came elsewhere, so it\'s not like his career numbers are solely the product of that ballpark. He wasn\'t Ted Williams at Coors Field and Neifi Perez elsewhere, you know? Playing at Coors Field undeniably boosted Walker\'s stats. The man was great everywhere he played though.\n\nWill he make it?\n\nThis is Walker\'s seventh year on the Hall of Fame ballot and he topped out at 22.9 percent of the vote back in 2009. According to Ryan Thibodaux\'s tracker, Walker has 

=== OFFSET 600000
' flexibility to employees and saves seating space for the employer, amongst many other benefits Working from Home entails. However, many employees often get caught up with the comfort a WFH option provides and resultantly deliver poor productivity. If you too are availing the Work from Home option or work from a Home-Office then here are 6 proven ways to optimize your productivity:1. Designate Time SlotsRemember Work from Home doesn’t shorten your work or work hours. Designate time slot(s) in the morning, afternoon or evening and stick to them if you really want to be productive. Likewise, schedule breaks in between your work hours to unwind.2. Assign a Tidy CornerAssign a tidy corner for working every day. Invest in an ergonomic chair and table to work for long hours in the right posture. Sitting on a sofa or bed all day will harm your back besides making you slow. Also, ensure this corner is not the place you’ll have breakfast, lunch or dinner. You need some change even if it means changing places within your boundary wall.3. CommunicateKeep communicating with your team members over the phone or messenger or emails to keep yourself engaged and focused at work. Similarly, limit personal chats or talks during your designated work time.4. Dress UpTake a bath and dress up in the morning to get into the professional mindset. Working in your PJs may appear cool but it hampers your

=== OFFSET 800000
" true\n        };\n        client.Send(&quot;MyEmailAddress@gmail.com&quot;, &quot;some.email@some.com&quot;, &quot;test&quot;, &quot;testbody&quot;); \n    }\n</code></pre>\n<p>Any ideas?</p>\n<p><strong>UPDATE</strong></p>\n<p>More details.</p>\n<p>Maybe I should say what other attempts I made that gave me the same error:\n(Note when i didn't specify a port it tryed port 25)</p>\n<pre><code>    public static void Attempt2()\n    {\n        var fromAddress = new MailAddress(&quot;MyEmailAddy@gmail.com&quot;, &quot;From Name&quot;);\n        var toAddress = new MailAddress(&quot;MyEmailAddy@dfdf.com&quot;, &quot;To Name&quot;);\n        const string fromPassword = &quot;pass&quot;;\n        const string subject = &quot;Subject&quot;;\n        const string body = &quot;Body&quot;;\n        var smtp = new SmtpClient\n        {\n            Host = &quot;smtp.gmail.com&quot;,\n            Port = 587,\n            EnableSsl = true,\n            DeliveryMethod = SmtpDeliveryMethod.Network,\n            UseDefaultCredentials = false,\n            Credentials = new NetworkCredential(fromAddress.Address, fromPassword)\n        };\n        using (var message = new MailMessage(fromAddress, toAddress)\n        {\n            Subject = subject,\n            Body = body\n        }\n            ) { smtp.Send(message); }\n    }\n\n\n    public static void Attempt3()\n    {\n        MailMessag

=== OFFSET 950000
' to the debug perspective, I can only see the class instance (on which the function was called) and the two parameters. I cannot expand the class instance to see the values of its member variables. There is a space to the left of the instance so I would assume that there should be an arrow there so one is able to expand it in a similar way as in the Outline.</p>\n\n<p>Is there a configuration or something I must enable for this? Or have I misunderstood the variables window?</p>\n\n<p>If it matters, this is Eclipse 3.2.2 in Ubuntu Linux.</p>\n\n<hr>\n\n<p><strong>[Update]</strong>\nI downloaded a new release from <a href="http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/galileo/R/eclipse-java-galileo-linux-gtk.tar.gz" rel="nofollow noreferrer">http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/galileo/R/eclipse-java-galileo-linux-gtk.tar.gz</a></p>\n\n<p>"About Eclipse" now reports "Build id: 20090619-0625".</p>\n\n<p>Testcase:</p>\n\n<pre><code>class Foo {\n    private int bar;\n\n    Foo() {\n        bar = 1;\n    }\n\n    public void set(int newbar) {\n        assert (newbar&lt;0);\n        bar = newbar;\n    }\n}\n\nclass Test {\n\n     public static void main (String[] args){\n         Foo f = new Foo();\n         f.set(5);\n     }\n}\n</code></pre>\n\n<p>Obviously, this code produces the assertion excep

[stdout]
=== OFFSET 0
' Beardmore was sufficiently impressed with Shackleton to offer financial support , but other donations proved hard to come by . Nevertheless , in February 1907 , Shackleton presented to the Royal Geographic Society his plans for an Antarctic expedition , the details of which , under the name British Antarctic Expedition , were published in the Royal Society \'s newsletter , Geographic Journal . The aim was the conquest of both the geographical South Pole and the South Magnetic Pole . Shackleton then worked hard to persuade others of his wealthy friends and acquaintances to contribute , including Sir Philip Lee Brocklehurst , who subscribed £ 2 @,@ 000 ( 2011 equivalent £ 157 @,@ 000 ) to secure a place on the expedition ; author Campbell Mackellar ; and Guinness baron Lord Iveagh , whose contribution was secured less than two weeks before the departure of the expedition ship Nimrod . On 4 August 1907 , Shackleton was appointed a Member of the Royal Victorian Order , 4th Class ( MVO ; the present @-@ day grade of Lieutenant ) . \n<|endoftext|> On 1 January 1908 , Nimrod sailed for the Antarctic from Lyttelton Harbour , New Zealand . Shackleton \'s original plans had envisaged using the old Discovery base in McMurdo Sound to launch his attempts on the South Pole and South Magnetic Pole . However , before leaving England , he had been pressured to give an undertaking to Scott tha

=== OFFSET 200000
' it followed the southern end of the Lodge Freeway . By the middle of 1961 , the Watervliet – Paw Paw and Jackson – Ann Arbor freeway gaps were completed , and the freeway was extended westward to Stevensville ; By the end of the year , I @-@ 94 / US 12 extended all the way to New Buffalo . In January 1962 , the state made the biggest rerouting change of all to US 12 : the designation was removed from the I @-@ 94 freeway from New Buffalo to Detroit and shifted to completely replace US 112 . \n<|endoftext|> In 1925 , US 112 was originally proposed to run from Oshkosh to Fremont , Wisconsin , on what later became U.S. Route 110 . When it was initially designated in November 1926 , US 112 made a sharp turn to the southwest to connect to US 20 in Elkhart , Indiana . In 1931 , a new trunkline highway was designated between M @-@ 60 at Niles and US 112 at Union . This highway was numbered M @-@ 151 . In 1933 , the section of US 112 from Union to Elkhart was renumbered US 112S . M @-@ 151 and US 112S each lasted until 1935 when US 112 was extended to replace M @-@ 151 . US 112 was also extended to run concurrently with M @-@ 60 to New Buffalo , and US 112S was renumbered M @-@ 205 . \n<|endoftext|> In 1936 , the section of US 112 along Michigan Avenue east of Ypsilanti was expanded into a " super highway " . In 1955 , a realignment of US 127 in southern Jackson County removed a shor

=== OFFSET 400000
' bite out of Walker\'s counting stats. He retired with 2,160 hits, 383 home runs and 230 stolen bases and that\'s very impressive, but those totals don\'t make you think "Hall of Famer" at first glance. Had he stayed healthy, Walker might have been able to eclipse 2,500 hits, 450 homers and 300 steals. Now those numbers grab your attention. The injuries hurt Walker\'s bulk production. No doubt about it.\n\nCoors Field: Walker played most of his career with the Rockies, which means he benefited from hitter friendly Coors Field. He was a career .381/.462/.710 hitter at Coors Field (!) and a career .282/.375/.501 hitter away from Coors Field. That\'s still really good! But clearly Walker\'s offensive stats were inflated by the thin mountain air.\n\nIt\'s important to keep in mind only 2,501 of Walker\'s 8,030 career plate appearances came at Coors Field, or 31.1 percent. Nearly 70 percent of his career plate appearances came elsewhere, so it\'s not like his career numbers are solely the product of that ballpark. He wasn\'t Ted Williams at Coors Field and Neifi Perez elsewhere, you know? Playing at Coors Field undeniably boosted Walker\'s stats. The man was great everywhere he played though.\n\nWill he make it?\n\nThis is Walker\'s seventh year on the Hall of Fame ballot and he topped out at 22.9 percent of the vote back in 2009. According to Ryan Thibodaux\'s tracker, Walker has 

=== OFFSET 600000
' flexibility to employees and saves seating space for the employer, amongst many other benefits Working from Home entails. However, many employees often get caught up with the comfort a WFH option provides and resultantly deliver poor productivity. If you too are availing the Work from Home option or work from a Home-Office then here are 6 proven ways to optimize your productivity:1. Designate Time SlotsRemember Work from Home doesn’t shorten your work or work hours. Designate time slot(s) in the morning, afternoon or evening and stick to them if you really want to be productive. Likewise, schedule breaks in between your work hours to unwind.2. Assign a Tidy CornerAssign a tidy corner for working every day. Invest in an ergonomic chair and table to work for long hours in the right posture. Sitting on a sofa or bed all day will harm your back besides making you slow. Also, ensure this corner is not the place you’ll have breakfast, lunch or dinner. You need some change even if it means changing places within your boundary wall.3. CommunicateKeep communicating with your team members over the phone or messenger or emails to keep yourself engaged and focused at work. Similarly, limit personal chats or talks during your designated work time.4. Dress UpTake a bath and dress up in the morning to get into the professional mindset. Working in your PJs may appear cool but it hampers your

=== OFFSET 800000
" true\n        };\n        client.Send(&quot;MyEmailAddress@gmail.com&quot;, &quot;some.email@some.com&quot;, &quot;test&quot;, &quot;testbody&quot;); \n    }\n</code></pre>\n<p>Any ideas?</p>\n<p><strong>UPDATE</strong></p>\n<p>More details.</p>\n<p>Maybe I should say what other attempts I made that gave me the same error:\n(Note when i didn't specify a port it tryed port 25)</p>\n<pre><code>    public static void Attempt2()\n    {\n        var fromAddress = new MailAddress(&quot;MyEmailAddy@gmail.com&quot;, &quot;From Name&quot;);\n        var toAddress = new MailAddress(&quot;MyEmailAddy@dfdf.com&quot;, &quot;To Name&quot;);\n        const string fromPassword = &quot;pass&quot;;\n        const string subject = &quot;Subject&quot;;\n        const string body = &quot;Body&quot;;\n        var smtp = new SmtpClient\n        {\n            Host = &quot;smtp.gmail.com&quot;,\n            Port = 587,\n            EnableSsl = true,\n            DeliveryMethod = SmtpDeliveryMethod.Network,\n            UseDefaultCredentials = false,\n            Credentials = new NetworkCredential(fromAddress.Address, fromPassword)\n        };\n        using (var message = new MailMessage(fromAddress, toAddress)\n        {\n            Subject = subject,\n            Body = body\n        }\n            ) { smtp.Send(message); }\n    }\n\n\n    public static void Attempt3()\n    {\n        MailMessag

=== OFFSET 950000
' to the debug perspective, I can only see the class instance (on which the function was called) and the two parameters. I cannot expand the class instance to see the values of its member variables. There is a space to the left of the instance so I would assume that there should be an arrow there so one is able to expand it in a similar way as in the Outline.</p>\n\n<p>Is there a configuration or something I must enable for this? Or have I misunderstood the variables window?</p>\n\n<p>If it matters, this is Eclipse 3.2.2 in Ubuntu Linux.</p>\n\n<hr>\n\n<p><strong>[Update]</strong>\nI downloaded a new release from <a href="http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/galileo/R/eclipse-java-galileo-linux-gtk.tar.gz" rel="nofollow noreferrer">http://www.eclipse.org/downloads/download.php?file=/technology/epp/downloads/release/galileo/R/eclipse-java-galileo-linux-gtk.tar.gz</a></p>\n\n<p>"About Eclipse" now reports "Build id: 20090619-0625".</p>\n\n<p>Testcase:</p>\n\n<pre><code>class Foo {\n    private int bar;\n\n    Foo() {\n        bar = 1;\n    }\n\n    public void set(int newbar) {\n        assert (newbar&lt;0);\n        bar = newbar;\n    }\n}\n\nclass Test {\n\n     public static void main (String[] args){\n         Foo f = new Foo();\n         f.set(5);\n     }\n}\n</code></pre>\n\n<p>Obviously, this code produces the assertion excep
[metadata] {"noOutputExpected": false}

docs 182016 total_chars 770537151 est_tokens 192634287
len pct [  160   408  1050  2246  4500 13064 34874]
html-ish 62 wikitext-artifact 0
n samp 24
---ID 2607 'PdfSR.com is a participant in the Amazon Services LLC Associates Program, an affiliate advertising program designed to provide a means for sites to earn advertising fees by advertising and linking to Amazon.com.\nHow to Read Faces : A System of Practical and Scientific Physiognomy - 2 Vols. ; A Manua'
---ID 5755 'I’ll admit, I am biased because I absolutely love chocolates from Bernard Callebaut. If you’re a Calgary resident, you know all about this chocolaterie, along with all of the recent press and drama surrounding the man who founded the company. Well now the company is in the hands of new owners, and I'
---ID 20161 'Our living room has lovely bones, elements that have outlasted many owners throughout its 100-year history. However, five of the best features of the room are also what make it so hard to decorate! The glass paned doors leading in from the foyer; the strangely pretty faux fireplace; the painted radi'
---ID 48749 '<|endoftext|>One of the mysteries of the English language finally explained.\n1A stickleback (now rare).\n2A member of the family Stephanoberycidae of small deep-sea fishes of tropical and subtropical waters, which bear spines ventrally and dorsally in front of the caudal fin and typically have toothe'
---ID 53401 "<|endoftext|>I want to scan port 445 on my Windows server 2003, but my scanner only has one type ipv6 address which is global not link-local. When I do this, I find that I can't find port 445 open. But I use the command\nnetstat -an to assure the port 445 is listening. Finally I find this confusing p"
---ID 58057 '<|endoftext|>Milan were beaten 4-0 and went out 4-2 on aggregate.\nâTonight Barcelona were extraordinary, especially in the first 30 minutes. Despite all that pressure, they had few real chances.\nâIn the first half we had a lot of problems emerging from our half, as they were pressing us and forcing '

[stdout]
docs 182016 total_chars 770537151 est_tokens 192634287
len pct [  160   408  1050  2246  4500 13064 34874]
html-ish 62 wikitext-artifact 0
n samp 24
---ID 2607 'PdfSR.com is a participant in the Amazon Services LLC Associates Program, an affiliate advertising program designed to provide a means for sites to earn advertising fees by advertising and linking to Amazon.com.\nHow to Read Faces : A System of Practical and Scientific Physiognomy - 2 Vols. ; A Manua'
---ID 5755 'I’ll admit, I am biased because I absolutely love chocolates from Bernard Callebaut. If you’re a Calgary resident, you know all about this chocolaterie, along with all of the recent press and drama surrounding the man who founded the company. Well now the company is in the hands of new owners, and I'
---ID 20161 'Our living room has lovely bones, elements that have outlasted many owners throughout its 100-year history. However, five of the best features of the room are also what make it so hard to decorate! The glass paned doors leading in from the foyer; the strangely pretty faux fireplace; the painted radi'
---ID 48749 '<|endoftext|>One of the mysteries of the English language finally explained.\n1A stickleback (now rare).\n2A member of the family Stephanoberycidae of small deep-sea fishes of tropical and subtropical waters, which bear spines ventrally and dorsally in front of the caudal fin and typically have toothe'
---ID 53401 "<|endoftext|>I want to scan port 445 on my Windows server 2003, but my scanner only has one type ipv6 address which is global not link-local. When I do this, I find that I can't find port 445 open. But I use the command\nnetstat -an to assure the port 445 is listening. Finally I find this confusing p"
---ID 58057 '<|endoftext|>Milan were beaten 4-0 and went out 4-2 on aggregate.\nâTonight Barcelona were extraordinary, especially in the first 30 minutes. Despite all that pressure, they had few real chances.\nâIn the first half we had a lot of problems emerging from our half, as they were pressing us and forcing '
[metadata] {"noOutputExpected": false}

16
               total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
eos-prefixed approx 37566 of 182016

[stdout]
16
               total        used        free      shared  buff/cache   available
Mem:            1024           0        1023           0           0        1023
eos-prefixed approx 37566 of 182016
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/submission/curate.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/submission/curate.py", "content": "#!/usr/bin/env python3\n\"\"\"\nCurate /workspace/data/pool.jsonl -> /workspace/submission/selection.json\n\nSTATED CRITERION\n----------------\nThe disclosed evaluation target is a *balanced four-register* high-quality English\nmixture: encyclopedic (Wikipedia), general high-quality web prose, news, and\ntechnical Q&A.  Decoding the provided dev target (data/multi_dev.npy) confirms it\nis four equal contiguous quarters, one per register.\n\nWe therefore select documents by *proximity to the target distribution, register by\nregister*, rather than by a generic \"quality\" heuristic:\n\n  1. Decode the dev target and split it into its four register blocks; split each\n     block on <|endoftext|> to recover real target documents (the positives).\n  2. Represent every document (pool and target) as L2-normalised hashed\n     unigram+bigram term frequencies (2^18 buckets).\n  3. Train a single 5-way softmax classifier on GPU: {register 0..3} vs {pool}.\n     Its per-register probability p_r(doc) is a calibrated \"how much does this pool\n     document look like target register r\" score.\n  4. Apply cheap corpus-hygiene filters (length, alphabetic fraction, line/ngram\n     repetition, boilerplate) and near-duplicate removal (MinHash on word 5-grams).\n  5. Fill the 12M-token budget with an *equal share per register* (25% each),\n     taking each register's highest-scoring surviving documents, and emit the ids\n     round-robin across registers so that any prefix of the list is also balanced.\n\nRationale: perplexity on a mixture is dominated by whichever register the model\nfits worst, so budget is allocated evenly instead of letting one easy-to-match\nregister crowd the others out.\n\"\"\"\nimport argparse, hashlib, json, os, re, sys, time\nimport numpy as np\n\nPOOL = \"/workspace/data/pool.jsonl\"\nDEV = \"/workspace/data/multi_dev.npy\"\nOUT = \"/workspace/submission/selection.json\"\nCACHE = \"/tmp/curate_cache\"\nBUDGET = 12_000_000\nN_REG = 4\nHASH_DIM = 1 << 18\nMAX_CHARS = 20_000          # chars of each doc used for featurisation\nTOK_RE = re.compile(r\"[a-z0-9']+|[^\\sa-z0-9']\")\n\nos.makedirs(CACHE, exist_ok=True)\ndef log(*a): print(f\"[{time.strftime('%H:%M:%S')}]\", *a, flush=True)\n\n\n# ---------------------------------------------------------------- featurisation\ndef hash_feats(text):\n    \"\"\"Hashed unigram+bigram counts -> (indices, counts) for one document.\"\"\"\n    toks = TOK_RE.findall(text.lower()[:MAX_CHARS])\n    if not toks:\n        return np.zeros(0, np.int32), np.zeros(0, np.float32)\n    h = np.fromiter(\n        (hash(t) for t in toks), dtype=np.int64, count=len(toks)\n    ) & 0x7FFFFFFF\n    uni = h % HASH_DIM\n    if len(toks) > 1:\n        bi = ((h[:-1] * 1000003) ^ h[1:]) & 0x7FFFFFFF % HASH_DIM\n        bi = bi % HASH_DIM\n        idx = np.concatenate([uni, bi])\n    else:\n        idx = uni\n    u, c = np.unique(idx, return_counts=True)\n    return u.astype(np.int32), c.astype(np.float32)\n\n\ndef featurise(texts, nproc=16):\n    \"\"\"-> CSR (indptr, indices, data) with L2-normalised sqrt-tf rows.\"\"\"\n    from multiprocessing import Pool as MPool\n    with MPool(nproc) as p:\n        res = p.map(_feat_one, texts, chunksize=256)\n    indptr = np.zeros(len(res) + 1, np.int64)\n    indptr[1:] = np.cumsum([len(r[0]) for r in res])\n    idx = np.concatenate([r[0] for r in res]) if res else np.zeros(0, np.int32)\n    dat = np.concatenate([r[1] for r in res]) if res else np.zeros(0, np.float32)\n    return indptr, idx, dat\n\n\ndef _feat_one(text):\n    i, c = hash_feats(text)\n    c = np.sqrt(c)                                  # sublinear tf\n    n = np.linalg.norm(c)\n    if n > 0:\n        c /= n\n    return i, c.astype(np.float32)\n\n\n# --------------------------------------------------------------------- hygiene\nBOILER = re.compile(\n    r\"(all rights reserved|privacy policy|terms of (use|service)|cookies?\"\n    r\"|click here|sign up|subscribe|add to cart|©|copyright \\d{4})\", re.I)\n\ndef hygiene(text):\n    \"\"\"Return (ok, n_chars). Cheap corpus-quality gate.\"\"\"\n    t = text\n    n = len(t)\n    if n < 600 or n > 400_000:\n        return False, n\n    head = t[:40_000]\n    alpha = sum(c.isalpha() or c.isspace() for c in head)\n    if alpha / len(head) < 0.80:                    # tables / markup / gibberish\n        return False, n\n    letters = sum(c.isalpha() for c in head)\n    if letters / len(head) < 0.60:\n        return False, n\n    nonascii = sum(ord(c) > 127 for c in head)\n    if nonascii / len(head) > 0.06:                 # mojibake / non-English\n        return False, n\n    words = head.split()\n    if len(words) < 80:\n        return False, n\n    meanw = sum(len(w) for w in words) / len(words)\n    if not (3.0 <= meanw <= 9.0):\n        return False, n\n    # duplicate-line ratio\n    lines = [l.strip() for l in head.split(\"\\n\") if len(l.strip()) > 15]\n    if lines and len(set(lines)) / len(lines) < 0.65:\n        return False, n\n    # repeated 10-gram ratio (spam / SEO churn)\n    if len(words) >= 40:\n        w = words[:5000]\n        grams = set()\n        tot = len(w) - 9\n        dup = 0\n        for i in range(tot):\n            g = \" \".join(w[i:i + 10])\n            if g in grams:\n                dup += 1\n            else:\n                grams.add(g)\n        if dup / max(1, tot) > 0.20:\n            return False, n\n    # boilerplate density per 1k chars\n    if len(BOILER.findall(head)) / (len(head) / 1000.0) > 1.2:\n        return False, n\n    # sentence punctuation present (prose, not a keyword dump)\n    if head.count(\".\") + head.count(\"?\") + head.count(\"!\") < len(head) / 800.0:\n        return False, n\n    return True, n\n\n\ndef _hyg_one(t):\n    return hygiene(t)\n\n\n# -------------------------------------------------------------------- minhash\ndef minhash_sig(text, k=64, seed=17):\n    w = TOK_RE.findall(text.lower()[:MAX_CHARS])\n    if len(w) < 10:\n        return np.full(k, np.iinfo(np.uint64).max, np.uint64)\n    grams = {\" \".join(w[i:i + 5]) for i in range(0, max(1, len(w) - 4))}\n    h = np.array(\n        [int.from_bytes(hashlib.blake2b(g.encode(), digest_size=8).digest(), \"little\")\n         for g in grams], dtype=np.uint64)\n    rng = np.random.default_rng(seed)\n    a = rng.integers(1, 1 << 61, size=k, dtype=np.uint64)\n    b = rng.integers(0, 1 << 61, size=k, dtype=np.uint64)\n    M = (a[:, None] * h[None, :] + b[:, None])\n    return M.min(axis=1)\n\n\ndef _mh_one(t):\n    return minhash_sig(t)\n\n\n# ----------------------------------------------------------------------- main\ndef main():\n    ap = argparse.ArgumentParser()\n    ap.add_argument(\"--out\", default=OUT)\n    ap.add_argument(\"--mode\", default=\"balanced\",\n                    choices=[\"balanced\", \"pooled\"])\n    ap.add_argument(\"--overfill\", type=float, default=2.2,\n                    help=\"emit this many x budget worth of tokens\")\n    ap.add_argument(\"--temp\", type=float, default=0.0,\n                    help=\"0 = strict top-k per register\")\n    a = ap.parse_args()\n\n    # ---- load pool\n    log(\"loading pool\")\n    ids, texts = [], []\n    for line in open(POOL):\n        r = json.loads(line)\n        ids.append(r[\"id\"]); texts.append(r[\"text\"])\n    ids = np.array(ids, np.int64)\n    N = len(ids)\n    log(f\"{N} pool docs\")\n\n    # ---- target positives, per register\n    from transformers import AutoTokenizer\n    tk = AutoTokenizer.from_pretrained(\"gpt2\")\n    dev = np.load(DEV)\n    q = len(dev) // N_REG\n    pos_texts, pos_lab = [], []\n    for r in range(N_REG):\n        blk = tk.decode(dev[r * q:(r + 1) * q].tolist())\n        docs = [d.strip() for d in blk.split(\"<|endoftext|>\")]\n        docs = [d for d in docs if len(d) > 500]\n        pos_texts += docs\n        pos_lab += [r] * len(docs)\n        log(f\"register {r}: {len(docs)} target docs\")\n    pos_lab = np.array(pos_lab, np.int64)\n\n    # ---- features\n    cf = f\"{CACHE}/feat.npz\"\n    if os.path.exists(cf):\n        z = np.load(cf)\n        pI, pX, pD = z[\"pI\"], z[\"pX\"], z[\"pD\"]\n        tI, tX, tD = z[\"tI\"], z[\"tX\"], z[\"tD\"]\n        log(\"features from cache\")\n    else:\n        log(\"featurising target\")\n        tI, tX, tD = featurise(pos_texts)\n        log(\"featurising pool\")\n        pI, pX, pD = featurise(texts)\n        np.savez(cf, pI=pI, pX=pX, pD=pD, tI=tI, tX=tX, tD=tD)\n        log(\"features done\")\n\n    # ---- hygiene + length\n    hf = f\"{CACHE}/hyg.npz\"\n    if os.path.exists(hf):\n        z = np.load(hf); ok, nch = z[\"ok\"], z[\"nch\"]\n    else:\n        log(\"hygiene pass\")\n        from multiprocessing import Pool as MPool\n        with MPool(16) as p:\n            res = p.map(_hyg_one, texts, chunksize=256)\n        ok = np.array([r[0] for r in res], bool)\n        nch = np.array([r[1] for r in res], np.int64)\n        np.savez(hf, ok=ok, nch=nch)\n    log(f\"hygiene keeps {ok.sum()} / {N}\")\n\n    # ---- train 5-way classifier (4 registers + pool-negative)\n    import torch\n    dv = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n    def to_sparse(I, X, D, nrow):\n        return torch.sparse_csr_tensor(\n            torch.from_numpy(I), torch.from_numpy(X.astype(np.int64)),\n            torch.from_numpy(D), size=(nrow, HASH_DIM), device=dv)\n\n    Tp = to_sparse(tI, tX, tD, len(pos_texts))\n    Pp = to_sparse(pI, pX, pD, N)\n\n    # negatives: random pool sample (pool is the background distribution)\n    rng = np.random.default_rng(0)\n    neg_idx = rng.choice(N, size=min(40000, N), replace=False)\n    neg_idx_t = torch.from_numpy(neg_idx).to(dv)\n\n    W = torch.zeros(HASH_DIM, N_REG + 1, device=dv, requires_grad=True)\n    b = torch.zeros(N_REG + 1, device=dv, requires_grad=True)\n    optim = torch.optim.Adam([W, b], lr=0.05)\n    yT = torch.from_numpy(pos_lab).to(dv)\n    yN = torch.full((len(neg_idx),), N_REG, device=dv, dtype=torch.long)\n\n    log(\"training classifier\")\n    # pre-densify the (small) training rows once for speed\n    Td = (Tp @ torch.eye(HASH_DIM, device=dv, dtype=torch.float32)) \\\n        if False else None  # avoid huge dense; use sparse matmul each step\n    for step in range(400):\n        lo_t = torch.sparse.mm(Tp, W) + b\n        # negative minibatch through sparse mm on the full pool then index\n        lo_p = (torch.sparse.mm(Pp, W) + b)[neg_idx_t]\n        loss = (torch.nn.functional.cross_entropy(lo_t, yT) * 1.0\n                + torch.nn.functional.cross_entropy(lo_p, yN) * 1.0\n                + 2e-4 * (W * W).sum())\n        optim.zero_grad(); loss.backward(); optim.step()\n        if step % 100 == 0:\n            with torch.no_grad():\n                acc = (lo_t.argmax(1) == yT).float().mean().item()\n            log(f\"  step {step} loss {loss.item():.4f} target-acc {acc:.3f}\")\n\n    with torch.no_grad():\n        logits = torch.sparse.mm(Pp, W) + b\n        prob = torch.softmax(logits, 1).cpu().numpy()\n    np.save(f\"{CACHE}/prob.npy\", prob)\n    log(\"scored pool\")\n\n    # ---- estimate tokens per doc (bytes/token calibrated on a sample)\n    est_tok = np.maximum(1, (nch / 4.05)).astype(np.int64)\n\n    # ---- near-dup removal restricted to plausible candidates (top of each reg)\n    cand = np.zeros(N, bool)\n    reg_score = prob[:, :N_REG]\n    order_per_reg = []\n    for r in range(N_REG):\n        s = reg_score[:, r].copy()\n        s[~ok] = -1.0\n        o = np.argsort(-s)\n        # generous candidate pool per register: 4x what the budget needs\n        take = o[:60000]\n        cand[take] = True\n        order_per_reg.append(o)\n    cidx = np.flatnonzero(cand)\n    log(f\"minhash on {len(cidx)} candidates\")\n    mf = f\"{CACHE}/mh.npy\"\n    if os.path.exists(mf):\n        sigs = np.load(mf)\n        if sigs.shape[0] != len(cidx):\n            sigs = None\n    else:\n        sigs = None\n    if sigs is None:\n        from multiprocessing import Pool as MPool\n        with MPool(16) as p:\n            sl = p.map(_mh_one, [texts[i] for i in cidx], chunksize=64)\n        sigs = np.stack(sl)\n        np.save(mf, sigs)\n    # banded LSH: 16 bands of 4\n    dup = np.zeros(N, bool)\n    pos_of = {int(v): k for k, v in enumerate(cidx)}\n    # priority for keeping: best register score\n    best = reg_score.max(1)\n    keep_rank = np.argsort(-best[cidx])           # process best-first\n    seen_bands = [dict() for _ in range(16)]\n    for j in keep_rank:\n        gid = cidx[j]\n        s = sigs[j]\n        keys = [hashlib.blake2b(s[bi * 4:(bi + 1) * 4].tobytes(),\n                                digest_size=8).digest() for bi in range(16)]\n        hit = sum(1 for bi, k in enumerate(keys) if k in seen_bands[bi])\n        if hit >= 2:\n            dup[gid] = True\n            continue\n        for bi, k in enumerate(keys):\n            seen_bands[bi][k] = gid\n    log(f\"near-dups removed: {dup.sum()}\")\n\n    # ---- balanced fill\n    valid = ok & ~dup\n    per_reg_budget = BUDGET * a.overfill / N_REG\n    chosen = [[] for _ in range(N_REG)]\n    taken = np.zeros(N, bool)\n    for r in range(N_REG):\n        s = reg_score[:, r].copy()\n        s[~valid] = -1.0\n        o = np.argsort(-s)\n        tot = 0\n        for i in o:\n            if s[i] <= -1.0 or taken[i]:\n                continue\n            taken[i] = True\n            chosen[r].append(int(ids[i]))\n            tot += int(est_tok[i])\n            if tot >= per_reg_budget:\n                break\n        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.1f}M est tokens\")\n\n    if a.mode == \"pooled\":\n        s = reg_score.sum(1).copy()\n        s[~valid] = -1.0\n        o = [i for i in np.argsort(-s) if s[i] > -1.0]\n        sel, tot = [], 0\n        for i in o:\n            sel.append(int(ids[i])); tot += int(est_tok[i])\n            if tot >= BUDGET * a.overfill:\n                break\n    else:\n        # round-robin interleave so every prefix is register-balanced\n        sel = []\n        ptr = [0] * N_REG\n        while any(ptr[r] < len(chosen[r]) for r in range(N_REG)):\n            for r in range(N_REG):\n                if ptr[r] < len(chosen[r]):\n                    sel.append(chosen[r][ptr[r]]); ptr[r] += 1\n\n    assert len(sel) == len(set(sel))\n    json.dump(sel, open(a.out, \"w\"))\n    log(f\"wrote {len(sel)} ids -> {a.out}\")\n\n\nif __name__ == \"__main__\":\n    main()\n", "structuredPatch": [], "originalFile": null, "userModified": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "import argparse, hashlib, json, os, re, sys, time\nimport numpy as np", "newString": "import argparse, hashlib, json, os, re, sys, time\n\n# str.__hash__ is salted per process; pin the salt so featurisation is\n# bit-reproducible across runs (children inherit it through fork).\nif os.environ.get(\"PYTHONHASHSEED\") != \"0\":\n    os.environ[\"PYTHONHASHSEED\"] = \"0\"\n    os.execv(sys.executable, [sys.executable] + sys.argv)\n\nimport numpy as np", "originalFile": null, "structuredPatch": [{"oldStart": 30, "oldLines": 6, "newStart": 30, "newLines": 13, "lines": [" register crowd the others out.", " \"\"\"", " import argparse, hashlib, json, os, re, sys, time", "+", "+# str.__hash__ is salted per process; pin the salt so featurisation is", "+# bit-reproducible across runs (children inherit it through fork).", "+if os.environ.get(\"PYTHONHASHSEED\") != \"0\":", "+    os.environ[\"PYTHONHASHSEED\"] = \"0\"", "+    os.execv(sys.executable, [sys.executable] + sys.argv)", "+", " import numpy as np", " ", " POOL = \"/workspace/data/pool.jsonl\""]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    uni = h % HASH_DIM\n    if len(toks) > 1:\n        bi = ((h[:-1] * 1000003) ^ h[1:]) & 0x7FFFFFFF % HASH_DIM\n        bi = bi % HASH_DIM\n        idx = np.concatenate([uni, bi])\n    else:\n        idx = uni", "newString": "    uni = h % HASH_DIM\n    if len(toks) > 1:\n        bi = (((h[:-1] * np.int64(1000003)) ^ h[1:]) & 0x7FFFFFFF) % HASH_DIM\n        idx = np.concatenate([uni, bi])\n    else:\n        idx = uni", "originalFile": null, "structuredPatch": [{"oldStart": 64, "oldLines": 8, "newStart": 64, "newLines": 7, "lines": ["     ) & 0x7FFFFFFF", "     uni = h % HASH_DIM", "     if len(toks) > 1:", "-        bi = ((h[:-1] * 1000003) ^ h[1:]) & 0x7FFFFFFF % HASH_DIM", "-        bi = bi % HASH_DIM", "+        bi = (((h[:-1] * np.int64(1000003)) ^ h[1:]) & 0x7FFFFFFF) % HASH_DIM", "         idx = np.concatenate([uni, bi])", "     else:", "         idx = uni"]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    Tp = to_sparse(tI, tX, tD, len(pos_texts))\n    Pp = to_sparse(pI, pX, pD, N)\n\n    # negatives: random pool sample (pool is the background distribution)\n    rng = np.random.default_rng(0)\n    neg_idx = rng.choice(N, size=min(40000, N), replace=False)\n    neg_idx_t = torch.from_numpy(neg_idx).to(dv)\n\n    W = torch.zeros(HASH_DIM, N_REG + 1, device=dv, requires_grad=True)\n    b = torch.zeros(N_REG + 1, device=dv, requires_grad=True)\n    optim = torch.optim.Adam([W, b], lr=0.05)\n    yT = torch.from_numpy(pos_lab).to(dv)\n    yN = torch.full((len(neg_idx),), N_REG, device=dv, dtype=torch.long)\n\n    log(\"training classifier\")\n    # pre-densify the (small) training rows once for speed\n    Td = (Tp @ torch.eye(HASH_DIM, device=dv, dtype=torch.float32)) \\\n        if False else None  # avoid huge dense; use sparse matmul each step\n    for step in range(400):\n        lo_t = torch.sparse.mm(Tp, W) + b\n        # negative minibatch through sparse mm on the full pool then index\n        lo_p = (torch.sparse.mm(Pp, W) + b)[neg_idx_t]\n        loss = (torch.nn.functional.cross_entropy(lo_t, yT) * 1.0\n                + torch.nn.functional.cross_entropy(lo_p, yN) * 1.0\n                + 2e-4 * (W * W).sum())\n        optim.zero_grad(); loss.backward(); optim.step()\n        if step % 100 == 0:\n            with torch.no_grad():\n                acc = (lo_t.argmax(1) == yT).float().mean().item()\n            log(f\"  step {step} loss {loss.item():.4f} target-acc {acc:.3f}\")\n\n    with torch.no_grad():\n        logits = torch.sparse.mm(Pp, W) + b\n        prob = torch.softmax(logits, 1).cpu().numpy()", "newString": "    def sub_csr(I, X, D, rows):\n        \"\"\"Extract the given rows of a CSR triple into a new CSR triple.\"\"\"\n        lens = (I[rows + 1] - I[rows]).astype(np.int64)\n        nI = np.zeros(len(rows) + 1, np.int64); nI[1:] = np.cumsum(lens)\n        take = np.concatenate([np.arange(I[r], I[r + 1]) for r in rows]) \\\n            if len(rows) else np.zeros(0, np.int64)\n        return nI, X[take], D[take]\n\n    # negatives: random pool sample = the background (do-nothing) distribution\n    rng = np.random.default_rng(0)\n    neg_idx = rng.choice(N, size=min(40000, N), replace=False)\n    nI, nX, nD = sub_csr(pI, pX, pD, neg_idx)\n\n    # positive train/val split, so reported accuracy is honest\n    pperm = rng.permutation(len(pos_texts))\n    nval = max(40, len(pos_texts) // 5)\n    vi, ti_ = pperm[:nval], pperm[nval:]\n    trI, trX, trD = sub_csr(tI, tX, tD, ti_)\n    vaI, vaX, vaD = sub_csr(tI, tX, tD, vi)\n\n    Ttr = to_sparse(trI, trX, trD, len(ti_))\n    Tva = to_sparse(vaI, vaX, vaD, len(vi))\n    Neg = to_sparse(nI, nX, nD, len(neg_idx))\n\n    yT = torch.from_numpy(pos_lab[ti_]).to(dv)\n    yV = torch.from_numpy(pos_lab[vi]).to(dv)\n    yN = torch.full((len(neg_idx),), N_REG, device=dv, dtype=torch.long)\n    # equalise register mass so no register dominates the decision boundary\n    cw = torch.ones(N_REG + 1, device=dv)\n    cnt = np.bincount(pos_lab[ti_], minlength=N_REG).astype(np.float32)\n    cw[:N_REG] = torch.from_numpy(cnt.mean() / np.maximum(cnt, 1)).to(dv)\n\n    W = torch.zeros(HASH_DIM, N_REG + 1, device=dv, requires_grad=True)\n    b = torch.zeros(N_REG + 1, device=dv, requires_grad=True)\n    optim = torch.optim.Adam([W, b], lr=0.05)\n    L2 = 3e-4\n\n    log(\"training classifier\")\n    best_va, best_state = -1.0, None\n    for step in range(600):\n        lo_t = torch.sparse.mm(Ttr, W) + b\n        lo_n = torch.sparse.mm(Neg, W) + b\n        loss = (torch.nn.functional.cross_entropy(lo_t, yT, weight=cw)\n                + torch.nn.functional.cross_entropy(lo_n, yN)\n                + L2 * (W * W).sum())\n        optim.zero_grad(); loss.backward(); optim.step()\n        if step % 50 == 49:\n            with torch.no_grad():\n                va = ((torch.sparse.mm(Tva, W) + b).argmax(1) == yV\n                      ).float().mean().item()\n                tra = (lo_t.argmax(1) == yT).float().mean().item()\n            if va > best_va:\n                best_va = va\n                best_state = (W.detach().clone(), b.detach().clone())\n            log(f\"  step {step+1} loss {loss.item():.4f} \"\n                f\"train-acc {tra:.3f} val-acc {va:.3f}\")\n    W, b = best_state\n    log(f\"best held-out register accuracy {best_va:.3f} (chance = {1/(N_REG+1):.2f})\")\n\n    with torch.no_grad():\n        Pp = to_sparse(pI, pX, pD, N)\n        logits = torch.sparse.mm(Pp, W) + b\n        prob = torch.softmax(logits, 1).cpu().numpy()", "originalFile": null, "structuredPatch": [{"oldStart": 246, "oldLines": 38, "newStart": 246, "newLines": 67, "lines": ["             torch.from_numpy(I), torch.from_numpy(X.astype(np.int64)),", "             torch.from_numpy(D), size=(nrow, HASH_DIM), device=dv)", " ", "-    Tp = to_sparse(tI, tX, tD, len(pos_texts))", "-    Pp = to_sparse(pI, pX, pD, N)", "+    def sub_csr(I, X, D, rows):", "+        \"\"\"Extract the given rows of a CSR triple into a new CSR triple.\"\"\"", "+        lens = (I[rows + 1] - I[rows]).astype(np.int64)", "+        nI = np.zeros(len(rows) + 1, np.int64); nI[1:] = np.cumsum(lens)", "+        take = np.concatenate([np.arange(I[r], I[r + 1]) for r in rows]) \\", "+            if len(rows) else np.zeros(0, np.int64)", "+        return nI, X[take], D[take]", " ", "-    # negatives: random pool sample (pool is the background distribution)", "+    # negatives: random pool sample = the background (do-nothing) distribution", "     rng = np.random.default_rng(0)", "     neg_idx = rng.choice(N, size=min(40000, N), replace=False)", "-    neg_idx_t = torch.from_numpy(neg_idx).to(dv)", "+    nI, nX, nD = sub_csr(pI, pX, pD, neg_idx)", " ", "+    # positive train/val split, so reported accuracy is honest", "+    pperm = rng.permutation(len(pos_texts))", "+    nval = max(40, len(pos_texts) // 5)", "+    vi, ti_ = pperm[:nval], pperm[nval:]", "+    trI, trX, trD = sub_csr(tI, tX, tD, ti_)", "+    vaI, vaX, vaD = sub_csr(tI, tX, tD, vi)", "+", "+    Ttr = to_sparse(trI, trX, trD, len(ti_))", "+    Tva = to_sparse(vaI, vaX, vaD, len(vi))", "+    Neg = to_sparse(nI, nX, nD, len(neg_idx))", "+", "+    yT = torch.from_numpy(pos_lab[ti_]).to(dv)", "+    yV = torch.from_numpy(pos_lab[vi]).to(dv)", "+    yN = torch.full((len(neg_idx),), N_REG, device=dv, dtype=torch.long)", "+    # equalise register mass so no register dominates the decision boundary", "+    cw = torch.ones(N_REG + 1, device=dv)", "+    cnt = np.bincount(pos_lab[ti_], minlength=N_REG).astype(np.float32)", "+    cw[:N_REG] = torch.from_numpy(cnt.mean() / np.maximum(cnt, 1)).to(dv)", "+", "     W = torch.zeros(HASH_DIM, N_REG + 1, device=dv, requires_grad=True)", "     b = torch.zeros(N_REG + 1, device=dv, requires_grad=True)", "     optim = torch.optim.Adam([W, b], lr=0.05)", "-    yT = torch.from_numpy(pos_lab).to(dv)", "-    yN = torch.full((len(neg_idx),), N_REG, device=dv, dtype=torch.long)", "+    L2 = 3e-4", " ", "     log(\"training classifier\")", "-    # pre-densify the (small) training rows once for speed", "-    Td = (Tp @ torch.eye(HASH_DIM, device=dv, dtype=torch.float32)) \\", "-        if False else None  # avoid huge dense; use sparse matmul each step", "-    for step in range(400):", "-        lo_t = torch.sparse.mm(Tp, W) + b", "-        # negative minibatch through sparse mm on the full pool then index", "-        lo_p = (torch.sparse.mm(Pp, W) + b)[neg_idx_t]", "-        loss = (torch.nn.functional.cross_entropy(lo_t, yT) * 1.0", "-                + torch.nn.functional.cross_entropy(lo_p, yN) * 1.0", "-                + 2e-4 * (W * W).sum())", "+    best_va, best_state = -1.0, None", "+    for step in range(600):", "+        lo_t = torch.sparse.mm(Ttr, W) + b", "+        lo_n = torch.sparse.mm(Neg, W) + b", "+        loss = (torch.nn.functional.cross_entropy(lo_t, yT, weight=cw)", "+                + torch.nn.functional.cross_entropy(lo_n, yN)", "+                + L2 * (W * W).sum())", "         optim.zero_grad(); loss.backward(); optim.step()", "-        if step % 100 == 0:", "+        if step % 50 == 49:", "             with torch.no_grad():", "-                acc = (lo_t.argmax(1) == yT).float().mean().item()", "-            log(f\"  step {step} loss {loss.item():.4f} target-acc {acc:.3f}\")", "+                va = ((torch.sparse.mm(Tva, W) + b).argmax(1) == yV", "+                      ).float().mean().item()", "+                tra = (lo_t.argmax(1) == yT).float().mean().item()", "+            if va > best_va:", "+                best_va = va", "+                best_state = (W.detach().clone(), b.detach().clone())", "+            log(f\"  step {step+1} loss {loss.item():.4f} \"", "+                f\"train-acc {tra:.3f} val-acc {va:.3f}\")", "+    W, b = best_state", "+    log(f\"best held-out register accuracy {best_va:.3f} (chance = {1/(N_REG+1):.2f})\")", " ", "     with torch.no_grad():", "+        Pp = to_sparse(pI, pX, pD, N)", "         logits = torch.sparse.mm(Pp, W) + b", "         prob = torch.softmax(logits, 1).cpu().numpy()", "     np.save(f\"{CACHE}/prob.npy\", prob)"]}], "userModified": false, "replaceAll": false}

48:HASH_DIM = 1 << 17
5180

[stdout]
48:HASH_DIM = 1 << 17
5180
[metadata] {"noOutputExpected": false}

EXIT 0
[00:59:59] loading pool
[01:00:07] 182016 pool docs
[01:00:11] register 0: 1077 target docs
[01:00:11] register 1: 203 target docs
[01:00:12] register 2: 405 target docs
[01:00:12] register 3: 350 target docs
[01:00:12] featurising target
[01:00:13] featurising pool
[01:00:23] features done
[01:00:23] hygiene pass
[01:00:38] hygiene keeps 135094 / 182016
/workspace/submission/curate.py:245: UserWarning: Sparse CSR tensor support is in beta state. If you miss a functionality in the sparse tensor support, please submit a feature request to https://github.com/pytorch/pytorch/issues. (Triggered internally at /pytorch/aten/src/ATen/SparseCsrTensorImpl.cpp:53.)
  return torch.sparse_csr_tensor(
[01:00:39] training classifier
[01:00:41]   step 50 loss 1.0252 train-acc 0.971 val-acc 0.853
[01:00:41]   step 100 loss 0.9679 train-acc 0.972 val-acc 0.855
[01:00:42]   step 150 loss 0.9538 train-acc 0.974 val-acc 0.862
[01:00:43]   step 200 loss 0.9488 train-acc 0.975 val-acc 0.865
[01:00:43]   step 250 loss 0.9469 train-acc 0.976 val-acc 0.865
[01:00:44]   step 300 loss 0.9460 train-acc 0.976 val-acc 0.867
[01:00:45]   step 350 loss 0.9456 train-acc 0.975 val-acc 0.867
[01:00:46]   step 400 loss 0.9454 train-acc 0.975 val-acc 0.867
[01:00:46]   step 450 loss 0.9453 train-acc 0.975 val-acc 0.867
[01:00:47]   step 500 loss 0.9453 train-acc 0.975 val-acc 0.867
[01:00:48]   step 550 loss 0.9452 train-acc 0.975 val-acc 0.867
[01:00:48]   step 600 loss 0.9452 train-acc 0.975 val-acc 0.867
[01:00:48] best held-out register accuracy 0.867 (chance = 0.20)
[01:00:49] scored pool
[01:00:49] minhash on 92725 candidates
[01:01:04] near-dups removed: 4935
[01:01:04] register 0: 11554 docs, ~6.6M est tokens
[01:01:04] register 1: 2122 docs, ~6.6M est tokens
[01:01:04] register 2: 6450 docs, ~6.6M est tokens
[01:01:04] register 3: 6300 docs, ~6.6M est tokens
[01:01:04] wrote 26426 ids -> /workspace/submission/selection.json

[stdout]
EXIT 0
[00:59:59] loading pool
[01:00:07] 182016 pool docs
[01:00:11] register 0: 1077 target docs
[01:00:11] register 1: 203 target docs
[01:00:12] register 2: 405 target docs
[01:00:12] register 3: 350 target docs
[01:00:12] featurising target
[01:00:13] featurising pool
[01:00:23] features done
[01:00:23] hygiene pass
[01:00:38] hygiene keeps 135094 / 182016
/workspace/submission/curate.py:245: UserWarning: Sparse CSR tensor support is in beta state. If you miss a functionality in the sparse tensor support, please submit a feature request to https://github.com/pytorch/pytorch/issues. (Triggered internally at /pytorch/aten/src/ATen/SparseCsrTensorImpl.cpp:53.)
  return torch.sparse_csr_tensor(
[01:00:39] training classifier
[01:00:41]   step 50 loss 1.0252 train-acc 0.971 val-acc 0.853
[01:00:41]   step 100 loss 0.9679 train-acc 0.972 val-acc 0.855
[01:00:42]   step 150 loss 0.9538 train-acc 0.974 val-acc 0.862
[01:00:43]   step 200 loss 0.9488 train-acc 0.975 val-acc 0.865
[01:00:43]   step 250 loss 0.9469 train-acc 0.976 val-acc 0.865
[01:00:44]   step 300 loss 0.9460 train-acc 0.976 val-acc 0.867
[01:00:45]   step 350 loss 0.9456 train-acc 0.975 val-acc 0.867
[01:00:46]   step 400 loss 0.9454 train-acc 0.975 val-acc 0.867
[01:00:46]   step 450 loss 0.9453 train-acc 0.975 val-acc 0.867
[01:00:47]   step 500 loss 0.9453 train-acc 0.975 val-acc 0.867
[01:00:48]   step 550 loss 0.9452 train-acc 0.975 val-acc 0.867
[01:00:48]   step 600 loss 0.9452 train-acc 0.975 val-acc 0.867
[01:00:48] best held-out register accuracy 0.867 (chance = 0.20)
[01:00:49] scored pool
[01:00:49] minhash on 92725 candidates
[01:01:04] near-dups removed: 4935
[01:01:04] register 0: 11554 docs, ~6.6M est tokens
[01:01:04] register 1: 2122 docs, ~6.6M est tokens
[01:01:04] register 2: 6450 docs, ~6.6M est tokens
[01:01:04] register 3: 6300 docs, ~6.6M est tokens
[01:01:04] wrote 26426 ids -> /workspace/submission/selection.json
[metadata] {"noOutputExpected": false}

==================== wiki/encyclopedic
 id 45423 p= [0.648 0.059 0.157 0.007 0.129] 'Jerry Dupont played for the Toronto Marlboros of the Ontario Hockey League at the age of 16. He was drafted in the first round, 15th overall by the Chicago Blackhawks in the 1980 NHL Entry Draft. He retired in 1987 after seven years of NHL with the Blackhawks '
 id 12762 p= [0.555 0.109 0.111 0.009 0.215] 'The Naviglio di Bereguardo is one of the manmade canals, originally used for internal navigation, which were excavated between the Late Middle Ages and 1800s in the Milan area. Started in 1420, most of it was built between 1457 and 1470, commissioned by France'
==================== news
 id 83442 p= [0.006 0.623 0.052 0.005 0.314] 'DILL AIR FORCE BASE, Fla. — The Obama administration plans to bolster the American military presence in the Persian Gulf after it withdraws the remaining troops from Iraq this year, according to officials and diplomats. That repositioning could include new com'
 id 20760 p= [0.013 0.62  0.056 0.005 0.305] 'Last week Donald Trump officially became the Republican Party’s nominee for President of the United States in 2016. On Thursday he gave his acceptance speech to the Republican National Convention in Cleveland, Ohio. His speech lasted about an hour and fifteen '
==================== web-prose
 id 7576 p= [0.029 0.114 0.726 0.009 0.122] 'Noida: Ace Indian shooter Abhinav Bindra Wednesday said adjusting to the newly-introduced rules will remain a priority for him in the 2013 season. "I would use 2013 as more of a training year. With rules changing in my sport, I think there is a bit of accommod'
 id 16187 p= [0.05  0.149 0.723 0.007 0.07 ] "NEW DELHI (AP) — India's prime minister demanded Wednesday that Italy return two marines accused of killing a pair of fishermen last year or face unspecified consequences, deepening a diplomatic dispute between the two countries.\nThe Indian government allowed "
==================== tech-QA
 id 6138 p= [0.012 0.034 0.028 0.646 0.28 ] 'XForms/Read and write with get and put\nSometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the file when the form is created and you k'
 id 9544 p= [0.03  0.048 0.034 0.632 0.256] "I'm not sure if I worded my topic title properly, which is probably why I haven't been able to search for the answer to my problem just yet (oh, have I tried, just can't find the right keywords I suppose). Anyways, I am trying to get a simple array of all item"

[stdout]
==================== wiki/encyclopedic
 id 45423 p= [0.648 0.059 0.157 0.007 0.129] 'Jerry Dupont played for the Toronto Marlboros of the Ontario Hockey League at the age of 16. He was drafted in the first round, 15th overall by the Chicago Blackhawks in the 1980 NHL Entry Draft. He retired in 1987 after seven years of NHL with the Blackhawks '
 id 12762 p= [0.555 0.109 0.111 0.009 0.215] 'The Naviglio di Bereguardo is one of the manmade canals, originally used for internal navigation, which were excavated between the Late Middle Ages and 1800s in the Milan area. Started in 1420, most of it was built between 1457 and 1470, commissioned by France'
==================== news
 id 83442 p= [0.006 0.623 0.052 0.005 0.314] 'DILL AIR FORCE BASE, Fla. — The Obama administration plans to bolster the American military presence in the Persian Gulf after it withdraws the remaining troops from Iraq this year, according to officials and diplomats. That repositioning could include new com'
 id 20760 p= [0.013 0.62  0.056 0.005 0.305] 'Last week Donald Trump officially became the Republican Party’s nominee for President of the United States in 2016. On Thursday he gave his acceptance speech to the Republican National Convention in Cleveland, Ohio. His speech lasted about an hour and fifteen '
==================== web-prose
 id 7576 p= [0.029 0.114 0.726 0.009 0.122] 'Noida: Ace Indian shooter Abhinav Bindra Wednesday said adjusting to the newly-introduced rules will remain a priority for him in the 2013 season. "I would use 2013 as more of a training year. With rules changing in my sport, I think there is a bit of accommod'
 id 16187 p= [0.05  0.149 0.723 0.007 0.07 ] "NEW DELHI (AP) — India's prime minister demanded Wednesday that Italy return two marines accused of killing a pair of fishermen last year or face unspecified consequences, deepening a diplomatic dispute between the two countries.\nThe Indian government allowed "
==================== tech-QA
 id 6138 p= [0.012 0.034 0.028 0.646 0.28 ] 'XForms/Read and write with get and put\nSometimes all you need to do is to put a nice user friendly form that edits a single static XML file. In this case a static file is any file where you know the exact pathname to the file when the form is created and you k'
 id 9544 p= [0.03  0.048 0.034 0.632 0.256] "I'm not sure if I worded my topic title properly, which is probably why I haven't been able to search for the answer to my problem just yet (oh, have I tried, just can't find the right keywords I suppose). Anyways, I am trying to get a simple array of all item"
[metadata] {"noOutputExpected": false}

File created successfully at: /workspace/pack_fast.py (file state is current in your context — no need to Read it back)

[metadata] {"type": "create", "filePath": "/workspace/pack_fast.py", "content": "\"\"\"Faster equivalent of pack_selection.py (same tokens, parallel + cached pool).\"\"\"\nimport json, os, sys, numpy as np\nfrom multiprocessing import Pool as MPool\nfrom transformers import AutoTokenizer\n\nSEL, OUT = sys.argv[1], sys.argv[2]\nBUDGET = int(sys.argv[3]) if len(sys.argv) > 3 else 12_000_000\n_tok = None\n\ndef init():\n    global _tok\n    os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n    _tok = AutoTokenizer.from_pretrained(\"gpt2\")\n\ndef enc(t):\n    return _tok(t, add_special_tokens=False).input_ids\n\ntext = {}\nfor line in open(\"/workspace/data/pool.jsonl\"):\n    r = json.loads(line); text[r[\"id\"]] = r[\"text\"]\nsel = json.load(open(SEL))\n\n# only tokenize as far as needed: ~4.05 bytes/token, take 1.6x margin\nneed, cut = 0, len(sel)\nfor k, i in enumerate(sel):\n    need += len(text[i]) / 4.05 + 1\n    if need > BUDGET * 1.6:\n        cut = k + 1; break\nsub = sel[:cut]\n\nwith MPool(16, initializer=init) as p:\n    outs = p.map(enc, [text[i] for i in sub], chunksize=16)\n\nparts, tot = [], 0\nEOS = 50256\nfor ids in outs:\n    parts.append(np.array(ids + [EOS], dtype=np.uint16)); tot += len(ids) + 1\n    if tot >= BUDGET: break\narr = np.concatenate(parts)[:BUDGET]\nnp.save(OUT, arr)\nprint(f\"packed {len(arr)} tokens from {len(sel)} ids ({cut} tokenized) -> {OUT}\")\n", "structuredPatch": [], "originalFile": null, "userModified": false}

random baseline written 60000
[01:02:08] register 2: 6450 docs, ~6.6M est tokens
[01:02:08] register 3: 6300 docs, ~6.6M est tokens
[01:02:08] wrote 20762 ids -> /tmp/sel_pooled.json
[1]   Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pack_$v.log 2>&1 )
[2]-  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pack_$v.log 2>&1 )
[3]+  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pack_$v.log 2>&1 )
tail: option used in invalid context -- 1
4948

[stdout]
random baseline written 60000
[01:02:08] register 2: 6450 docs, ~6.6M est tokens
[01:02:08] register 3: 6300 docs, ~6.6M est tokens
[01:02:08] wrote 20762 ids -> /tmp/sel_pooled.json
[1]   Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pack_$v.log 2>&1 )
[2]-  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pack_$v.log 2>&1 )
[3]+  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pack_$v.log 2>&1 )
tail: option used in invalid context -- 1
4948
[metadata] {"noOutputExpected": false}

random: packed 12000000 tokens from 60000 ids (18467 tokenized) -> /tmp/tr_random.npy
balanced: packed 12000000 tokens from 26426 ids (16931 tokenized) -> /tmp/tr_balanced.npy
pooled: packed 12000000 tokens from 20762 ids (14948 tokenized) -> /tmp/tr_pooled.npy
-rw-r--r-- 1 root root 24000128 Aug  4 01:03 /tmp/tr_balanced.npy
-rw-r--r-- 1 root root 24000128 Aug  4 01:03 /tmp/tr_pooled.npy
-rw-r--r-- 1 root root 24000128 Aug  4 01:03 /tmp/tr_random.npy

[stdout]
random: packed 12000000 tokens from 60000 ids (18467 tokenized) -> /tmp/tr_random.npy
balanced: packed 12000000 tokens from 26426 ids (16931 tokenized) -> /tmp/tr_balanced.npy
pooled: packed 12000000 tokens from 20762 ids (14948 tokenized) -> /tmp/tr_pooled.npy
-rw-r--r-- 1 root root 24000128 Aug  4 01:03 /tmp/tr_balanced.npy
-rw-r--r-- 1 root root 24000128 Aug  4 01:03 /tmp/tr_pooled.npy
-rw-r--r-- 1 root root 24000128 Aug  4 01:03 /tmp/tr_random.npy
[metadata] {"noOutputExpected": false}

[1]   Done                    ( python train_nano.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/ev_$v.json --max_iters 3000 --seed 1337 > /tmp/tro_$v.log 2>&1 )
[2]-  Done                    ( python train_nano.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/ev_$v.json --max_iters 3000 --seed 1337 > /tmp/tro_$v.log 2>&1 )
[3]+  Done                    ( python train_nano.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/ev_$v.json --max_iters 3000 --seed 1337 > /tmp/tro_$v.log 2>&1 )
HELDOUT_PPL 333.298  loss 5.8090  train_s 178  /tmp/tr_balanced.npy
HELDOUT_PPL 477.843  loss 6.1693  train_s 178  /tmp/tr_random.npy
HELDOUT_PPL 444.425  loss 6.0968  train_s 178  /tmp/tr_pooled.npy
4748

[stdout]
[1]   Done                    ( python train_nano.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/ev_$v.json --max_iters 3000 --seed 1337 > /tmp/tro_$v.log 2>&1 )
[2]-  Done                    ( python train_nano.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/ev_$v.json --max_iters 3000 --seed 1337 > /tmp/tro_$v.log 2>&1 )
[3]+  Done                    ( python train_nano.py --train_npy /tmp/tr_$v.npy --heldout_npy data/multi_dev.npy --out_json /tmp/ev_$v.json --max_iters 3000 --seed 1337 > /tmp/tro_$v.log 2>&1 )
HELDOUT_PPL 333.298  loss 5.8090  train_s 178  /tmp/tr_balanced.npy
HELDOUT_PPL 477.843  loss 6.1693  train_s 178  /tmp/tr_random.npy
HELDOUT_PPL 444.425  loss 6.0968  train_s 178  /tmp/tr_pooled.npy
4748
[metadata] {"noOutputExpected": false}

quarters written 250000
[1]   Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
[2]   Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
[3]-  Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
[4]+  Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
register 0: HELDOUT_PPL 848.703
register 1: HELDOUT_PPL 267.878
register 2: HELDOUT_PPL 336.009
register 3: HELDOUT_PPL 160.883

[stdout]
quarters written 250000
[1]   Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
[2]   Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
[3]-  Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
[4]+  Done                    ( python train_nano.py --train_npy /tmp/tr_balanced.npy --heldout_npy /tmp/dev_q$r.npy --out_json /tmp/q$r.json --max_iters 3000 --seed 1337 > /tmp/q$r.log 2>&1 )
register 0: HELDOUT_PPL 848.703
register 1: HELDOUT_PPL 267.878
register 2: HELDOUT_PPL 336.009
register 3: HELDOUT_PPL 160.883
[metadata] {"noOutputExpected": false}

reg 0 artifact-ish token share 0.127
reg 1 artifact-ish token share 0.004
reg 2 artifact-ish token share 0.004
reg 3 artifact-ish token share 0.015

pool docs with punct: 116582
spaced-punct frac pctiles [0.    0.031 0.188 0.268 0.617]
docs with frac>0.5: 157  >0.3: 457  >0.15: 1658
1.0 176291 140182
1.0 175090 6541
1.0 172730 6227
1.0 152178 140647
1.0 67363 1595

[stdout]
reg 0 artifact-ish token share 0.127
reg 1 artifact-ish token share 0.004
reg 2 artifact-ish token share 0.004
reg 3 artifact-ish token share 0.015

pool docs with punct: 116582
spaced-punct frac pctiles [0.    0.031 0.188 0.268 0.617]
docs with frac>0.5: 157  >0.3: 457  >0.15: 1658
1.0 176291 140182
1.0 175090 6541
1.0 172730 6227
1.0 152178 140647
1.0 67363 1595
[metadata] {"noOutputExpected": false}

est tokens in docs with frac>0.15: 2.80M
est tokens frac>0.3: 0.96M
--- frac 1.00 id 176291 len 140182
" du site d'apercu\nTelecharger pdf livre gratuitement\nNom de fichier\nDescription\nType de fichier\nplaying the changes guitar a linear approach to improvising softcover with cd Description A propos de playing the changes guitar a linear approach to improvising softcover with cd Pas Disponible Telecharger playing the changes guitar "
--- frac 1.00 id 175090 len 6541
'com<|endoftext|>97 Camaro Fuse Box - Auto Electrical Wiring Diagram\nWiring Diagram\nHome\n97 camaro fuse box\nWiring Diagram | Schema Cablage | Diagrama De Cableado | Ledningsdiagram | Del Schaltplan | Bedradings Schema | Schaltplang\nNew Update\n2003 mazda 6 cooling system diagram , resistor led circuit , midwest spa panel wiring di'
--- frac 1.00 id 172730 len 6227
' Schema Cablage - Auto Electrical Wiring Diagram\nWiring Diagram\nHome\n1990 cutlass supreme schema cablage\nWiring Diagram | Schema Cablage | Diagrama De Cableado | Ledningsdiagram | Del Schaltplan | Bedradings Schema | Schaltplang\nAnother Wiring Diagram Related With 1990 cutlass supreme schema cablage\n5 pin relay Schaltplang groun'
--- frac 0.37 id 75714 len 1393
'.<|endoftext|>|Reviews for In the Water|\n| pinksugarrush chapter 1 . 8/23/2013\nI absolutely love the ending!\nKeep up the great work!\n| Guest chapter 1 . 1/25/2013\nI love the last few lines :) Hothothot! :D\n| Guest chapter 1 . 10/5/2012\nOMG so turned on if I was a guy I would have a boner\n| xTheTomatoBoxFairyx chapter 1 . 6/18/20'
--- frac 0.37 id 50476 len 1816
"<|endoftext|>Saturday, August 11, 2007\nA baby blessing\nmade this for our friend's baby boy together with the tiny baby card.\nIt was supposed to be a regular Friday morning with cooking except that we had to be at a Brit(circumcision party/event) at 12:30pm .Well I was making laundry the day before ,when I realized that the machi"
--- frac 0.36 id 126266 len 8338
'1904.14)<|endoftext|>Stock Market News | NSDQ, NYSE, and AMEX Stock Market News, Market News Categories, Market Indicators | Business\nAll Sections\nSearch\nStar Tribune\nLog In\nWelcome, User\nManage Account\neEdition\nSupport\nSubscribe\nLog Out\nSupport\neEdition\nSubscribe\n44°\nForecast\nTraffic\nBusiness\nHome\nLocal\nSports\nBusiness\nOpinion\n'
--- frac 0.18 id 172482 len 5425
' myembassy.net<|endoftext|>SPB Section Metric V-Belts (16.3mm Top Width, 13mm Thick ... - This must be seen!\nToggle navigation\nmajalahkartini.online\nCategories - 35 categories\nTeen sex\nSolo\nLingerie sex\nAmateur porn\nPorn orgy\nBeautiful women\nInterracial\nAnal porn\nPorn butts\nPantyhose women\nExotica\nHard fucking\nSexy legs\nSexy bik'
--- frac 0.18 id 169127 len 4713
'obs powered by<|endoftext|>Mazut, sell trade lead, Moscow\n|\n|\nphone_in_talk +91-9210 373801\nBuy\nAll Buy Offers\nPost Buy Offer\n|\nSell\nAll Sell Offers\nPost Sell Offer\n|\nBiz\nAgent / Distributor Leads\nPost Biz Offer\n|\nJoin Free\n|\nSign inperson\nSign In\nUser ID / Email\nperson\nError Text Massage\nPassword\nvpn_key\nvisibility_off\nRemember'

[stdout]
est tokens in docs with frac>0.15: 2.80M
est tokens frac>0.3: 0.96M
--- frac 1.00 id 176291 len 140182
" du site d'apercu\nTelecharger pdf livre gratuitement\nNom de fichier\nDescription\nType de fichier\nplaying the changes guitar a linear approach to improvising softcover with cd Description A propos de playing the changes guitar a linear approach to improvising softcover with cd Pas Disponible Telecharger playing the changes guitar "
--- frac 1.00 id 175090 len 6541
'com<|endoftext|>97 Camaro Fuse Box - Auto Electrical Wiring Diagram\nWiring Diagram\nHome\n97 camaro fuse box\nWiring Diagram | Schema Cablage | Diagrama De Cableado | Ledningsdiagram | Del Schaltplan | Bedradings Schema | Schaltplang\nNew Update\n2003 mazda 6 cooling system diagram , resistor led circuit , midwest spa panel wiring di'
--- frac 1.00 id 172730 len 6227
' Schema Cablage - Auto Electrical Wiring Diagram\nWiring Diagram\nHome\n1990 cutlass supreme schema cablage\nWiring Diagram | Schema Cablage | Diagrama De Cableado | Ledningsdiagram | Del Schaltplan | Bedradings Schema | Schaltplang\nAnother Wiring Diagram Related With 1990 cutlass supreme schema cablage\n5 pin relay Schaltplang groun'
--- frac 0.37 id 75714 len 1393
'.<|endoftext|>|Reviews for In the Water|\n| pinksugarrush chapter 1 . 8/23/2013\nI absolutely love the ending!\nKeep up the great work!\n| Guest chapter 1 . 1/25/2013\nI love the last few lines :) Hothothot! :D\n| Guest chapter 1 . 10/5/2012\nOMG so turned on if I was a guy I would have a boner\n| xTheTomatoBoxFairyx chapter 1 . 6/18/20'
--- frac 0.37 id 50476 len 1816
"<|endoftext|>Saturday, August 11, 2007\nA baby blessing\nmade this for our friend's baby boy together with the tiny baby card.\nIt was supposed to be a regular Friday morning with cooking except that we had to be at a Brit(circumcision party/event) at 12:30pm .Well I was making laundry the day before ,when I realized that the machi"
--- frac 0.36 id 126266 len 8338
'1904.14)<|endoftext|>Stock Market News | NSDQ, NYSE, and AMEX Stock Market News, Market News Categories, Market Indicators | Business\nAll Sections\nSearch\nStar Tribune\nLog In\nWelcome, User\nManage Account\neEdition\nSupport\nSubscribe\nLog Out\nSupport\neEdition\nSubscribe\n44°\nForecast\nTraffic\nBusiness\nHome\nLocal\nSports\nBusiness\nOpinion\n'
--- frac 0.18 id 172482 len 5425
' myembassy.net<|endoftext|>SPB Section Metric V-Belts (16.3mm Top Width, 13mm Thick ... - This must be seen!\nToggle navigation\nmajalahkartini.online\nCategories - 35 categories\nTeen sex\nSolo\nLingerie sex\nAmateur porn\nPorn orgy\nBeautiful women\nInterracial\nAnal porn\nPorn butts\nPantyhose women\nExotica\nHard fucking\nSexy legs\nSexy bik'
--- frac 0.18 id 169127 len 4713
'obs powered by<|endoftext|>Mazut, sell trade lead, Moscow\n|\n|\nphone_in_talk +91-9210 373801\nBuy\nAll Buy Offers\nPost Buy Offer\n|\nSell\nAll Sell Offers\nPost Sell Offer\n|\nBiz\nAgent / Distributor Leads\nPost Biz Offer\n|\nJoin Free\n|\nSign inperson\nSign In\nUser ID / Email\nperson\nError Text Massage\nPassword\nvpn_key\nvisibility_off\nRemember'
[metadata] {"noOutputExpected": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    # ---- balanced fill\n    valid = ok & ~dup\n    per_reg_budget = BUDGET * a.overfill / N_REG\n    chosen = [[] for _ in range(N_REG)]\n    taken = np.zeros(N, bool)\n    for r in range(N_REG):\n        s = reg_score[:, r].copy()\n        s[~valid] = -1.0\n        o = np.argsort(-s)\n        tot = 0\n        for i in o:\n            if s[i] <= -1.0 or taken[i]:\n                continue\n            taken[i] = True\n            chosen[r].append(int(ids[i]))\n            tot += int(est_tok[i])\n            if tot >= per_reg_budget:\n                break\n        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.1f}M est tokens\")\n\n    if a.mode == \"pooled\":\n        s = reg_score.sum(1).copy()\n        s[~valid] = -1.0\n        o = [i for i in np.argsort(-s) if s[i] > -1.0]\n        sel, tot = [], 0\n        for i in o:\n            sel.append(int(ids[i])); tot += int(est_tok[i])\n            if tot >= BUDGET * a.overfill:\n                break\n    else:\n        # round-robin interleave so every prefix is register-balanced\n        sel = []\n        ptr = [0] * N_REG\n        while any(ptr[r] < len(chosen[r]) for r in range(N_REG)):\n            for r in range(N_REG):\n                if ptr[r] < len(chosen[r]):\n                    sel.append(chosen[r][ptr[r]]); ptr[r] += 1", "newString": "    # ---- fill, with an explicit TOKEN-share target per register\n    valid = ok & ~dup\n    w = np.array([float(x) for x in a.weights.split(\",\")], np.float64)\n    assert len(w) == N_REG\n    w = w / w.sum()\n    log(f\"target token mix {np.round(w, 3).tolist()}\")\n\n    # Each register claims its own docs best-first; a doc goes to whichever\n    # register wants it most (argmax score) to avoid double-counting.\n    owner = reg_score.argmax(1)\n    chosen = [[] for _ in range(N_REG)]     # (id, est_tokens) best-first\n    for r in range(N_REG):\n        s = reg_score[:, r].copy()\n        s[~valid] = -1.0\n        s[owner != r] = -1.0\n        o = np.argsort(-s)\n        tot = 0\n        cap = BUDGET * a.overfill * w[r]\n        for i in o:\n            if s[i] <= -1.0:\n                continue\n            chosen[r].append((int(ids[i]), int(est_tok[i])))\n            tot += int(est_tok[i])\n            if tot >= cap:\n                break\n        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.2f}M est tokens \"\n            f\"(cap {cap/1e6:.2f}M)\")\n\n    if a.mode == \"pooled\":\n        s = reg_score[:, :N_REG].sum(1).copy()\n        s[~valid] = -1.0\n        sel, tot = [], 0\n        for i in np.argsort(-s):\n            if s[i] <= -1.0:\n                continue\n            sel.append(int(ids[i])); tot += int(est_tok[i])\n            if tot >= BUDGET * a.overfill:\n                break\n    else:\n        # Interleave by TOKEN share, not doc count: always emit from the register\n        # that is currently furthest below its quota. Documents differ ~5x in\n        # length between registers, so doc-count round-robin would badly skew the\n        # realised token mix. This makes every prefix weight-correct.\n        sel = []\n        ptr = [0] * N_REG\n        cum = np.zeros(N_REG)\n        while True:\n            avail = [r for r in range(N_REG) if ptr[r] < len(chosen[r])]\n            if not avail:\n                break\n            r = min(avail, key=lambda r: cum[r] / w[r])\n            i, nt = chosen[r][ptr[r]]; ptr[r] += 1\n            sel.append(i); cum[r] += nt\n        # report the realised mix over the part that actually fits the budget\n        c2, tot = np.zeros(N_REG), 0\n        for r_, (i, nt) in ((owner[np.searchsorted(ids, i)], (i, nt))\n                            for r in range(N_REG) for (i, nt) in chosen[r]):\n            pass\n        log(f\"emitted {len(sel)} ids; per-register est tokens \"\n            f\"{np.round(cum/1e6, 2).tolist()}M\")", "originalFile": null, "structuredPatch": [{"oldStart": 362, "oldLines": 43, "newStart": 362, "newLines": 66, "lines": ["             seen_bands[bi][k] = gid", "     log(f\"near-dups removed: {dup.sum()}\")", " ", "-    # ---- balanced fill", "+    # ---- fill, with an explicit TOKEN-share target per register", "     valid = ok & ~dup", "-    per_reg_budget = BUDGET * a.overfill / N_REG", "-    chosen = [[] for _ in range(N_REG)]", "-    taken = np.zeros(N, bool)", "+    w = np.array([float(x) for x in a.weights.split(\",\")], np.float64)", "+    assert len(w) == N_REG", "+    w = w / w.sum()", "+    log(f\"target token mix {np.round(w, 3).tolist()}\")", "+", "+    # Each register claims its own docs best-first; a doc goes to whichever", "+    # register wants it most (argmax score) to avoid double-counting.", "+    owner = reg_score.argmax(1)", "+    chosen = [[] for _ in range(N_REG)]     # (id, est_tokens) best-first", "     for r in range(N_REG):", "         s = reg_score[:, r].copy()", "         s[~valid] = -1.0", "+        s[owner != r] = -1.0", "         o = np.argsort(-s)", "         tot = 0", "+        cap = BUDGET * a.overfill * w[r]", "         for i in o:", "-            if s[i] <= -1.0 or taken[i]:", "+            if s[i] <= -1.0:", "                 continue", "-            taken[i] = True", "-            chosen[r].append(int(ids[i]))", "+            chosen[r].append((int(ids[i]), int(est_tok[i])))", "             tot += int(est_tok[i])", "-            if tot >= per_reg_budget:", "+            if tot >= cap:", "                 break", "-        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.1f}M est tokens\")", "+        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.2f}M est tokens \"", "+            f\"(cap {cap/1e6:.2f}M)\")", " ", "     if a.mode == \"pooled\":", "-        s = reg_score.sum(1).copy()", "+        s = reg_score[:, :N_REG].sum(1).copy()", "         s[~valid] = -1.0", "-        o = [i for i in np.argsort(-s) if s[i] > -1.0]", "         sel, tot = [], 0", "-        for i in o:", "+        for i in np.argsort(-s):", "+            if s[i] <= -1.0:", "+                continue", "             sel.append(int(ids[i])); tot += int(est_tok[i])", "             if tot >= BUDGET * a.overfill:", "                 break", "     else:", "-        # round-robin interleave so every prefix is register-balanced", "+        # Interleave by TOKEN share, not doc count: always emit from the register", "+        # that is currently furthest below its quota. Documents differ ~5x in", "+        # length between registers, so doc-count round-robin would badly skew the", "+        # realised token mix. This makes every prefix weight-correct.", "         sel = []", "         ptr = [0] * N_REG", "-        while any(ptr[r] < len(chosen[r]) for r in range(N_REG)):", "-            for r in range(N_REG):", "-                if ptr[r] < len(chosen[r]):", "-                    sel.append(chosen[r][ptr[r]]); ptr[r] += 1", "+        cum = np.zeros(N_REG)", "+        while True:", "+            avail = [r for r in range(N_REG) if ptr[r] < len(chosen[r])]", "+            if not avail:", "+                break", "+            r = min(avail, key=lambda r: cum[r] / w[r])", "+            i, nt = chosen[r][ptr[r]]; ptr[r] += 1", "+            sel.append(i); cum[r] += nt", "+        # report the realised mix over the part that actually fits the budget", "+        c2, tot = np.zeros(N_REG), 0", "+        for r_, (i, nt) in ((owner[np.searchsorted(ids, i)], (i, nt))", "+                            for r in range(N_REG) for (i, nt) in chosen[r]):", "+            pass", "+        log(f\"emitted {len(sel)} ids; per-register est tokens \"", "+            f\"{np.round(cum/1e6, 2).tolist()}M\")", " ", "     assert len(sel) == len(set(sel))", "     json.dump(sel, open(a.out, \"w\"))"]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "        # report the realised mix over the part that actually fits the budget\n        c2, tot = np.zeros(N_REG), 0\n        for r_, (i, nt) in ((owner[np.searchsorted(ids, i)], (i, nt))\n                            for r in range(N_REG) for (i, nt) in chosen[r]):\n            pass\n        log(f\"emitted {len(sel)} ids; per-register est tokens \"", "newString": "        log(f\"emitted {len(sel)} ids; per-register est tokens \"", "originalFile": null, "structuredPatch": [{"oldStart": 415, "oldLines": 11, "newStart": 415, "newLines": 6, "lines": ["             r = min(avail, key=lambda r: cum[r] / w[r])", "             i, nt = chosen[r][ptr[r]]; ptr[r] += 1", "             sel.append(i); cum[r] += nt", "-        # report the realised mix over the part that actually fits the budget", "-        c2, tot = np.zeros(N_REG), 0", "-        for r_, (i, nt) in ((owner[np.searchsorted(ids, i)], (i, nt))", "-                            for r in range(N_REG) for (i, nt) in chosen[r]):", "-            pass", "         log(f\"emitted {len(sel)} ids; per-register est tokens \"", "             f\"{np.round(cum/1e6, 2).tolist()}M\")", " "]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    ap.add_argument(\"--temp\", type=float, default=0.0,\n                    help=\"0 = strict top-k per register\")", "newString": "    ap.add_argument(\"--weights\", default=\"0.25,0.25,0.25,0.25\",\n                    help=\"target TOKEN share per register (wiki,news,web,techQA)\")", "originalFile": null, "structuredPatch": [{"oldStart": 179, "oldLines": 8, "newStart": 179, "newLines": 8, "lines": ["                     choices=[\"balanced\", \"pooled\"])", "     ap.add_argument(\"--overfill\", type=float, default=2.2,", "                     help=\"emit this many x budget worth of tokens\")", "-    ap.add_argument(\"--temp\", type=float, default=0.0,", "-                    help=\"0 = strict top-k per register\")", "+    ap.add_argument(\"--weights\", default=\"0.25,0.25,0.25,0.25\",", "+                    help=\"target TOKEN share per register (wiki,news,web,techQA)\")", "     a = ap.parse_args()", " ", "     # ---- load pool"]}], "userModified": false, "replaceAll": false}

[eq] [01:13:49] target token mix [0.25, 0.25, 0.25, 0.25]
[eq] [01:13:50] emitted 29759 ids; per-register est tokens [7.2, 7.21, 7.2, 1.15]M
[w0] [01:14:20] target token mix [0.4, 0.2, 0.2, 0.2]
[w0] [01:14:21] emitted 29364 ids; per-register est tokens [8.26, 5.76, 5.76, 1.15]M
[w0b] [01:14:51] target token mix [0.34, 0.18, 0.24, 0.24]
[w0b] [01:14:51] emitted 31119 ids; per-register est tokens [8.26, 5.2, 6.91, 1.15]M
[1]   Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pk_$v.log 2>&1 )
[2]-  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pk_$v.log 2>&1 )
[3]+  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pk_$v.log 2>&1 )
packed 12000000 tokens from 29364 ids (27580 tokenized) -> /tmp/tr_w0.npy
packed 12000000 tokens from 29759 ids (24907 tokenized) -> /tmp/tr_eq.npy
packed 12000000 tokens from 31119 ids (28037 tokenized) -> /tmp/tr_w0b.npy
4176

[stdout]
[eq] [01:13:49] target token mix [0.25, 0.25, 0.25, 0.25]
[eq] [01:13:50] emitted 29759 ids; per-register est tokens [7.2, 7.21, 7.2, 1.15]M
[w0] [01:14:20] target token mix [0.4, 0.2, 0.2, 0.2]
[w0] [01:14:21] emitted 29364 ids; per-register est tokens [8.26, 5.76, 5.76, 1.15]M
[w0b] [01:14:51] target token mix [0.34, 0.18, 0.24, 0.24]
[w0b] [01:14:51] emitted 31119 ids; per-register est tokens [8.26, 5.2, 6.91, 1.15]M
[1]   Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pk_$v.log 2>&1 )
[2]-  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pk_$v.log 2>&1 )
[3]+  Done                    ( python pack_fast.py /tmp/sel_$v.json /tmp/tr_$v.npy > /tmp/pk_$v.log 2>&1 )
packed 12000000 tokens from 29364 ids (27580 tokenized) -> /tmp/tr_w0.npy
packed 12000000 tokens from 29759 ids (24907 tokenized) -> /tmp/tr_eq.npy
packed 12000000 tokens from 31119 ids (28037 tokenized) -> /tmp/tr_w0b.npy
4176
[metadata] {"noOutputExpected": false}

Command running in background with ID: b38ljr7dy. Output is being written to: /tmp/claude-0/-workspace/5c34ea41-584d-44c9-bb08-1f0e72305853/tasks/b38ljr7dy.output. You will be notified when it completes. To check interim output, use Read on that file path.
Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands.

[metadata] {"noOutputExpected": false, "backgroundTaskId": "b38ljr7dy", "backgroundCwdHint": "Session cwd remains /workspace; directory changes made by the backgrounded command do not apply to subsequent commands."}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "    # Each register claims its own docs best-first; a doc goes to whichever\n    # register wants it most (argmax score) to avoid double-counting.\n    owner = reg_score.argmax(1)\n    chosen = [[] for _ in range(N_REG)]     # (id, est_tokens) best-first\n    for r in range(N_REG):\n        s = reg_score[:, r].copy()\n        s[~valid] = -1.0\n        s[owner != r] = -1.0\n        o = np.argsort(-s)\n        tot = 0\n        cap = BUDGET * a.overfill * w[r]\n        for i in o:\n            if s[i] <= -1.0:\n                continue\n            chosen[r].append((int(ids[i]), int(est_tok[i])))\n            tot += int(est_tok[i])\n            if tot >= cap:\n                break\n        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.2f}M est tokens \"\n            f\"(cap {cap/1e6:.2f}M)\")", "newString": "    # Each register ranks ALL surviving docs by its own score. Registers are not\n    # given exclusive ownership: technical Q&A is rare in this web pool, and an\n    # argmax-ownership rule starves it far below quota. Contention is instead\n    # resolved during the interleave (first register to reach a doc takes it),\n    # which lets the best-matching register pick first while still allowing a\n    # scarce register to fill its quota from its next-best documents.\n    cand_order = []\n    for r in range(N_REG):\n        s = reg_score[:, r].copy()\n        s[~valid] = -1.0\n        o = np.argsort(-s)\n        o = o[s[o] > -1.0]\n        cand_order.append(o)\n        log(f\"register {r}: {len(o)} eligible docs, \"\n            f\"top-score {s[o[0]]:.3f}, quota {BUDGET*w[r]/1e6:.2f}M tokens\")\n    chosen = None", "originalFile": null, "structuredPatch": [{"oldStart": 369, "oldLines": 26, "newStart": 369, "newLines": 22, "lines": ["     w = w / w.sum()", "     log(f\"target token mix {np.round(w, 3).tolist()}\")", " ", "-    # Each register claims its own docs best-first; a doc goes to whichever", "-    # register wants it most (argmax score) to avoid double-counting.", "-    owner = reg_score.argmax(1)", "-    chosen = [[] for _ in range(N_REG)]     # (id, est_tokens) best-first", "+    # Each register ranks ALL surviving docs by its own score. Registers are not", "+    # given exclusive ownership: technical Q&A is rare in this web pool, and an", "+    # argmax-ownership rule starves it far below quota. Contention is instead", "+    # resolved during the interleave (first register to reach a doc takes it),", "+    # which lets the best-matching register pick first while still allowing a", "+    # scarce register to fill its quota from its next-best documents.", "+    cand_order = []", "     for r in range(N_REG):", "         s = reg_score[:, r].copy()", "         s[~valid] = -1.0", "-        s[owner != r] = -1.0", "         o = np.argsort(-s)", "-        tot = 0", "-        cap = BUDGET * a.overfill * w[r]", "-        for i in o:", "-            if s[i] <= -1.0:", "-                continue", "-            chosen[r].append((int(ids[i]), int(est_tok[i])))", "-            tot += int(est_tok[i])", "-            if tot >= cap:", "-                break", "-        log(f\"register {r}: {len(chosen[r])} docs, ~{tot/1e6:.2f}M est tokens \"", "-            f\"(cap {cap/1e6:.2f}M)\")", "+        o = o[s[o] > -1.0]", "+        cand_order.append(o)", "+        log(f\"register {r}: {len(o)} eligible docs, \"", "+            f\"top-score {s[o[0]]:.3f}, quota {BUDGET*w[r]/1e6:.2f}M tokens\")", "+    chosen = None", " ", "     if a.mode == \"pooled\":", "         s = reg_score[:, :N_REG].sum(1).copy()"]}], "userModified": false, "replaceAll": false}

The file /workspace/submission/curate.py has been updated successfully. (file state is current in your context — no need to Read it back)

[metadata] {"filePath": "/workspace/submission/curate.py", "oldString": "        sel = []\n        ptr = [0] * N_REG\n        cum = np.zeros(N_REG)\n        while True:\n            avail = [r for r in range(N_REG) if ptr[r] < len(chosen[r])]\n            if not avail:\n                break\n            r = min(avail, key=lambda r: cum[r] / w[r])\n            i, nt = chosen[r][ptr[r]]; ptr[r] += 1\n            sel.append(i); cum[r] += nt\n        log(f\"emitted {len(sel)} ids; per-register est tokens \"\n            f\"{np.round(cum/1e6, 2).tolist()}M\")", "newString": "        sel = []\n        ptr = [0] * N_REG\n        cum = np.zeros(N_REG)\n        used = np.zeros(N, bool)\n        target = BUDGET * a.overfill\n        cum_at_budget = None\n        while cum.sum() < target:\n            avail = [r for r in range(N_REG) if ptr[r] < len(cand_order[r])]\n            if not avail:\n                break\n            r = min(avail, key=lambda r: cum[r] / w[r])\n            # advance this register's cursor past docs already taken\n            i = None\n            while ptr[r] < len(cand_order[r]):\n                c = cand_order[r][ptr[r]]; ptr[r] += 1\n                if not used[c]:\n                    i = c; break\n            if i is None:\n                continue\n            used[i] = True\n            sel.append(int(ids[i])); cum[r] += int(est_tok[i])\n            if cum_at_budget is None and cum.sum() >= BUDGET:\n                cum_at_budget = cum.copy()\n        if cum_at_budget is None:\n            cum_at_budget = cum.copy()\n        log(f\"emitted {len(sel)} ids (~{cum.sum()/1e6:.1f}M est tokens)\")\n        log(f\"realised mix within the 12M budget: \"\n            f\"{np.round(cum_at_budget / cum_at_budget.sum(), 3).tolist()}\")", "originalFile": null, "structuredPatch": [{"oldStart": 404, "oldLines": 15, "newStart": 404, "newLines": 31, "lines": ["         sel = []", "         ptr = [0] * N_REG", "         cum = np.zeros(N_REG)", "-        while True:", "-            avail = [r for r in range(N_REG) if ptr[r] < len(chosen[r])]", "+        used = np.zeros(N, bool)", "+        target = BUDGET * a.overfill", "+        cum_at_budget = None", "+        while cum.sum() < target:", "+            avail = [r for r in range(N_REG) if ptr[r] < len(cand_order[r])]", "             if not avail:", "                 break", "             r = min(avail, key=lambda r: cum[r] / w[r])", "-            i, nt = chosen[r][ptr[r]]; ptr[r] += 1", "-            sel.append(i); cum[r] += nt", "-        log(f\"emitted {len(sel)} ids; per-register est tokens \"", "-            f\"{np.round(cum/1e6, 2).tolist()}M\")", "+            # advance this register's cursor past docs already taken", "+            i = None", "+            while ptr[r] < len(cand_order[r]):", "+                c = cand_order[r][ptr[r]]; ptr[r] += 1", "+                if not used[c]:", "+                    i = c; break", "+            if i is None:", "+                continue", "+            used[i] = True", "+            sel.append(int(ids[i])); cum[r] += int(est_tok[i])", "+            if cum_at_budget is None and cum.sum() >= BUDGET:", "+                cum_at_budget = cum.copy()", "+        if cum_at_budget is None:", "+            cum_at_budget = cum.copy()", "+        log(f\"emitted {len(sel)} ids (~{cum.sum()/1e6:.1f}M est tokens)\")", "+        log(f\"realised mix within the 12M budget: \"", "+            f\"{np.round(cum_at_budget / cum_at_budget.sum(), 3).tolist()}\")", " ", "     assert len(sel) == len(set(sel))", "     json.dump(sel, open(a.out, \"w\"))"]}], "userModified": false, "replaceAll": false}

4036

[stdout]
4036
[metadata] {"noOutputExpected": false}