gpt-5.6-sol

Script completed
Wall time 7.7 seconds
Output:

Script running with cell ID 132
Wall time 11.0 seconds
Output:

Script completed
Wall time 11.8 seconds
Output:
Token indices sequence length is longer than the specified maximum sequence length for this model (1251 > 1024). Running this sequence through the model will result in indexing errors
documents 16564 domain_tokens [3350001, 3350741, 3351560, 1108738] prefix_tokens 11161040

Script completed
Wall time 5.7 seconds
Output:
[120, 8, 0, 0, 0, 0] [0, 0, 0, 0, 0, 0]

Script completed
Wall time 10.2 seconds
Output:

Script completed
Wall time 0.1 seconds
Output:
  PID     ELAPSED CMD
3278

Script completed
Wall time 0.0 seconds
Output:
{}

Script completed
Wall time 0.2 seconds
Output:
de9964ee9c4cccd010efc25781f79b0f543d5c0deff99a9811f4f32c1545b29f  submission/selection.json
de9964ee9c4cccd010efc25781f79b0f543d5c0deff99a9811f4f32c1545b29f  /tmp/cand4_selection.json
{'ids': 16564, 'unique': 16564, 'min': 17, 'max': 181984}
{"train_npy": "/tmp/cand4.npy", "heldout_ppl": 375.83288278347226, "mean_loss": 5.929144583905714, "max_iters": 3000, "seed": 1337, "train_s": 54.8}3258

Script completed
Wall time 0.1 seconds
Output:
#!/usr/bin/env python3
"""Rank raw web pages by clean-prose and target-register heuristics.

The selector is deliberately label-free.  It assigns each page to one of four
registers (encyclopedic, general prose, news, technical Q&A), rejects common
web-extraction failure modes, and interleaves the four quality-ranked queues by
estimated GPT-2 tokens.  The interleaving prevents an easy-to-find register
such as news from consuming the complete fixed training budget.
"""
import json, math, re
from pathlib import Path
from transformers import GPT2TokenizerFast

POOL = Path('/workspace/data/pool.jsonl')
OUT = Path('/workspace/submission/selection.json')
TARGET_PER_DOMAIN = 3_350_000   # surplus makes the output safely exceed 12M

word_re = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?")

def score_doc(t):
    n=len(t); words=word_re.findall(t); nw=len(words)
    rough_tech = '<code>' in t.lower() or '<pre>' in t.lower()
    if n < 900 or n > 80_000 or (n > 30_000 and not rough_tech) or nw < 150: return None
    low=t.lower(); lines=t.splitlines(); alpha=sum(c.isalpha() for c in t)/n
    # Measures extraction cleanliness and sustained prose.
    sent=len(re.findall(r'[.!?](?:[\"\')\]]+)?(?:\s|$)',t))
    avg=n/max(nw,1); short=sum(len(x.strip())<35 for x in lines)/max(len(lines),1)
    uniq=len(set(w.lower() for w in words))/nw
    bad=sum(low.count(x) for x in ('cookie policy','enable javascript','shopping cart',
        'all rights reserved','skip to content','privacy policy','sign up','log in',
        'free shipping','add to cart','search results','subscribe to our','404 not found'))
    eots=low.count('<|endoftext|>')
    if eots > 2 and not rough_tech: return None
    stop={'the','of','to','and','a','in','is','that','for','it','on','with','as','was',
          'are','be','by','this','from','at','or','an','have','has','not','but','which'}
    stopfrac=sum(w.lower() in stop for w in words)/nw
    weird=sum(not(c.isalnum() or c.isspace() or c in ".,;:!?()[]{}'\"-/+*=<>%&@#$£€_`~|") for c in t)/n
    repeat=1-uniq
    q = 2.8*alpha + .18*math.log1p(sent) - .24*abs(avg-5.4)
    q += .25*min(uniq,.55) - .7*short - .20*bad - 3*weird
    q -= 7*abs(stopfrac-.27)
    adult=sum(low.count(x) for x in (' porn',' fuck',' pussy',' blowjob',' escort',' horny',
                                      ' sex video',' nude girls',' viagra',' casino bonus'))
    q -= .45*adult
    if sent < 6 or alpha < .62 or avg > 7.2 or avg < 3.8: q -= 2
    if repeat > .78: q -= 1

    html=sum(low.count(x) for x in ('<p>','<pre>','<code>','</a>','blockquote'))
    tech=sum(low.count(x) for x in (' error ',' function ',' code ',' python',' java',
        ' javascript',' database',' server',' class ',' method ',' linux',' api ',' sql',
        ' compiler',' algorithm',' command line',' stack trace',' exception'))
    news=sum(low.count(x) for x in ('according to','reported',' reporters',' minister',
        'president','government','police','officials','statement','spokesman','agency',
        'election','court',' on monday',' on tuesday',' on wednesday',' on thursday',
        ' on friday','reuters','associated press'))
    encycl=sum(low.count(x) for x in (' was born',' is a ',' was a ',' refers to ',
        ' known as ',' history of',' population',' species',' century',' located in',
        ' consists of',' became the',' university',' established in',' reign of'))
    first=low[:180]
    encycl += 2*int(bool(re.search(r'\b(is|was|are) (an?|the)\b',first)))
    qa = html*1.8 + tech + 2*low.count('?') + 2*low.count('<code>')
    # Register assignment uses structural evidence, then lexical evidence.
    if qa >= 6 and (html >= 1 or '<code>' in low): dom=3; fit=min(qa,35)/35
    elif news >= 8: dom=2; fit=min(news,25)/25
    elif encycl >= 7 and news < 8: dom=0; fit=min(encycl,22)/22
    else: dom=1; fit=min(sent,30)/60
    # Favor 1--8k token documents: enough context, limited boilerplate domination.
    length_bonus = -.24*abs(math.log(max(n,1200)/4500))
    return dom, q + .55*fit + length_bonus

def main():
    tok=GPT2TokenizerFast.from_pretrained('gpt2',local_files_only=True)
    buckets=[[] for _ in range(4)]
    with POOL.open() as f:
        for line in f:
            r=json.loads(line); z=score_doc(r['text'])
            if z is not None: buckets[z[0]].append((z[1],r['id'],r['text']))
    for b in buckets: b.sort(reverse=True)
    chosen=[]; totals=[0]*4; pos=[0]*4
    # Token-balanced round robin. Quality within each register is monotone.
    active=True
    while active:
        active=False
        for d in range(4):
            if totals[d] >= TARGET_PER_DOMAIN or pos[d] >= len(buckets[d]): continue
            active=True
            while pos[d] < len(buckets[d]):
                _,i,t=buckets[d][pos[d]]; pos[d]+=1
                nt=len(tok(t,add_special_tokens=False).input_ids)+1
                chosen.append(i); totals[d]+=nt
                break
    # Append remaining valid IDs in global score order as an unused safety tail.
    used=set(chosen)
    tail=sorted((x for b in buckets for x in b if x[1] not in used),reverse=True)
    chosen.extend(i for _,i,_ in tail[:10000])
    OUT.parent.mkdir(parents=True,exist_ok=True)
    OUT.write_text(json.dumps(chosen))
    print('documents',len(chosen),'domain_tokens',totals,'prefix_tokens',sum(totals))

if __name__ == '__main__': main()