gpt-5.6-sol

Script completed
Wall time 11.6 seconds
Output:
{"chunk_id":"edcfac","wall_time_seconds":30.001098178,"session_id":72867,"original_token_count":0,"output":""}

Script running with cell ID 142
Wall time 11.0 seconds
Output:

Script completed
Wall time 7.2 seconds
Output:
{"chunk_id":"23975c","wall_time_seconds":24.776904229,"exit_code":0,"original_token_count":16,"output":"HELDOUT_PPL 354.826  loss 5.8716  train_s 56  /tmp/cand9.npy\n"}

Script completed
Wall time 0.0 seconds
Output:
{}

Script completed
Wall time 0.1 seconds
Output:
#!/usr/bin/env python3
"""Rank raw-web documents for a balanced, high-quality 12M-token mixture.

The criterion is fully text-derived: English/prose quality, scraper-noise penalties,
and four bounded register-affinity scores (encyclopedic, general prose, news,
technical Q&A). We rank globally by quality; affinity only breaks close ties so
that weak genre inference cannot force poor pages into the training set.
No pool ids or external labels are embedded in this program.
"""
import json, math, re, hashlib, collections
from pathlib import Path
import numpy as np
from transformers import AutoTokenizer

POOL = Path("/workspace/data/pool.jsonl")
OUT = Path("/workspace/submission/selection.json")

WORD = re.compile(r"[A-Za-z]+(?:'[A-Za-z]+)?")
SENT = re.compile(r"[.!?](?:[\"')\]]*)\s+[A-Z]")
COMMON = set("the of and to in a is that for it on as with was are by this be from or an at which have has but not their they its he she we you were can will would about into than more when who what how also one all other been had".split())
TECH = set("python java javascript csharp php ruby linux windows api database sql code function class method variable array string file server error compiler algorithm command shell html css git library framework object pointer query application software program programming".split())
ENC = set("species genus known located population history century region district river family born died established refers consists including according named became served university".split())
NEWS = set("said says told reported announced officials government president minister police company percent according monday tuesday wednesday thursday friday saturday sunday yesterday spokesman spokeswoman reporters".split())

def features(text, target_counts, target_total):
    low = text.lower(); ws = WORD.findall(low); n = len(ws)
    if n < 90: return None
    counts = {}
    for w in ws: counts[w] = counts.get(w, 0) + 1
    chars = max(1, len(text)); lines = text.splitlines()
    alpha = sum(c.isalpha() for c in text) / chars
    common = sum(counts.get(w, 0) for w in COMMON) / n
    uniq = len(counts) / n
    sentences = len(SENT.findall(text)) + text.count('.\n')
    avg_sent = n / max(1, sentences)
    short_lines = sum(len(x.strip()) < 35 for x in lines) / max(1, len(lines))
    repeats = max(counts.values()) / n
    target_ll = sum(math.log((target_counts.get(w, 0) + .2) / target_total) for w in ws) / n

    # Smooth quality score modeled after published web-corpus filtering rules.
    q = 0.0
    q += 5.0 * min(common, .18)
    q += 1.2 * min(alpha, .78)
    q += 0.8 * min(uniq, .55)
    q += 0.35 * math.log1p(min(n, 3500))
    q -= 2.0 * max(0, short_lines - .35)
    q -= 8.0 * max(0, repeats - .035)
    q -= 1.2 * (avg_sent < 8 or avg_sent > 55)
    q -= 1.0 * (text.count('|') / chars > .004)
    q -= 1.2 * (text.count('{') + text.count('}')) / max(30, n)
    bad = sum(low.count(x) for x in ('cookie policy','all rights reserved','toggle navigation','sign in','log in','privacy policy','javascript is disabled','404 not found','click here','related posts','view cart','subscribe to our newsletter'))
    q -= min(3.0, bad * .35)
    # Boilerplate/navigation produces many tiny lines and few real sentences.
    q -= 1.4 * (len(lines) > 25 and short_lines > .62)
    # The disclosed target supplies a broad English reference vocabulary.  Mean
    # unigram likelihood is deliberately capped: it rejects word-salad and
    # machine-spun SEO text without selecting documents merely by topic overlap.
    q += max(-4.0, min(2.0, 1.8 * (target_ll + 8.25)))
    q -= min(2.0, max(0, low.count('<|endoftext|>') - 1) * .12)
    q -= 2.5 * any(x in low for x in ('cialis online','viagra price','payday loans online','custom essay writing','xxx clips','levitra online'))

    dens = lambda vocab: sum(counts.get(w, 0) for w in vocab) / math.sqrt(n)
    tech = dens(TECH) + 0.30*low.count('<code') + 0.18*low.count('```') + 0.08*low.count('stackoverflow')
    tech += .08 * sum(low.count(x) for x in ('()','==','->','import ','select ','exception','error:'))
    news = dens(NEWS) + .20*('(reuters)' in low) + .12*('associated press' in low) + .08*(' /prnewswire/' in low)
    enc = dens(ENC) + .16*sum(low.count(x) for x in (' is a ',' was a ',' refers to ',' may refer to ',' is an '))
    # Penalize first-person/chatty language for the reference-like register.
    enc -= .30 * sum(counts.get(w,0) for w in ('i','me','my','we','our')) / math.sqrt(n)
    return q, tech, news, enc, n, chars, counts

def main():
    tok = AutoTokenizer.from_pretrained('gpt2')
    target_ids=np.load('/workspace/data/multi_dev.npy')
    target_text = tok.decode(target_ids)
    target_counts = collections.Counter(WORD.findall(target_text.lower()))
    target_total = sum(target_counts.values())
    domain_names=('encyclopedic','general','news','technical')
    domain_counts=[]
    for j in range(4):
        segment=tok.decode(target_ids[j*250_000:(j+1)*250_000]).lower()
        domain_counts.append(collections.Counter(WORD.findall(segment)))
    buckets = {k: [] for k in ('encyclopedic','general','news','technical')}
    ranked=[]
    seen_hashes=set()
    for line in POOL.open():
        r=json.loads(line)
        norm=' '.join(r['text'].lower().split())
        h=hashlib.blake2b(norm.encode(),digest_size=12).digest()
        if h in seen_hashes: continue
        seen_hashes.add(h)
        f=features(r['text'], target_counts, target_total)
        if not f: continue
        q,tech,news,enc,n,chars,word_counts=f
        # Require stronger evidence for the specialized, relatively rare Q&A class.
        # Equal-prior multinomial Bayes routing from the four disclosed target
        # quarters. P(domain|word) ratios make this about register, not length.
        ds=[0.0,0.0,0.0,0.0]
        for w,nw in word_counts.items():
            den=target_counts.get(w,0)+2.0
            for j in range(4): ds[j] += nw*math.log((domain_counts[j].get(w,0)+.5)/den)
        ds=[x/max(1,n) for x in ds]
        best=max(range(4),key=lambda j:ds[j]); domain=domain_names[best]
        margin=ds[best]-sorted(ds)[-2]
        affinity=min(.35,margin*.8)
        # Very long pages often concatenate boilerplate or unrelated articles.
        length_pen = max(0, math.log(max(1, chars/24000))) * .30
        final_score=q+affinity-length_pen
        buckets[domain].append((final_score, r['id'], chars))
        ranked.append((final_score, r['id'], chars, domain))
    for b in buckets.values(): b.sort(reverse=True)

    # Take equal character surpluses, then weave one document from each register.
    # Similar top-document lengths make every prefix close to balanced, and the
    # 72M-character surplus safely exceeds the 12M-token packing budget.
    weights={'encyclopedic':.60,'general':.10,'news':.25,'technical':.05}
    quotas={'encyclopedic':35_000_000,'general':8_000_000,'news':18_000_000,'technical':4_000_000}
    by={}; mix=collections.Counter()
    for domain,b in buckets.items():
        by[domain]=[]; total=0
        for score,i,chars in b:
            by[domain].append((i,chars)); total += chars; mix[domain] += chars
            if total >= quotas[domain]: break
    order=[]; pos={k:0 for k in by}; emitted={k:0 for k in by}
    while True:
        live=[k for k in ('encyclopedic','general','news','technical') if pos[k] < len(by[k])]
        if not live: break
        k=min(live,key=lambda x:emitted[x]/weights[x])
        i,chars=by[k][pos[k]]; order.append(i); pos[k]+=1; emitted[k]+=chars
    total=sum(mix.values())
    OUT.parent.mkdir(parents=True,exist_ok=True)
    OUT.write_text(json.dumps(order))
    print('ids',len(order),'chars',total,'estimated mix',dict(mix))

if __name__ == '__main__': main()

Script completed
Wall time 0.0 seconds
Output:
{}

Script running with cell ID 146
Wall time 11.0 seconds
Output:

Script completed
Wall time 5.8 seconds
Output:
{"chunk_id":"d63278","wall_time_seconds":30.001591293,"session_id":7303,"original_token_count":0,"output":""}

Script running with cell ID 147
Wall time 11.0 seconds
Output:

Script completed
Wall time 14.5 seconds
Output:
{"chunk_id":"96dbca","wall_time_seconds":30.00262065,"session_id":7303,"original_token_count":0,"output":""}

Script running with cell ID 148
Wall time 11.0 seconds
Output:

Script completed
Wall time 14.3 seconds
Output:
{"chunk_id":"4afeeb","wall_time_seconds":30.001513535,"session_id":7303,"original_token_count":0,"output":""}

Script running with cell ID 149
Wall time 11.0 seconds
Output:

Script completed
Wall time 15.0 seconds
Output:
{"chunk_id":"acbf6a","wall_time_seconds":30.002095003,"session_id":7303,"original_token_count":0,"output":""}

Script running with cell ID 150
Wall time 11.0 seconds
Output:

Script completed
Wall time 12.6 seconds
Output:
{"chunk_id":"6470cf","wall_time_seconds":30.001410476,"session_id":7303,"original_token_count":0,"output":""}

Script running with cell ID 151
Wall time 11.0 seconds
Output:

Script completed
Wall time 5.1 seconds
Output:
{"chunk_id":"e87f2d","wall_time_seconds":30.001581199,"session_id":7303,"original_token_count":0,"output":""}

Script completed
Wall time 2.1 seconds
Output:
{"chunk_id":"abec73","wall_time_seconds":2.136820031,"exit_code":0,"original_token_count":32,"output":"ids 17616 chars 65073380 estimated mix {'encyclopedic': 35026939, 'general': 8006434, 'news': 18000317, 'technical': 4039690}\r\n"}

Script completed
Wall time 0.0 seconds
Output:
{}

Script running with cell ID 154
Wall time 11.0 seconds
Output:

Script completed
Wall time 15.6 seconds
Output:
{"chunk_id":"5ceb00","wall_time_seconds":30.001042205,"session_id":22371,"original_token_count":60,"output":"ids 17616 unique 17616 min 5 max 182015 all_int True\nToken indices sequence length is longer than the specified maximum sequence length for this model (3134 > 1024). Running this sequence through the model will result in indexing errors\n"}

Script completed
Wall time 3.2 seconds
Output:
{"chunk_id":"dda1a4","wall_time_seconds":3.150155785,"exit_code":0,"original_token_count":68,"output":"packed 12000000 tokens from 17616 ids -> /tmp/final.npy\nTESTED_CANDIDATE_EXACT_MATCH\n-rw-r--r-- 1 root root 3.0K Aug  4 02:18 claim.md\n-rw-r--r-- 1 root root 7.6K Aug  4 02:14 submission/curate.py\n-rw-r--r-- 1 root root 123K Aug  4 02:18 submission/selection.json\n1224\n"}