import urllib.request,csv,re,sys
# Re-run of robots-study-2026-10-25.py on the same site list (robots-study-2026-10-25-sites.txt).
# Change vs 10-25: if the first fetch fails (HTTP error or timeout), retry once with a different common browser UA.
DATE='2026-10-26'
BOTS=['GPTBot','ClaudeBot','PerplexityBot']
UA1='Mozilla/5.0 (compatible; PaperboyResearch/1.0)'
UA2='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36'
def parse(txt):
    groups=[];cur=None;last_ua=False
    for line in txt.splitlines():
        line=line.split('#')[0].strip()
        if ':' not in line: continue
        k,v=[x.strip() for x in line.split(':',1)]
        k=k.lower()
        if k=='user-agent':
            if not last_ua: cur={'uas':[],'rules':[]};groups.append(cur)
            cur['uas'].append(v.lower());last_ua=True
        else:
            last_ua=False
            if cur is not None and k in('disallow','allow'): cur['rules'].append((k,v))
    return groups
def status(groups,bot):
    b=bot.lower();g=[x for x in groups if b in x['uas']]
    if not g: return 'no_specific_rule'
    rules=[r for x in g for r in x['rules']]
    if ('disallow','/') in rules and ('allow','/') not in rules: return 'blocked'
    if any(k=='disallow' and v for k,v in rules): return 'partial'
    return 'allowed'
def fetch(s,ua):
    req=urllib.request.Request(f'https://{s}/robots.txt',headers={'User-Agent':ua})
    r=urllib.request.urlopen(req,timeout=20);return r.status,r.read(2000000).decode('utf8','replace')
def measure(s):
    try:
        code,body=fetch(s,UA1);note=''
    except Exception as e1:
        try:
            code,body=fetch(s,UA2);note='retry_ok_with_chrome_ua'
        except Exception as e2:
            return [s,getattr(e2,'code','err'),0,'n/a','n/a','n/a','','failed twice (2 UAs): '+str(e2)[:50]]
    if re.match(r'\s*<(!doctype|html)',body,re.I): return [s,code,len(body),'n/a','n/a','n/a','','HTML not robots.txt']
    g=parse(body)
    star='blocked_all' if any('*' in x['uas'] and ('disallow','/') in x['rules'] for x in g) else 'open_or_partial'
    return [s,code,len(body)]+[status(g,b) for b in BOTS]+[star,note]
sites=open(sys.argv[1] if len(sys.argv)>1 else 'robots-study-2026-10-25-sites.txt').read().split()
rows=[measure(s) for s in sites]
with open('robots-study-2026-10-26.csv','w',newline='') as f:
    w=csv.writer(f);w.writerow(['date','site','http_status','robots_bytes']+BOTS+['wildcard_disallow_root','note'])
    for r in rows: w.writerow([DATE]+r)