#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Curl triage of candidate URLs: HTTP status + rough body length + title sniff."""
import json, subprocess, sys, re

UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
      "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")

def fetch(u):
    try:
        r = subprocess.run(
            ["curl", "-sSL", "--compressed", "--max-time", "25",
             "-A", UA, "-H", "Accept-Language: zh-CN,zh;q=0.9", u],
            capture_output=True, timeout=35)
        raw = r.stdout
        try:
            txt = raw.decode("utf-8")
        except UnicodeDecodeError:
            txt = raw.decode("gbk", "ignore")
        code = subprocess.run(
            ["curl", "-sS", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "25",
             "-A", UA, u], capture_output=True, text=True, timeout=35).stdout.strip()
        return code, len(raw), txt
    except Exception as e:
        return "ERR", 0, str(e)

pool = json.load(open("/data/www/files/cand_pool.json", encoding="utf-8"))
out = []
for i, p in enumerate(pool, 1):
    code, n, txt = fetch(p["url"])
    # sniff visible title markers
    m = re.search(r"<title>(.*?)</title>", txt, re.S | re.I)
    ttl = (m.group(1).strip()[:60] if m else "")
    has_title = p["title"][:8] in txt
    p["http"] = code
    p["bytes"] = n
    p["page_title"] = ttl
    p["title_in_page"] = has_title
    out.append(p)
    print(f"{i:02d} {code} {n:>8}B inpage={has_title} | {p['title'][:26]} | {ttl[:40]}", flush=True)

json.dump(out, open("/data/www/files/cand_triage.json", "w", encoding="utf-8"),
          ensure_ascii=False, indent=1)
ok = [p for p in out if p["http"] == "200"]
print(f"\nTOTAL={len(out)} HTTP200={len(ok)}")