import json, subprocess, time, sys, urllib.parse

URL = 'https://ggzyjy.sc.gov.cn/inteligentsearch/rest/esinteligentsearch/getFullTextDataNew'
HDRS = [
    '-H', 'Content-Type: application/json;charset=UTF-8',
    '-H', 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/149.0.0.0 Safari/537.36',
    '-H', 'Referer: https://ggzyjy.sc.gov.cn/jyxx/002002/transactionInfo.html',
    '-H', 'Cookie: userGuid=498136447',
]

def q(wd, fields="title", pn=0, rn=50):
    body = {
        "pn": pn, "rn": rn,
        "sdt": "2026-08-17 00:00:00", "edt": "2026-09-17 23:59:59",
        "cl": 400, "noParticiple": "1",
        "sort": '{"webdate":"0"}',
        "wd": wd, "fields": fields,
    }
    cmd = ['curl', '-s', '-m', '40', URL] + HDRS + ['--data-raw', json.dumps(body, ensure_ascii=False)]
    for attempt in range(3):
        try:
            out = subprocess.run(cmd, capture_output=True, text=True, timeout=50).stdout
            return json.loads(out)
        except Exception as e:
            time.sleep(2)
    return {"result": {"totalcount": 0, "records": []}}

records = {}
meta = {}
for kw in ["党校","组织部","党员教育","智慧","信息化","培训","融媒体","党建","展陈","干部培训","智慧校园","宣传","视频"]:
    for f in ["title", "title;content"]:
        j = q(kw, f)
        r = j.get("result") or {}
        meta[f"{kw}|{f}"] = r.get("totalcount")
        for x in (r.get("records") or []):
            u = x.get("linkurl") or ""
            if not u:
                continue
            if u not in records:
                records[u] = {
                    "title": x.get("titlenew") or x.get("title"),
                    "cat": x.get("categorynum"),
                    "date": (x.get("infodate") or "")[:10],
                    "url": u,
                    "content": x.get("content") or "",
                    "kws": [],
                }
            records[u]["kws"].append(kw)
        time.sleep(0.4)

json.dump({"meta": meta, "records": records}, open('/data/www/files/ggzy_raw.json', 'w'), ensure_ascii=False, indent=1)
print("meta:", json.dumps(meta, ensure_ascii=False))
print("uniq:", len(records))
from collections import Counter
print("cats:", Counter(v["cat"] for v in records.values()))