import json, subprocess, time

URL = 'https://ggzyjy.sc.gov.cn/inteligentsearch/rest/esinteligentsearch/getFullTextDataNew'
HDRS = [
    '-H', 'Content-Type: application/json;charset=UTF-8',
    '-H', 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/149.0.0.0 Safari/537.36',
    '-H', 'Referer: https://ggzyjy.sc.gov.cn/jyxx/002002/transactionInfo.html',
    '-H', 'Cookie: userGuid=498136447',
]
SDT = "2026-08-25 00:00:00"
EDT = "2026-09-25 23:59:59"

def q(wd, fields="title", pn=0, rn=50):
    body = {
        "pn": pn, "rn": rn,
        "sdt": SDT, "edt": EDT,
        "cl": 400, "noParticiple": "1",
        "sort": '{"webdate":"0"}',
        "wd": wd, "fields": fields,
    }
    cmd = ['curl', '-s', '-m', '40', URL] + HDRS + ['--data-raw', json.dumps(body, ensure_ascii=False)]
    for attempt in range(3):
        try:
            out = subprocess.run(cmd, capture_output=True, text=True, timeout=50).stdout
            return json.loads(out)
        except Exception:
            time.sleep(2)
    return {"result": {"totalcount": 0, "records": []}}

records = {}
meta = {}
KWS = ["党校", "干部学院", "组织部", "党员教育", "党建", "智慧党建", "信息化", "培训",
       "能力提升", "融媒体", "展陈", "宣传片", "视频", "课件", "课程", "智慧校园",
       "干部教育", "行政学院", "思政", "直播", "运维", "数字化"]
for kw in KWS:
    for f in ["title"]:
        j = q(kw, f)
        r = j.get("result") or {}
        meta["%s|%s" % (kw, f)] = r.get("totalcount")
        for x in (r.get("records") or []):
            u = x.get("linkurl") or ""
            if not u:
                continue
            if u not in records:
                records[u] = {
                    "title": (x.get("titlenew") or x.get("title") or "").strip(),
                    "cat": x.get("categorynum"),
                    "date": (x.get("infodate") or "")[:10],
                    "url": u,
                    "content": x.get("content") or "",
                    "kws": [],
                }
            records[u]["kws"].append(kw)
        time.sleep(0.3)
    print("done", kw, flush=True)

json.dump({"meta": meta, "records": records}, open('/data/www/files/ggzy_raw_0925.json', 'w'),
          ensure_ascii=False, indent=1)
print("meta:", json.dumps(meta, ensure_ascii=False))
print("uniq:", len(records))
from collections import Counter
print("cats:", Counter(v["cat"] for v in records.values()))