import json, subprocess, time
from collections import Counter

URL = 'https://ggzyjy.sc.gov.cn/inteligentsearch/rest/esinteligentsearch/getFullTextDataNew'
HDRS = [
    '-H', 'Content-Type: application/json;charset=UTF-8',
    '-H', 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/149.0.0.0 Safari/537.36',
    '-H', 'Referer: https://ggzyjy.sc.gov.cn/jyxx/002002/transactionInfo.html',
    '-H', 'Cookie: userGuid=498136447',
]
SDT = "2026-09-30 00:00:00"
EDT = "2026-10-30 23:59:59"


def q(wd, fields="title", pn=0, rn=50):
    body = {
        "pn": pn, "rn": rn,
        "sdt": SDT, "edt": EDT,
        "cl": 400, "noParticiple": "1",
        "sort": '{"webdate":"0"}',
        "wd": wd, "fields": fields,
    }
    cmd = ['curl', '-s', '-m', '40', URL] + HDRS + ['--data-raw', json.dumps(body, ensure_ascii=False)]
    for attempt in range(3):
        try:
            out = subprocess.run(cmd, capture_output=True, text=True, timeout=50).stdout
            return json.loads(out)
        except Exception:
            time.sleep(2)
    return {"result": {"totalcount": 0, "records": []}}


records = {}
meta = {}
KWS = ["党校", "干部学院", "组织部", "党员教育", "党建", "智慧党建", "信息化", "培训",
       "能力提升", "融媒体", "展陈", "宣传片", "视频", "课件", "课程", "智慧校园",
       "干部教育", "行政学院", "思政", "直播", "运维", "数字化", "纪念馆", "党群",
       "布展", "教育基地", "政务", "系统建设"]
for kw in KWS:
    j = q(kw, "title")
    r = j.get("result") or {}
    meta[kw] = r.get("totalcount")
    for x in (r.get("records") or []):
        u = x.get("linkurl") or ""
        if not u:
            continue
        if u.startswith("/"):
            u = "https://ggzyjy.sc.gov.cn" + u
        if u not in records:
            records[u] = {
                "title": (x.get("titlenew") or x.get("title") or "").strip(),
                "cat": x.get("categorynum"),
                "date": (x.get("infodate") or "")[:10],
                "url": u,
                "content": x.get("content") or "",
                "kws": [],
            }
        records[u]["kws"].append(kw)
    print("done", kw, meta[kw], flush=True)
    time.sleep(0.25)

json.dump({"meta": meta, "records": records}, open('/data/www/files/ggzy_raw_1030.json', 'w'),
          ensure_ascii=False, indent=1)
print("meta:", json.dumps(meta, ensure_ascii=False))
print("uniq:", len(records))
print("cats:", Counter(v["cat"] for v in records.values()))