#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Harvest ggzy.gov.cn 交易公开 (national platform) for other provinces."""
import json, re, time, sys, urllib.parse, urllib.request, concurrent.futures as cf

API = "https://www.ggzy.gov.cn/information/pubTradingInfo/getTradList"
BASE = "https://www.ggzy.gov.cn"
HDRS = {
    "Content-Type": "application/x-www-form-urlencoded",
    "Referer": "https://www.ggzy.gov.cn/deal/dealList.html",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120 Safari/537.36",
    "X-Pass-Token": "",
}
KWS = ["党校", "党建", "党员教育", "干部培训", "融媒体", "展陈", "宣传片", "信息化", "培训"]
PROVS = {
    "530000": "云南", "520000": "贵州", "450000": "广西", "430000": "湖南",
    "420000": "湖北", "610000": "陕西", "360000": "江西", "140000": "山西",
    "130000": "河北", "340000": "安徽", "350000": "福建", "370000": "山东",
    "410000": "河南", "210000": "辽宁", "230000": "黑龙江", "220000": "吉林",
    "150000": "内蒙古", "650000": "新疆", "320000": "江苏", "330000": "浙江",
}
DATE_B, DATE_E = "2026-09-30", "2026-10-30"

RELEVANT = re.compile(r"党校|党建|党员|干部培训|干部教育|组工|融媒体|融媒|展陈|布展|宣传片|专题片|"
                      r"微党课|党性|红色教育|主题教育|培训|课程|课件|宣传|文化|档案|政务|"
                      r"信息化|智慧|数字化|媒体|讲解|教育")


def post(kw, prov, page):
    data = urllib.parse.urlencode({
        "DEAL_TIME": "06", "TIMEBEGIN": DATE_B, "TIMEEND": DATE_E,
        "FINDTXT": kw, "DEAL_PROVINCE": prov, "PAGENUMBER": str(page)}).encode()
    req = urllib.request.Request(API, data=data, headers=HDRS, method="POST")
    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=30) as r:
                return json.loads(r.read().decode("utf-8"))
        except Exception as e:
            if attempt == 2:
                print("ERR", kw, prov, page, e, file=sys.stderr)
                return None
            time.sleep(2)


lock = __import__("threading").Lock()
rows = {}


def job(args):
    kw, prov = args
    page, pages = 1, 1
    got = 0
    while page <= pages and page <= 3:
        j = post(kw, prov, page)
        if not j or j.get("code") != 200:
            break
        d = j["data"]
        pages = d.get("pages") or 1
        if pages == 50:
            pages = 100
        for rec in d.get("records") or []:
            u = rec.get("url") or ""
            t = (rec.get("title") or "").strip()
            if not u or not RELEVANT.search(t):
                continue
            with lock:
                e = rows.setdefault(u, {
                    "id": rec.get("id"), "title": t,
                    "url": BASE + u, "pub_date": rec.get("publishTime"),
                    "type": rec.get("businessTypeText") or rec.get("informationTypeText"),
                    "info_type": rec.get("informationTypeText"),
                    "province": rec.get("provinceText"),
                    "province_code": rec.get("province"),
                    "platform": rec.get("transactionSourcesPlatformText") or "",
                    "kws": []})
                if kw not in e["kws"]:
                    e["kws"].append(kw)
                got += 1
        page += 1
        time.sleep(0.15)
    return kw, prov, got


tasks = [(kw, prov) for kw in KWS for prov in PROVS]
done = 0
with cf.ThreadPoolExecutor(max_workers=10) as ex:
    for kw, prov, got in ex.map(job, tasks):
        done += 1
        if done % 20 == 0:
            print("progress %d/%d uniq=%d" % (done, len(tasks), len(rows)), file=sys.stderr, flush=True)

print("harvested", len(rows), file=sys.stderr)
json.dump(list(rows.values()), open("/data/www/files/ggzy_nat_raw_1030.json", "w"),
          ensure_ascii=False, indent=1)
from collections import Counter
print(Counter(v["province"] for v in rows.values()))