import json, subprocess, time, urllib.parse
from collections import Counter

BASE = 'https://www.ccgp-gansu.gov.cn/gpcms/rest/web/v2/info/selectInfoForIndex'
SITE = '793f8fcd-77c4-4c76-9ce6-9de8deddee8d'
UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
SDT = '2026-10-06 00:00:00'
EDT = '2026-11-06 23:59:59'

NOISE = {'001054', '00105A'}


def q(kw, page=1, size=30):
    params = {
        'currPage': str(page), 'pageSize': str(size), 'siteId': SITE,
        'channel': '', 'noticeType': '', 'purchaser': '', 'agency': '',
        'operationStartTime': SDT, 'operationEndTime': EDT,
        'searchKey': kw, 'regionCode': '', 'selectTimeName': 'noticeTime',
        'cityOrAreal': '', '_t': str(int(time.time() * 1000)),
    }
    url = BASE + '?' + urllib.parse.urlencode(params)
    cmd = ['curl', '-s', '-m', '40', url,
           '-H', 'User-Agent: ' + UA,
           '-H', 'Referer: https://www.ccgp-gansu.gov.cn/maincms-web/fullSearchingGs']
    for _ in range(3):
        try:
            out = subprocess.run(cmd, capture_output=True, text=True, timeout=50).stdout
            j = json.loads(out)
            return (j.get('data') or {}).get('rows') or []
        except Exception:
            time.sleep(2)
    return []


KWS = ["党校", "干部学院", "组织部", "党员教育", "党建", "智慧党建", "信息化", "培训",
       "能力提升", "融媒体", "展陈", "宣传片", "视频", "课件", "课程", "智慧校园",
       "干部教育", "行政学院", "思政", "直播", "数字化", "纪念馆", "布展", "党群"]

recs = {}
for kw in KWS:
    rows = q(kw)
    n_new = 0
    for r in rows:
        nt = r.get('noticeType') or ''
        if nt in NOISE:
            continue
        title = (r.get('title') or '').replace("<font color='#be0108'>", '').replace('</font>', '')
        if '印刷' in title or '复印纸' in title:
            continue
        i = r.get('id')
        if not i or i in recs:
            continue
        recs[i] = {
            'id': i, 'title': title.strip(), 'purchaser': r.get('purchaser') or '',
            'budget': r.get('budget'), 'noticeTime': (r.get('noticeTime') or '')[:19],
            'noticeType': nt, 'region': r.get('regionName') or '',
            'desc': (r.get('description') or '')[:400],
            'kw': kw,
        }
        n_new += 1
    print("gansu %s -> %d rows, +%d" % (kw, len(rows), n_new), flush=True)
    time.sleep(0.4)

json.dump(recs, open('/data/www/files/gansu_raw_1106.json', 'w'), ensure_ascii=False, indent=1)
print("Gansu unique:", len(recs))
print(Counter(v['noticeType'] for v in recs.values()))