import json, subprocess, re, io, html, time, urllib.parse
from collections import Counter

UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36'
KWS = ["党校", "干部学院", "组织部", "党员教育", "党建", "智慧党建", "信息化", "培训",
       "能力提升", "融媒体", "展陈", "宣传片", "视频", "课件", "课程", "智慧校园",
       "干部教育", "行政学院", "思政", "直播", "数字化", "纪念馆", "布展", "党群"]

ENTRY = re.compile(
    r'href="(/searchDetail/[^"]+)"[^>]*>([^<]{4,120})</a>'
    r'.*?信息类型：([^<]{2,40})</span>\s*<span>发布时间：(\d{4}-\d{2}-\d{2})',
    re.S)


def fetch(kw):
    u = 'https://www.cqggzy.com/search?keyword=' + urllib.parse.quote(kw)
    r = subprocess.run(['curl', '-s', '-m', '35', u, '-H', 'User-Agent: ' + UA],
                       capture_output=True, text=True, timeout=50)
    return r.stdout


recs = {}
for kw in KWS:
    h = fetch(kw)
    items = ENTRY.findall(h)
    n = 0
    for path, title, itype, date in items:
        url = 'https://www.cqggzy.com' + html.unescape(path)
        if url in recs:
            continue
        recs[url] = {
            'title': html.unescape(title).strip(),
            'url': url,
            'source': '重庆市公共资源交易网',
            'type': html.unescape(itype).strip(),
            'pub_date': date,
            'province': '重庆',
            'buyer': '',
            'budget': None,
            'kw': kw,
        }
        n += 1
    print('cq %s -> %d items, +%d' % (kw, len(items), n), flush=True)
    time.sleep(0.2)

json.dump(list(recs.values()), io.open('/data/www/files/cq_raw_1030.json', 'w', encoding='utf-8'),
          ensure_ascii=False, indent=1)
print('CQ unique:', len(recs))
print(Counter(v['type'] for v in recs.values()))