#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Triage all scraped raw data into a candidate pool for the weekly report."""
import json, io, os, re, datetime, html
from collections import Counter

BASE = "/data/www/files"
START = datetime.date(2026, 8, 25)
END = datetime.date(2026, 9, 25)

CORE_KW = ["党校", "干部学院", "行政学院", "干部教育", "干部培训", "党员教育", "党建",
           "组织部", "党群", "融媒体", "展陈", "展览", "布展", "陈列", "宣传片", "专题片",
           "纪录片", "微短剧", "宣传视频", "视频拍摄", "视频制作", "课件", "微课", "课程",
           "智慧校园", "教育信息化", "思政", "红色", "纪念馆", "廉政", "廉洁", "培训",
           "能力提升", "信息化", "数字化", "智慧", "直播", "录制", "演播", "摄制",
           "文化馆", "文史", "宣传", "平台建设", "系统建设", "运维", "新媒体"]
# strong noise -> drop outright
NOISE = ["印刷", "复印纸", "打印纸", "食堂", "餐饮", "食材", "配餐", "家具", "空调",
         "物业", "保洁", "桌椅", "宿舍", "床上用品", "被褥", "后勤用品", "办公耗材",
         "医疗设备", "药品", "疫苗", "试剂", "车辆", "加油", "保险", "绿化", "苗木",
         "污水处理", "道路", "路灯", "管网", "取暖", "煤", "安保", "消防维保",
         "电梯", "垃圾", "化肥", "饲料", "种公畜", "藏羊", "牛", "羊", "地勘",
         "皮卡", "货车", "多功能一体机", "复印机", "计算机采购", "打印机", "扫描仪",
         "血液透析", "影像设备", "白大褂", "工作服", "健身器材", "体育器材", "玩具",
         "窗帘", "门头", "标牌", "广告", "体检", "健康体检", "审计服务", "法律服务"]


def parse_d(s):
    if not s:
        return None
    m = re.search(r"(20\d{2})[-/年.](\d{1,2})[-/月.](\d{1,2})", s)
    if not m:
        return None
    try:
        return datetime.date(int(m.group(1)), int(m.group(2)), int(m.group(3)))
    except Exception:
        return None


def clean(t):
    t = re.sub(r"<[^>]+>", "", t or "")
    return html.unescape(t).strip()


cands = []
seen = set()


def add(title, buyer, budget, pub, url, source, ptype, prov, content=""):
    title = clean(title)
    if not title or not url:
        return
    key = url.split("?")[0]
    if key in seen:
        return
    if not any(k in title for k in CORE_KW):
        return
    if any(n in title for n in NOISE):
        return
    d = parse_d(pub)
    if d and (d < START or d > END):
        return
    seen.add(key)
    cands.append({"title": title, "buyer": clean(buyer), "budget": budget or "未披露",
                  "pub_date": pub or "", "url": url, "source": source, "type": ptype,
                  "province": prov, "content": content[:6000]})


# ---- 1. Sichuan trading center API ----
raw = json.load(io.open(os.path.join(BASE, "ggzy_raw_0925.json"), encoding="utf-8"))
CATMAP = {"002002001": "招标公告", "002002003": "中标公告", "002002002": "更正公告"}
for u, v in (raw.get("records") or {}).items():
    cat = v.get("cat")
    if cat not in CATMAP:
        continue
    add(v.get("title"), "", "", v.get("date"), u, "四川省公共资源交易信息网",
        CATMAP[cat], "四川", v.get("content") or "")

# ---- 2. subagent: Sichuan govt ----
p = os.path.join(BASE, "sc_gov_20260925.json")
if os.path.exists(p):
    for r in json.load(io.open(p, encoding="utf-8")):
        add(r.get("title"), r.get("buyer"), r.get("budget"), r.get("pub_date"),
            r.get("url"), r.get("source") or "四川政府采购网",
            r.get("type") or "招标公告", r.get("province") or "四川", r.get("page_text") or "")

# ---- 3. subagent: Chongqing + Chengdu ----
p = os.path.join(BASE, "cq_cd_20260925.json")
if os.path.exists(p):
    for r in json.load(io.open(p, encoding="utf-8")):
        add(r.get("title"), r.get("buyer"), r.get("budget"), r.get("pub_date"),
            r.get("url"), r.get("source") or "", r.get("type") or "招标公告",
            r.get("province") or "重庆", r.get("page_text") or "")

# ---- 4. Gansu ----
p = os.path.join(BASE, "gansu_raw_0925.json")
if os.path.exists(p):
    for _, r in json.load(io.open(p, encoding="utf-8")).items():
        url = "https://www.ccgp-gansu.gov.cn/maincms-web/articleGs?type=article&id=" + r["id"]
        add(r.get("title"), r.get("purchaser"), r.get("budget"), r.get("noticeTime", "")[:10],
            url, "甘肃省政府采购网", "招标公告", "甘肃", r.get("desc") or "")

print("candidates:", len(cands))
print(Counter(c["province"] for c in cands))
print(Counter(c["source"] for c in cands))
json.dump(cands, io.open(os.path.join(BASE, "cand_pool_0925.json"), "w", encoding="utf-8"),
          ensure_ascii=False, indent=1)
for c in cands[:200]:
    print("%-6s | %-10s | %-40s | %s" % (c["province"], c["type"], c["title"][:40], c["pub_date"]))