#!/usr/bin/env python3
"""一键批量抓取四川各市州2026年组织部/党校预算PDF
使用Selenium模拟浏览器突破WAF，导航到预算公开页面提取PDF链接"""
import json, time, re, os, subprocess
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

PDF_DIR = "/data/www/files/budget_pdfs"
RESULTS_FILE = "/tmp/cities_2026_budget_final.json"

# 已有的市州数据不需要重复抓
existing = set()
if os.path.exists(RESULTS_FILE):
    with open(RESULTS_FILE) as f:
        old = json.load(f)
        for k,v in old.items():
            if v.get("org",{}).get("pdfs") or v.get("dx",{}).get("pdfs"):
                existing.add(k)

# 目标市州 - 根据域名可达性排序
cities = [
    # 第一批：已确认可达（首页返回200）
    {"slug": "leshan", "name": "乐山市", "domain": "leshan.gov.cn", "prefix": "lsswszf"},
    {"slug": "dazhou", "name": "达州市", "domain": "dazhou.gov.cn", "prefix": "dazhou"},
    {"slug": "yaan", "name": "雅安市", "domain": "yaan.gov.cn", "prefix": "yaan"},
    {"slug": "ziyang", "name": "资阳市", "domain": "ziyang.gov.cn", "prefix": "ziyang"},
    {"slug": "ganzi", "name": "甘孜州", "domain": "ganzi.gov.cn", "prefix": "ganzi"},
    {"slug": "yibin", "name": "宜宾市", "domain": "yibin.gov.cn", "prefix": "yibin"},
    {"slug": "mianyang", "name": "绵阳市", "domain": "mianyang.gov.cn", "prefix": "mianyang"},
    {"slug": "deyang", "name": "德阳市", "domain": "deyang.gov.cn", "prefix": "deyang"},
    {"slug": "nanchong", "name": "南充市", "domain": "nanchong.gov.cn", "prefix": "nanchong"},
    {"slug": "luzhou", "name": "泸州市", "domain": "luzhou.gov.cn", "prefix": "luzhou"},
    {"slug": "suining", "name": "遂宁市", "domain": "suining.gov.cn", "prefix": "suining"},
    {"slug": "bazhong", "name": "巴中市", "domain": "bazhong.gov.cn", "prefix": "bazhong"},
    # 第二批：可能需要WAF突破
    {"slug": "guangan", "name": "广安市", "domain": "guangan.gov.cn", "prefix": "guangan"},
    {"slug": "meishan", "name": "眉山市", "domain": "meishan.gov.cn", "prefix": "meishan"},
    {"slug": "zigong", "name": "自贡市", "domain": "zigong.gov.cn", "prefix": "zigong"},
    {"slug": "neijiang", "name": "内江市", "domain": "neijiang.gov.cn", "prefix": "neijiang"},
    {"slug": "liangshan", "name": "凉山州", "domain": "liangshan.gov.cn", "prefix": "liangshan"},
    {"slug": "guangyuan", "name": "广元市", "domain": "guangyuan.gov.cn", "prefix": "guangyuan"},
    {"slug": "panzhihua", "name": "攀枝花市", "domain": "panzhihua.gov.cn", "prefix": "panzhihua"},
]

def setup_driver():
    opts = Options()
    opts.add_argument("--headless=new")
    opts.add_argument("--no-sandbox")
    opts.add_argument("--disable-dev-shm-usage")
    opts.add_argument("--disable-blink-features=AutomationControlled")
    opts.add_argument("--window-size=1920,1080")
    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
    opts.add_experimental_option('useAutomationExtension', False)
    d = webdriver.Chrome(options=opts)
    d.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
        "source": "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"
    })
    return d

def download_pdf(driver, url, filename):
    """通过浏览器Fetch API下载PDF"""
    filepath = os.path.join(PDF_DIR, filename)
    if os.path.exists(filepath) and os.path.getsize(filepath) > 2000:
        print(f"    📄 已存在: {filename}")
        return filepath
    
    try:
        # 构造绝对URL
        if url.startswith('/'):
            abs_url = f"https://www.{driver.current_url.split('/')[2]}{url}"
        else:
            abs_url = url
            
        js = f"""
        var callback = arguments[0];
        fetch('{abs_url}')
            .then(r => r.blob())
            .then(blob => {{
                var reader = new FileReader();
                reader.onload = () => callback({{b64: reader.result.split(',')[1], len: reader.result.length}});
                reader.onerror = () => callback({{error: 'read fail'}});
                reader.readAsDataURL(blob);
            }})
            .catch(e => callback({{error: e.message}}));
        """
        result = driver.execute_async_script(js, timeout=60000)
        
        if result and 'b64' in result:
            import base64
            data = base64.b64decode(result['b64'])
            with open(filepath, 'wb') as f:
                f.write(data)
            size = len(data)
            if data[:4] == b'%PDF' or data[:4] == b'PK\x03\x04':
                print(f"    ✓ 下载成功: {filename} ({size} bytes)")
                return filepath
            else:
                print(f"    ✗ 无效文件: {filename} ({size} bytes, header={data[:10]})")
                os.remove(filepath)
        elif result and 'error' in result:
            print(f"    ✗ Fetch失败: {result['error']}")
    except Exception as e:
        print(f"    ✗ 下载异常: {e}")
    return None

def scrape_city(driver, city):
    """抓取单个市州的组织部和党校预算"""
    slug = city["slug"]
    name = city["name"]
    domain = city["domain"]
    prefix = city["prefix"]
    
    result = {"name": name, "org": {"links": [], "pdfs": []}, "dx": {"links": [], "pdfs": []}}
    
    print(f"\n{'='*50}")
    print(f"🔍 {name} ({domain})")
    
    # 导航到预算公开页面
    budget_urls = [
        f"https://www.{domain}/{prefix}/ysjs/",
        f"https://www.{domain}/{prefix}/ysjs/index.html",
        f"https://www.{domain}/ysjs/",
    ]
    
    budget_page_found = False
    for burl in budget_urls:
        try:
            driver.get(burl)
            time.sleep(3)
            html = driver.page_source
            if len(html) > 500 and ("预算" in html or "2026" in html):
                print(f"  ✓ 找到预算页面: {burl}")
                budget_page_found = True
                break
        except:
            continue
    
    if not budget_page_found:
        # 尝试政府信息公开页面
        try:
            driver.get(f"https://www.{domain}/{prefix}/jgjj/")
            time.sleep(2)
            # 点击"预算/决算"链接
            links = driver.find_elements(By.PARTIAL_LINK_TEXT, "预算")
            if links:
                links[0].click()
                time.sleep(2)
                budget_page_found = True
                print(f"  ✓ 导航到预算页面")
        except:
            pass
    
    if not budget_page_found:
        # 最后尝试：首页搜索
        try:
            driver.get(f"https://www.{domain}/")
            time.sleep(3)
            # 在页面中查找预算公开入口
            all_links = driver.find_elements(By.PARTIAL_LINK_TEXT, "预算")
            for link in all_links[:3]:
                try:
                    href = link.get_attribute("href") or ""
                    if "ysjs" in href or "预算" in link.text:
                        driver.get(href)
                        time.sleep(2)
                        budget_page_found = True
                        print(f"  ✓ 从首页导航到: {href}")
                        break
                except:
                    continue
        except:
            pass
    
    if not budget_page_found:
        print(f"  ✗ 未找到预算公开页面")
        return result
    
    # 在预算公开页面查找组织部和党校
    try:
        # 点击组织部
        org_spans = driver.find_elements(By.XPATH, "//span[contains(text(),'组织部')]")
        for span in org_spans:
            try:
                span.click()
                time.sleep(1)
                # 找到展开的内容
                org_links = driver.find_elements(By.PARTIAL_LINK_TEXT, "2026")
                for link in org_links:
                    try:
                        title = link.text.strip()
                        href = link.get_attribute("href") or ""
                        if title and href and "预算" in title:
                            result["org"]["links"].append({"title": title, "url": href})
                            print(f"    → 组织部: {title}")
                            # 导航到详情页提取PDF
                            driver.get(href)
                            time.sleep(2)
                            pdf_links = driver.find_elements(By.XPATH, "//a[contains(@href,'.pdf') or contains(@href,'.xls') or contains(@href,'.xlsx')]")
                            for pl in pdf_links:
                                try:
                                    purl = pl.get_attribute("href") or ""
                                    ptitle = pl.text.strip() or purl.split("/")[-1]
                                    if purl:
                                        fname = f"{slug}_org_{purl.split('/')[-1].split('?')[0]}"
                                        pdf_path = download_pdf(driver, purl, fname)
                                        if pdf_path:
                                            result["org"]["pdfs"].append({
                                                "title": ptitle,
                                                "url": purl,
                                                "local": f"/budget_pdfs/{fname}"
                                            })
                                except:
                                    continue
                            # 返回预算列表页
                            driver.back()
                            time.sleep(2)
                    except:
                        continue
            except:
                continue
        
        # 点击党校
        dx_spans = driver.find_elements(By.XPATH, "//span[contains(text(),'党校')]")
        for span in dx_spans:
            try:
                span.click()
                time.sleep(1)
                dx_links = driver.find_elements(By.PARTIAL_LINK_TEXT, "2026")
                for link in dx_links:
                    try:
                        title = link.text.strip()
                        href = link.get_attribute("href") or ""
                        if title and href and "预算" in title:
                            result["dx"]["links"].append({"title": title, "url": href})
                            print(f"    → 党校: {title}")
                            driver.get(href)
                            time.sleep(2)
                            pdf_links = driver.find_elements(By.XPATH, "//a[contains(@href,'.pdf') or contains(@href,'.xls') or contains(@href,'.xlsx')]")
                            for pl in pdf_links:
                                try:
                                    purl = pl.get_attribute("href") or ""
                                    ptitle = pl.text.strip() or purl.split("/")[-1]
                                    if purl:
                                        fname = f"{slug}_dx_{purl.split('/')[-1].split('?')[0]}"
                                        pdf_path = download_pdf(driver, purl, fname)
                                        if pdf_path:
                                            result["dx"]["pdfs"].append({
                                                "title": ptitle,
                                                "url": purl,
                                                "local": f"/budget_pdfs/{fname}"
                                            })
                                except:
                                    continue
                            driver.back()
                            time.sleep(2)
                    except:
                        continue
            except:
                continue
    except Exception as e:
        print(f"  提取出错: {e}")
    
    return result

# 主程序
print("="*60)
print("批量抓取四川各市州2026年预算数据")
print("="*60)

driver = setup_driver()
all_results = {}

# 加载已有结果
if os.path.exists(RESULTS_FILE):
    with open(RESULTS_FILE) as f:
        all_results = json.load(f)

for city in cities:
    slug = city["slug"]
    if slug in existing:
        print(f"\n⊘ {city['name']} 已有数据，跳过")
        continue
    
    result = scrape_city(driver, city)
    all_results[slug] = result
    
    # 每抓完一个就保存
    with open(RESULTS_FILE, "w") as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)

driver.quit()

# 汇总
print(f"\n{'='*60}")
print("抓取汇总:")
for slug, data in all_results.items():
    name = data.get("name", slug)
    org_pdfs = len(data.get("org",{}).get("pdfs",[]))
    dx_pdfs = len(data.get("dx",{}).get("pdfs",[]))
    org_links = len(data.get("org",{}).get("links",[]))
    dx_links = len(data.get("dx",{}).get("links",[]))
    status = "✓" if org_pdfs or dx_pdfs else "✗"
    print(f"  {status} {name}: 组织部{org_links}条({org_pdfs}PDF) 党校{dx_links}条({dx_pdfs}PDF)")

total_pdf = sum(len(v.get("org",{}).get("pdfs",[]))+len(v.get("dx",{}).get("pdfs",[])) for v in all_results.values())
print(f"\n总计: {len(all_results)}个市州, {total_pdf}个PDF文件")