#!/usr/bin/env python3 """高效批量抓取剩余市州组织部/党校预算 - Selenium版""" import json, time, os, re, base64, subprocess from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC SAVE_DIR = "/data/www/files/budget_pdfs" os.makedirs(SAVE_DIR, exist_ok=True) subprocess.run(["pkill", "-9", "chrome"], capture_output=True) time.sleep(1) # 配置：市州名称 -> 政府网站域名前缀 cities = { "乐山": "leshan", "达州": "dazhou", "资阳": "ziyang", "雅安": "yaan", "甘孜": "gzz", } # 常见URL模式（基于金牛区的URL结构） # /xx/c110728/ = 区级部门预决算 # /xx/c110737/ = 区委党校类 # 但市州级不同，需要先探索 def init_driver(): opts = Options() opts.add_argument('--headless=new') opts.add_argument('--no-sandbox') opts.add_argument('--disable-gpu') opts.add_argument('--disable-dev-shm-usage') opts.add_argument('--disable-blink-features=AutomationControlled') opts.add_argument('--window-size=1920,1080') opts.add_experimental_option("excludeSwitches", ["enable-automation"]) d = webdriver.Chrome(options=opts) d.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', { 'source': 'Object.defineProperty(navigator, "webdriver", {get: () => undefined})' }) d.set_page_load_timeout(25) return d def safe_get(driver, url, wait=6): try: driver.get(url) time.sleep(wait) return len(driver.page_source) > 500 except: return False def extract_links(driver, keywords): """提取页面中包含关键字的链接""" links = [] try: elems = driver.find_elements(By.TAG_NAME, "a") for a in elems: text = (a.text or "").strip() href = a.get_attribute("href") or "" if text and href and any(kw in text for kw in keywords): links.append({"text": text, "href": href}) except: pass return links def download_via_fetch(driver, url, filepath): """通过浏览器Fetch API下载文件绕过WAF""" try: result = driver.execute_async_script(""" var url = arguments[0]; var callback = arguments[1]; fetch(url) .then(r => { if(!r.ok) throw new Error('HTTP '+r.status); return r.arrayBuffer(); }) .then(buf => { var arr = new Uint8Array(buf); var binary = ''; for (var i = 0; i < arr.length; i++) binary += String.fromCharCode(arr[i]); callback({status:'ok', size:arr.length, data:btoa(binary)}); }) .catch(e => callback({status:'error', msg:e.message})); """, url) if result and result.get('status') == 'ok': data = base64.b64decode(result['data']) with open(filepath, 'wb') as f: f.write(data) return os.path.getsize(filepath) except Exception as e: print(f" 下载异常: {str(e)[:80]}") return 0 # 关键字 kw_finance = ['财政信息', '财政预决算', '部门预算', '预决算公开', '财政公开', '预决算'] kw_dept = ['组织部', '党校'] kw_budget_file = ['2026', '预算'] results = {} driver = init_driver() for city, prefix in cities.items(): print(f"\n{'='*50}") print(f"🔍 {city} ({prefix}.gov.cn)") base = f"https://www.{prefix}.gov.cn" city_result = {"city": city, "base_url": base, "org": {}, "school": {}} # 第1步：访问首页 if not safe_get(driver, base, 8): print(f" ❌ 首页无法访问") city_result["status"] = "首页不可达" results[city] = city_result continue print(f" ✅ 首页已加载: {driver.title}") driver.save_screenshot(f"{SAVE_DIR}/screenshot_{city}_home.png") # 第2步：找政务公开/信息公开链接 gov_links = extract_links(driver, ['政务公开', '信息公开', '公开']) print(f" 找到公开栏目链接: {len(gov_links)}") finance_url = None # 第3步：逐一访问公开栏目，找财政信息 for gl in gov_links[:3]: print(f" → 访问: {gl['text']}") if safe_get(driver, gl['href'], 6): fin_links = extract_links(driver, kw_finance) if fin_links: finance_url = fin_links[0]['href'] print(f" ✅ 找到财政栏目: {fin_links[0]['text']}") break # 第4步：如果没找到，尝试直接构造URL if not finance_url: # 尝试常见路径 paths = [ f"/{prefix}/c106665/", # 乐山模式 f"/{prefix}/zwgk/", # 政务公开 f"/{prefix}/c100549/", # 信息主动公开 f"/{prefix}/jgjj/", # 机构简介 ] for p in paths: test_url = base + p if safe_get(driver, test_url, 5): flinks = extract_links(driver, kw_finance) if flinks: finance_url = flinks[0]['href'] print(f" ✅ 构造路径成功: {flinks[0]['text']}") break # 第5步：在财政信息页搜索组织部和党校 dept_found = {} if finance_url: safe_get(driver, finance_url, 6) driver.save_screenshot(f"{SAVE_DIR}/screenshot_{city}_finance.png") # 找部门列表中的组织部和党校 page_links = extract_links(driver, kw_dept + ['部门预决算', '区级部门', '部门预算', '部门及']) print(f" 财政页面链接: {len(page_links)}") # 先找部门预决算入口 dept_entry = [l for l in page_links if any(kw in l['text'] for kw in ['部门预决算', '区级部门', '部门及'])] if dept_entry: print(f" → 进入部门预决算: {dept_entry[0]['text']}") safe_get(driver, dept_entry[0]['href'], 6) dept_links = extract_links(driver, kw_dept) for dl in dept_links: text = dl['text'] if '组织部' in text: dept_found['org'] = dl elif '党校' in text: dept_found['school'] = dl # 如果没找到入口，直接搜索部门名 if not dept_found: for dl in page_links: text = dl['text'] if '组织部' in text and 'org' not in dept_found: dept_found['org'] = dl elif '党校' in text and 'school' not in dept_found: dept_found['school'] = dl # 第6步：访问组织部和党校的预算页面 for dtype, info in dept_found.items(): dept_name = "组织部" if dtype == "org" else "党校" print(f" 📄 访问{city}{dept_name}: {info['text']}") safe_get(driver, info['href'], 6) driver.save_screenshot(f"{SAVE_DIR}/screenshot_{city}_{dtype}.png") # 找2026预算附件 page_text = driver.find_element(By.TAG_NAME, "body").text # 找附件链接 att_links = [] elems = driver.find_elements(By.TAG_NAME, "a") for a in elems: href = a.get_attribute("href") or "" text = (a.text or "").strip() if any(href.endswith(ext) for ext in ['.pdf', '.xlsx', '.xls', '.doc', '.docx']): att_links.append({"text": text, "href": href}) # 也搜索页面内的列表链接（2026年预算公告） budget_links = [] for a in elems: text = (a.text or "").strip() href = a.get_attribute("href") or "" if text and any(kw in text for kw in ['2026', '预算']) and href: budget_links.append({"text": text, "href": href}) city_result[dtype] = { "page_url": info['href'], "page_title": info['text'], "matching_text": [l for l in page_text.split('\n') if any(kw in l for kw in ['培训', '信息化', '系统', '课件', '视频', '网络培训', '能力提升'])], "attachments": att_links, "budget_links": budget_links[:5] } # 下载附件 for att in att_links[:3]: # 最多3个附件 ext = os.path.splitext(att['href'])[1] or '.pdf' fname = f"{city}_{dtype}_2026{ext}" fpath = os.path.join(SAVE_DIR, fname) size = download_via_fetch(driver, att['href'], fpath) if size > 0: att['local_file'] = fname att['file_size'] = size print(f" ✅ 下载: {fname} ({size//1024}KB)") else: print(f" ❌ 下载失败: {att['href'][:60]}") # 如果有预算公告链接，进入找附件 for bl in budget_links[:2]: if safe_get(driver, bl['href'], 5): sub_atts = [] elems2 = driver.find_elements(By.TAG_NAME, "a") for a in elems2: href = a.get_attribute("href") or "" if any(href.endswith(ext) for ext in ['.pdf', '.xlsx', '.xls', '.doc', '.docx']): ext = os.path.splitext(href)[1] or '.pdf' fname = f"{city}_{dtype}_2026_budget{ext}" fpath = os.path.join(SAVE_DIR, fname) size = download_via_fetch(driver, href, fpath) if size > 0: sub_atts.append({"fname": fname, "size": size}) print(f" ✅ 下载预算附件: {fname} ({size//1024}KB)") if sub_atts: city_result[dtype]["budget_attachments"] = sub_atts city_result["status"] = "✅" if dept_found else "⚠️未找到组织部/党校" results[city] = city_result print(f" 状态: {city_result['status']}") driver.quit() # 保存结果 with open(f"{SAVE_DIR}/remaining_cities_v2.json", 'w') as f: json.dump(results, f, ensure_ascii=False, indent=2) print(f"\n{'='*50}") print(f"抓取完成！") for city, r in results.items(): org = '✅' if r.get('org', {}).get('page_url') else '❌' school = '✅' if r.get('school', {}).get('page_url') else '❌' atts = len(r.get('org', {}).get('attachments', [])) + len(r.get('school', {}).get('attachments', [])) print(f" {city}: 组织部{org} 党校{school} 附件{atts}个")