#!/usr/bin/env python3
"""Parse all budget files in 2026-yjs-wf/ directory."""
import os
import re
import json
import traceback

# Keywords to match
KEYWORDS = [
    "党员教育", "干部教育", "干部培训", "教育培训",
    "人才", "党建", "党员", "干部",
    "网络", "信息化", "智慧",
    "平台", "系统", "运维", "维护",
    "视频", "会议"
]

# City mapping
CITY_MAP = {
    "凉山州": "凉山州",
    "南充市": "南充市",
    "广元市": "广元市",
    "广安市": "广安市",
    "德阳市": "德阳市",
    "遂宁市": "遂宁市",
    "阿坝州": "阿坝州",
}

def get_city(path):
    for city in CITY_MAP:
        if f"/{city}/" in path or path.startswith(city + "/"):
            return city
    return "未分类"

def get_county(path, filename):
    """Extract county from path."""
    parts = path.split("/")
    if len(parts) < 2:
        return "市级"
    # parts[0] = city, parts[1+] = subdirs or files
    sub = parts[1] if len(parts) > 1 else ""
    
    # Check if it's a county directory
    county_keywords = ["县", "区", "市", "州"]
    if any(k in sub for k in county_keywords) and sub != parts[0]:
        # Check if it's a sub-directory (county level)
        if len(parts) > 2:
            return sub
        # Direct file in city dir
        # Try to extract from filename
        for kw in county_keywords:
            idx = filename.find(kw)
            if idx > 0:
                # Extract county name
                start = max(0, idx - 6)
                county = filename[start:idx+1]
                # Clean up
                county = re.sub(r'.*?([\u4e00-\u9fa5]+[县区市])$', r'\1', county)
                if len(county) <= 8:
                    return county
        return "市级"
    return "市级"

def get_unit_type(filename, path):
    """Determine if it's 党校 or 组织部."""
    text = filename + " " + path
    if "党校" in text or "dangxiao" in text.lower():
        return "党校"
    if "组织部" in text or "zuzhibu" in text.lower() or "组织" in text:
        return "组织部"
    if "社工部" in text:
        return "社工部"
    if "财政局" in text:
        return "财政局"
    if "地方志" in text:
        return "其他"
    return "其他"

def parse_excel(filepath):
    """Parse Excel file."""
    from openpyxl import load_workbook
    from xlrd import open_workbook
    
    results = []
    try:
        if filepath.endswith('.xls'):
            wb = open_workbook(filepath)
            for sheet in wb.sheets():
                for row_idx in range(sheet.nrows):
                    row_data = [str(sheet.cell_value(row_idx, col)) for col in range(sheet.ncols)]
                    row_text = " ".join(row_data)
                    for kw in KEYWORDS:
                        if kw in row_text:
                            # Extract amount
                            amount = extract_amount(row_data)
                            results.append({
                                "keyword": kw,
                                "content": row_text[:500],
                                "amount": amount,
                                "location": f"Sheet: {sheet.name}, Row: {row_idx+1}",
                            })
        else:
            wb = load_workbook(filepath, data_only=True)
            for sheet_name in wb.sheetnames:
                ws = wb[sheet_name]
                for row in ws.iter_rows(values_only=True):
                    row_data = [str(c) if c is not None else "" for c in row]
                    row_text = " ".join(row_data)
                    for kw in KEYWORDS:
                        if kw in row_text:
                            amount = extract_amount(row_data)
                            results.append({
                                "keyword": kw,
                                "content": row_text[:500],
                                "amount": amount,
                                "location": f"Sheet: {sheet_name}, Row: {row[0].row if hasattr(row[0], 'row') else ''}",
                            })
    except Exception as e:
        print(f"  Error parsing Excel {filepath}: {e}")
    return results

def parse_word(filepath):
    """Parse Word document."""
    results = []
    try:
        if filepath.endswith('.doc'):
            # Try antiword first
            import subprocess
            try:
                result = subprocess.run(['antiword', filepath], capture_output=True, text=True, timeout=30)
                if result.returncode == 0:
                    text = result.stdout
                else:
                    # Try catdoc
                    result = subprocess.run(['catdoc', filepath], capture_output=True, text=True, timeout=30)
                    text = result.stdout if result.returncode == 0 else ""
            except:
                try:
                    result = subprocess.run(['catdoc', filepath], capture_output=True, text=True, timeout=30)
                    text = result.stdout if result.returncode == 0 else ""
                except:
                    text = ""
        else:
            from docx import Document
            doc = Document(filepath)
            text = "\n".join([p.text for p in doc.paragraphs])
            # Also check tables
            for table in doc.tables:
                for row in table.rows:
                    for cell in row.cells:
                        text += "\n" + cell.text
        
        if text:
            lines = text.split("\n")
            for i, line in enumerate(lines):
                for kw in KEYWORDS:
                    if kw in line:
                        amount = extract_amount([line])
                        results.append({
                            "keyword": kw,
                            "content": line[:500],
                            "amount": amount,
                            "location": f"Line: {i+1}",
                        })
    except Exception as e:
        print(f"  Error parsing Word {filepath}: {e}")
    return results

def parse_pdf(filepath):
    """Parse PDF file."""
    results = []
    try:
        import subprocess
        result = subprocess.run(['pdftotext', '-layout', filepath, '-'], capture_output=True, text=True, timeout=60)
        text = result.stdout
        if text:
            lines = text.split("\n")
            for i, line in enumerate(lines):
                for kw in KEYWORDS:
                    if kw in line:
                        amount = extract_amount([line])
                        results.append({
                            "keyword": kw,
                            "content": line[:500],
                            "amount": amount,
                            "location": f"Line: {i+1}",
                        })
    except Exception as e:
        print(f"  Error parsing PDF {filepath}: {e}")
    return results

def extract_amount(row_data):
    """Extract budget amount from row data."""
    if isinstance(row_data, str):
        row_data = [row_data]
    
    for cell in row_data:
        if not cell:
            continue
        cell = str(cell).strip()
        # Match patterns like 123.45万元, 1234.56万, 12345元
        patterns = [
            r'([\d,]+\.?\d*)\s*万元',
            r'([\d,]+\.?\d*)\s*万',
            r'([\d,]+\.?\d*)\s*元',
        ]
        for pattern in patterns:
            matches = re.findall(pattern, cell)
            for m in matches:
                try:
                    val = float(m.replace(",", ""))
                    if "万元" in cell or "万" in cell:
                        if val < 100000:  # Reasonable budget in 万
                            return val
                    elif "元" in cell:
                        if val < 1000000000 and val > 100:
                            return val / 10000  # Convert to 万
                except:
                    pass
        
        # Check if cell is a pure number (budget amount)
        try:
            val = float(cell.replace(",", ""))
            # Heuristic: amounts between 1 and 100000 (万元) 
            if 1 <= val <= 100000 and "." in cell:
                return val
        except:
            pass
    
    return None

def main():
    base_dir = "/data/www/files/2026-yjs-wf"
    all_results = []
    file_count = 0
    
    for root, dirs, files in os.walk(base_dir):
        for filename in files:
            filepath = os.path.join(root, filename)
            rel_path = os.path.relpath(filepath, base_dir)
            ext = os.path.splitext(filename)[1].lower()
            
            if ext in ['.xlsx', '.xls']:
                print(f"Parsing Excel: {rel_path}")
                matches = parse_excel(filepath)
            elif ext in ['.doc', '.docx']:
                print(f"Parsing Word: {rel_path}")
                matches = parse_word(filepath)
            elif ext == '.pdf':
                print(f"Parsing PDF: {rel_path}")
                matches = parse_pdf(filepath)
            elif ext in ['.html', '.htm']:
                print(f"Skipping HTML: {rel_path}")
                continue
            else:
                continue
            
            if matches:
                file_count += 1
                city = get_city(rel_path)
                county = get_county(rel_path, filename)
                unit_type = get_unit_type(filename, rel_path)
                
                for m in matches:
                    m["city"] = city
                    m["county"] = county
                    m["unit_type"] = unit_type
                    m["source_file"] = rel_path
                    m["filename"] = filename
                    all_results.append(m)
                print(f"  Found {len(matches)} matches")
    
    # Save JSON
    output_path = "/data/www/files/wf-2026-data.json"
    with open(output_path, 'w', encoding='utf-8') as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)
    
    print(f"\n=== Summary ===")
    print(f"Files parsed: {file_count}")
    print(f"Total matches: {len(all_results)}")
    
    # Stats
    cities = {}
    keywords = {}
    total_amount = 0
    amount_count = 0
    
    for m in all_results:
        city = m["city"]
        if city not in cities:
            cities[city] = 0
        cities[city] += 1
        
        kw = m["keyword"]
        if kw not in keywords:
            keywords[kw] = {"count": 0, "amount": 0}
        keywords[kw]["count"] += 1
        if m["amount"]:
            keywords[kw]["amount"] += m["amount"]
            total_amount += m["amount"]
            amount_count += 1
    
    print(f"\nBy city:")
    for c, n in sorted(cities.items()):
        print(f"  {c}: {n}")
    
    print(f"\nBy keyword:")
    for k, v in sorted(keywords.items(), key=lambda x: -x[1]["count"]):
        print(f"  {k}: {v['count']} matches, {v['amount']:.2f}万元")
    
    print(f"\nTotal amount: {total_amount:.2f}万元 ({amount_count} entries with amounts)")
    print(f"\nJSON saved to: {output_path}")

if __name__ == "__main__":
    main()