#!/usr/bin/env python3
"""重新分析所有预算文件，提取关键字匹配项 + 预算金额"""
import os, json, re, sys

KEYWORDS = ['网络培训','培训','能力提升','视频拍摄','党员教育片','课程制作','直播','视频','课件','系统','信息化','网络','数字','设备','平台','维护','软件']

BASE = '/data/www/files/scQxYjs/'
HTTP_BASE = 'http://192.168.99.133:5678/scQxYjs/'

def classify_unit(filename):
    fn = filename.lower()
    if 'org' in fn or '组织部' in filename or '组织' in filename:
        return '组织部'
    if '_dx_' in fn or '党校' in filename:
        return '党校'
    return '其他'

def extract_county(filename, parent_dir):
    """从文件名或父目录提取区县名"""
    # 先从父目录提取
    if parent_dir:
        for suffix in ['组织部','党校','组织','委员会']:
            if parent_dir.endswith(suffix):
                parent_dir = parent_dir[:-len(suffix)]
        # 清理
        parent_dir = parent_dir.replace(' ', '').replace('县', '县').replace('区', '区')
        if parent_dir:
            return parent_dir
    # 从文件名提取
    fn = filename
    # 去掉后缀
    for ext in ['.xlsx','.xls','.pdf','.doc','.docx','.et']:
        if fn.endswith(ext):
            fn = fn[:-len(ext)]
    # 提取区县名（简化逻辑）
    patterns = [
        r'([^/\\]+?)(?:县|区|市)',
        r'([^/\\]+?)(?:县委|区委|市委)',
    ]
    for p in patterns:
        m = re.search(p, fn)
        if m:
            return m.group(1) + re.search(r'(县|区|市)', fn).group(1) if re.search(r'(县|区|市)', fn) else m.group(1)
    return filename[:20]

def analyze_excel(filepath):
    """分析Excel文件，提取关键字匹配项和金额"""
    import openpyxl
    results = []
    try:
        wb = openpyxl.load_workbook(filepath, data_only=True, read_only=True)
    except:
        try:
            import xlrd
            wb = None  # xlrd处理
            wb_xls = xlrd.open_workbook(filepath)
            for sn in wb_xls.sheet_names():
                ws = wb_xls.sheet_by_name(sn)
                for r in range(ws.nrows):
                    row = [str(ws.cell_value(r, c) or '') for c in range(ws.ncols)]
                    row_text = ' '.join(row)
                    # 找金额（在数字列中）
                    for kw in KEYWORDS:
                        if kw in row_text:
                            # 找金额：从第6列开始找数字
                            amount = ''
                            for c in range(min(5, ws.ncols), ws.ncols):
                                v = str(ws.cell_value(r, c) or '')
                                try:
                                    float(v)
                                    if float(v) > 0:
                                        amount = v
                                        break
                                except:
                                    pass
                            results.append({
                                '关键字': kw,
                                '内容': row_text[:100].strip(),
                                '位置': f'{sn},行{r+1}',
                                '金额': amount
                            })
            return results
        except Exception as e:
            return [{'关键字': '无匹配', '内容': f'文件解析失败: {e}', '位置': '-', '金额': ''}]
    
    for sn in wb.sheetnames:
        ws = wb[sn]
        for row in ws.iter_rows(min_row=1, values_only=False):
            row_text = ' '.join([str(c.value or '') for c in row])
            if not row_text.strip():
                continue
            for kw in KEYWORDS:
                if kw in row_text:
                    # 找金额：优先第6列（F列，总计），其次第7列（G列）
                    amount = ''
                    # F列(index 5) = 总计, G列(index 6) = 合计
                    for col_idx in [5, 6, 7, 8, 9, 4]:
                        if col_idx < len(row):
                            v = row[col_idx].value
                            if v is not None:
                                try:
                                    fv = float(v)
                                    if fv > 0:
                                        amount = str(fv)
                                        break
                                except:
                                    pass
                    r = row[0].row if hasattr(row[0], 'row') else '?'
                    results.append({
                        '关键字': kw,
                        '内容': row_text[:100].strip(),
                        '位置': f'{sn},行{r}',
                        '金额': amount
                    })
    wb.close()
    if not results:
        results = [{'关键字': '无匹配', '内容': '文件中未找到任何关键字', '位置': '-', '金额': ''}]
    return results

def analyze_pdf(filepath):
    """分析PDF文件，提取关键字匹配项和金额"""
    import subprocess
    results = []
    # 先用pdftotext提取
    try:
        proc = subprocess.run(['pdftotext', '-layout', filepath, '-'], capture_output=True, text=True, timeout=30)
        text = proc.stdout
    except:
        text = ''
    
    if not text.strip():
        # 尝试OCR
        try:
            from pdf2image import convert_from_path
            import pytesseract
            images = convert_from_path(filepath, first_page=1, last_page=10)
            text = ''
            for img in images:
                text += pytesseract.image_to_string(img, lang='chi_sim') + '\n'
        except:
            return [{'关键字': '无匹配', '内容': 'PDF无法解析(可能需要OCR)', '位置': '-', '金额': ''}]
    
    lines = text.split('\n')
    for i, line in enumerate(lines):
        if not line.strip():
            continue
        for kw in KEYWORDS:
            if kw in line:
                # 尝试从行中提取金额
                amount = ''
                # 找数字（万元）
                amt_match = re.search(r'(\d+\.?\d*)\s*万?元?', line)
                if amt_match:
                    try:
                        fv = float(amt_match.group(1))
                        if fv > 0:
                            amount = str(fv)
                    except:
                        pass
                results.append({
                    '关键字': kw,
                    '内容': line.strip()[:100],
                    '位置': f'第{i+1}行',
                    '金额': amount
                })
    if not results:
        results = [{'关键字': '无匹配', '内容': '文件中未找到任何关键字', '位置': '-', '金额': ''}]
    return results

def analyze_doc(filepath):
    """分析Word文档"""
    import subprocess
    results = []
    # 用antiword或catdoc提取
    for tool in ['antiword', 'catdoc']:
        try:
            proc = subprocess.run([tool, filepath], capture_output=True, text=True, timeout=30)
            if proc.stdout.strip():
                text = proc.stdout
                break
        except:
            text = ''
    else:
        # 尝试python-docx
        try:
            from docx import Document
            doc = Document(filepath)
            text = '\n'.join([p.text for p in doc.paragraphs])
            # 也提取表格
            for table in doc.tables:
                for row in table.rows:
                    row_text = ' '.join([cell.text for cell in row.cells])
                    text += '\n' + row_text
        except:
            return [{'关键字': '无匹配', '内容': 'Word文档无法解析', '位置': '-', '金额': ''}]
    
    lines = text.split('\n')
    for i, line in enumerate(lines):
        if not line.strip():
            continue
        for kw in KEYWORDS:
            if kw in line:
                amount = ''
                amt_match = re.search(r'(\d+\.?\d*)\s*万?元?', line)
                if amt_match:
                    try:
                        fv = float(amt_match.group(1))
                        if fv > 0:
                            amount = str(fv)
                    except:
                        pass
                results.append({
                    '关键字': kw,
                    '内容': line.strip()[:100],
                    '位置': f'第{i+1}行',
                    '金额': amount
                })
    if not results:
        results = [{'关键字': '无匹配', '内容': '文件中未找到任何关键字', '位置': '-', '金额': ''}]
    return results

def main():
    # 遍历所有文件
    all_results = {}  # {city: {unit: {county: [{file, matches}]}}}
    file_count = 0
    match_count = 0
    
    # 找到所有预算文件
    for root, dirs, files in os.walk(BASE):
        for f in files:
            if f.endswith(('.xlsx','.xls','.pdf','.doc','.docx','.et')) and not f.startswith('_'):
                filepath = os.path.join(root, f)
                rel_path = os.path.relpath(filepath, BASE)
                
                # 确定市州（一级目录）
                parts = rel_path.split('/')
                city = parts[0] if len(parts) > 1 else '其他'
                
                # 确定单位类型
                unit = classify_unit(f)
                if unit == '其他':
                    # 检查路径中是否含组织部/党校
                    full_path_text = rel_path
                    if '党校' in full_path_text or '_dx_' in full_path_text.lower():
                        unit = '党校'
                    elif '组织' in full_path_text or '_org_' in full_path_text.lower():
                        unit = '组织部'
                    else:
                        unit = '其他'
                
                # 确定区县
                county = ''
                # 从文件名提取
                fn = f
                # 去掉后缀
                for ext in ['.xlsx','.xls','.pdf','.doc','.docx','.et']:
                    if fn.lower().endswith(ext):
                        fn = fn[:-len(ext)]
                        break
                
                # 尝试提取区县
                # 方法1: 从路径中的子目录提取
                if len(parts) > 2:
                    county = parts[-2]  # 父目录
                    # 清理后缀
                    for suffix in ['组织部','党校','组织','委员会','县委','区委']:
                        if county.endswith(suffix):
                            county = county[:-len(suffix)]
                            break
                    county = county.strip()
                
                # 方法2: 从文件名提取
                if not county:
                    m = re.search(r'((?:[^\s/\\]+?)(?:县|区|市))', fn)
                    if m:
                        county = m.group(1)
                    else:
                        county = fn[:15]
                
                # 分析文件
                ext = f.lower()
                if ext.endswith(('.xlsx', '.xls')):
                    matches = analyze_excel(filepath)
                elif ext.endswith('.pdf'):
                    matches = analyze_pdf(filepath)
                elif ext.endswith(('.doc', '.docx')):
                    matches = analyze_doc(filepath)
                elif ext.endswith('.et'):
                    # WPS的et文件，尝试用Excel方式打开
                    matches = analyze_excel(filepath)
                else:
                    matches = [{'关键字': '无匹配', '内容': '不支持的文件格式', '位置': '-', '金额': ''}]
                
                # 计算匹配数（排除"无匹配"）
                real_matches = [m for m in matches if m['关键字'] != '无匹配']
                match_count += len(real_matches)
                
                # 构建文件记录
                file_record = {
                    '文件': rel_path,
                    '文件URL': HTTP_BASE + rel_path,
                    '匹配项': matches
                }
                
                # 存入结果
                if city not in all_results:
                    all_results[city] = {}
                if unit not in all_results[city]:
                    all_results[city][unit] = {}
                if county not in all_results[city][unit]:
                    all_results[city][unit][county] = []
                all_results[city][unit][county].append(file_record)
                
                file_count += 1
                if file_count % 50 == 0:
                    print(f"已处理 {file_count} 个文件, {match_count} 个匹配项...", flush=True)
    
    # 保存结果
    outpath = os.path.join(BASE, '_all_results_v2.json')
    with open(outpath, 'w', encoding='utf-8') as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)
    
    print(f"\n完成! {file_count}个文件, {match_count}个匹配项")
    print(f"结果保存到: {outpath}")
    
    # 统计有金额的记录
    has_amount = 0
    total = 0
    for city in all_results:
        for unit in all_results[city]:
            for county in all_results[city][unit]:
                for fr in all_results[city][unit][county]:
                    for m in fr['匹配项']:
                        total += 1
                        if m.get('金额') and m['金额'] not in ('', '0', '0.0'):
                            has_amount += 1
    print(f"有金额记录: {has_amount}/{total}")

if __name__ == '__main__':
    main()