#!/usr/bin/env python3
"""
Fetch all published articles from WeChat Official Account via API.

Uses the material/batchget API to retrieve all published news articles,
saves their HTML content and metadata for style analysis.

Also supports fetching individual article URLs (for when API isn't configured).

Usage:
    # Fetch all articles via API
    python fetch_articles.py --api --output ./articles/

    # Fetch specific article URLs
    python fetch_articles.py --urls url1.txt --output ./articles/

    # Search for articles from a specific account via Sogou
    python fetch_articles.py --search "抟微科技Tiny Wings" --output ./articles/

    # Fetch from a list of URLs in a file
    python fetch_articles.py --url-file urls.txt --output ./articles/
"""

import argparse
import json
import os
import sys
import time
import re
import urllib.request
import urllib.parse
import urllib.error
from pathlib import Path
from datetime import datetime


# ============================================================
# Config Loading (shared with publish_to_wechat.py)
# ============================================================

CONFIG_PATHS = [
    Path.cwd() / ".wechat" / "config.json",
    Path.home() / ".wechat" / "config.json",
    Path.cwd() / "wechat_config.json",
]

def load_config(config_path=None):
    """Load WeChat config from file or environment."""
    config = {}
    paths = [Path(config_path)] if config_path else CONFIG_PATHS
    for p in paths:
        if p.exists():
            with open(p, "r", encoding="utf-8") as f:
                config = json.load(f)
            break
    if os.environ.get("WECHAT_APP_ID"):
        config["app_id"] = os.environ["WECHAT_APP_ID"]
    if os.environ.get("WECHAT_APP_SECRET"):
        config["app_secret"] = os.environ["WECHAT_APP_SECRET"]
    return config


# ============================================================
# WeChat API Client (extends publish script)
# ============================================================

class WeChatFetcher:
    """Fetch published articles from WeChat Official Account."""

    BASE_URL = "https://api.weixin.qq.com/cgi-bin"

    def __init__(self, app_id, app_secret):
        self.app_id = app_id
        self.app_secret = app_secret
        self._access_token = None
        self._token_expires = 0

    def _get_access_token(self):
        if self._access_token and time.time() < self._token_expires - 300:
            return self._access_token

        url = f"{self.BASE_URL}/token"
        params = {
            "grant_type": "client_credential",
            "appid": self.app_id,
            "secret": self.app_secret,
        }
        url = f"{url}?{urllib.parse.urlencode(params)}"

        req = urllib.request.Request(url, method="GET")
        with urllib.request.urlopen(req, timeout=30) as resp:
            result = json.loads(resp.read().decode("utf-8"))
            if "access_token" in result:
                self._access_token = result["access_token"]
                self._token_expires = time.time() + result.get("expires_in", 7200)
                return self._access_token
            raise Exception(f"Failed to get access token: {result}")

    def _http_post(self, url, data):
        data = json.dumps(data, ensure_ascii=False).encode("utf-8")
        req = urllib.request.Request(url, data=data, method="POST")
        req.add_header("Content-Type", "application/json; charset=utf-8")
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read().decode("utf-8"))

    def get_material_count(self):
        """Get total material count by type."""
        token = self._get_access_token()
        url = f"{self.BASE_URL}/material/get_materialcount?access_token={token}"
        req = urllib.request.Request(url, method="GET")
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read().decode("utf-8"))

    def batch_get_news(self, offset=0, count=20):
        """Batch get published news articles (materials of type 'news').
        
        Returns list of articles with their content.
        Each article item contains:
        - media_id: unique ID
        - content: HTML content of the article
        - update_time: last update timestamp
        - title: article title (inside content.news_item[0])
        """
        token = self._get_access_token()
        url = f"{self.BASE_URL}/material/batchget?access_token={token}"
        data = {
            "type": "news",
            "offset": offset,
            "count": count,
            "no_content": 0  # 0 = include content, 1 = no content
        }
        return self._http_post(url, data)

    def get_all_articles(self, max_count=100):
        """Fetch all published articles with pagination.
        
        Args:
            max_count: Maximum number of articles to fetch (safety limit)
        
        Returns:
            List of article dicts with title, content, url, update_time
        """
        all_articles = []
        offset = 0
        page_size = 20

        # First, get total count
        try:
            counts = self.get_material_count()
            total = counts.get("news_count", 0)
            print(f"Total published articles: {total}")
        except Exception as e:
            print(f"Warning: Could not get article count: {e}")
            total = max_count

        total = min(total, max_count)

        while offset < total:
            print(f"Fetching articles {offset+1}-{min(offset+page_size, total)}...")
            try:
                result = self.batch_get_news(offset=offset, count=page_size)

                if "item" not in result or not result["item"]:
                    break

                for item in result["item"]:
                    content = item.get("content", {})
                    news_items = content.get("news_item", [])

                    for news in news_items:
                        article = {
                            "media_id": item.get("media_id", ""),
                            "title": news.get("title", ""),
                            "author": news.get("author", ""),
                            "digest": news.get("digest", ""),
                            "content": news.get("content", ""),
                            "url": news.get("url", ""),
                            "thumb_url": news.get("thumb_url", ""),
                            "update_time": item.get("update_time", 0),
                            "update_date": datetime.fromtimestamp(
                                item.get("update_time", 0)
                            ).strftime("%Y-%m-%d %H:%M"),
                        }
                        all_articles.append(article)

                offset += page_size
                # Rate limiting
                time.sleep(1)

            except Exception as e:
                print(f"Error fetching page at offset {offset}: {e}")
                break

        print(f"\nFetched {len(all_articles)} articles total.")
        return all_articles


# ============================================================
# URL Fetcher (for individual article URLs)
# ============================================================

def fetch_article_from_url(url):
    """Fetch article HTML content from a WeChat article URL."""
    try:
        req = urllib.request.Request(url, method="GET")
        req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
        with urllib.request.urlopen(req, timeout=30) as resp:
            html = resp.read().decode("utf-8", errors="replace")
        
        # Extract title
        title_match = re.search(r'<h1[^>]*class="[^"]*rich_media_title[^"]*"[^>]*>(.*?)</h1>', html, re.DOTALL)
        title = title_match.group(1).strip() if title_match else "Unknown Title"
        
        # Extract article content area
        content_match = re.search(
            r'<div[^>]*class="[^"]*rich_media_content[^"]*"[^>]*id="js_content"[^>]*>(.*?)</div>\s*<script',
            html, re.DOTALL
        )
        content = content_match.group(1).strip() if content_match else html

        # Extract publish date
        date_match = re.search(r'var\s+createTime\s*=\s*[\'"]([^\'\"]+)[\'"]', html)
        publish_date = date_match.group(1) if date_match else ""

        # Extract author
        author_match = re.search(r'var\s+nickname\s*=\s*[\'"]([^\'\"]+)[\'"]', html)
        author = author_match.group(1) if author_match else ""

        return {
            "title": title,
            "content": content,
            "url": url,
            "author": author,
            "publish_date": publish_date,
            "raw_html": html,
        }
    except Exception as e:
        print(f"Error fetching URL {url}: {e}")
        return None


# ============================================================
# Article Classification
# ============================================================

def classify_article(title, content, digest=""):
    """Classify article into type based on title and content keywords.
    
    Returns one of: tech, industry, culture, marketing, exhibition, default
    """
    text = f"{title} {digest} {content[:500]}".lower()

    # Exhibition / event
    exhibition_keywords = ["展会", "博览会", "工博会", "亮相", "参展", "展位", "邀您", "预告", "回顾"]
    if any(kw in text for kw in exhibition_keywords):
        return "exhibition"

    # Culture / Holiday
    culture_keywords = ["开工", "大吉", "妇女节", "致敬", "元宵", "春节", "祝福", "节日",
                        "团队", "活动", "阅兵", "芳华", "奋斗"]
    if any(kw in text for kw in culture_keywords):
        return "culture"

    # Marketing
    marketing_keywords = ["神器", "后悔", "一锅粥", "追问", "底气", "选它",
                          "车间乱", "真香", "种草"]
    if any(kw in text for kw in marketing_keywords):
        return "marketing"

    # Industry insight
    industry_keywords = ["趋势", "行业", "洞察", "发展方向", "白皮书",
                         "市场分析", "行业报告"]
    if any(kw in text for kw in industry_keywords):
        return "industry"

    # Tech (default for this account)
    tech_keywords = ["mes", "系统", "数字化", "转型", "制造", "智能", "sfc",
                     "mom", "数据", "架构", "微服务", "流程", "管理"]
    if any(kw in text for kw in tech_keywords):
        return "tech"

    return "default"


# ============================================================
# Style Analysis
# ============================================================

def analyze_styles(articles):
    """Analyze HTML styles across articles and extract common patterns.
    
    Returns a style profile dict with:
    - common_colors: most used colors
    - common_fonts: most used font families
    - common_sizes: most used font sizes
    - section_patterns: detected section header patterns
    - structure_patterns: detected article structure patterns
    - per_type_profiles: style profiles per article type
    """
    color_counts = {}
    font_sizes = {}
    bg_colors = {}
    border_colors = {}
    section_patterns = {"PART": 0, "数字序号": 0, "一、二、三": 0, "前言/结语": 0}
    structure_patterns = {"has_header_number": 0, "has_author": 0, "has_end": 0, "has_cta": 0}

    per_type_articles = {}

    for article in articles:
        content = article.get("content", "") or article.get("raw_html", "")
        title = article.get("title", "")
        digest = article.get("digest", "")

        # Classify
        article_type = classify_article(title, content, digest)
        article["type"] = article_type

        if article_type not in per_type_articles:
            per_type_articles[article_type] = []
        per_type_articles[article_type].append(article)

        # Extract colors from inline styles
        color_matches = re.findall(r'color:\s*(#[0-9a-fA-F]{3,8})', content)
        for c in color_matches:
            c_lower = c.lower()
            color_counts[c_lower] = color_counts.get(c_lower, 0) + 1

        # Extract background colors
        bg_matches = re.findall(r'background[^:]*:\s*(#[0-9a-fA-F]{3,8})', content)
        for c in bg_matches:
            c_lower = c.lower()
            bg_colors[c_lower] = bg_colors.get(c_lower, 0) + 1

        # Extract border colors
        border_matches = re.findall(r'border[^:]*:\s*\d+px\s+\w+\s+(#[0-9a-fA-F]{3,8})', content)
        for c in border_matches:
            c_lower = c.lower()
            border_colors[c_lower] = border_colors.get(c_lower, 0) + 1

        # Extract font sizes
        size_matches = re.findall(r'font-size:\s*(\d+)px', content)
        for s in size_matches:
            font_sizes[s] = font_sizes.get(s, 0) + 1

        # Detect section patterns
        if re.search(r'PART\.?\s*\d+', content, re.IGNORECASE):
            section_patterns["PART"] += 1
        if re.search(r'<[^>]*>\s*\d{2}\s*</[^>]*>', content):
            section_patterns["数字序号"] += 1
        if re.search(r'[一二三四五六七八九十]+、', content):
            section_patterns["一、二、三"] += 1
        if "前言" in content or "结语" in content:
            section_patterns["前言/结语"] += 1

        # Detect structure elements
        if "陪伴你的第" in content:
            structure_patterns["has_header_number"] += 1
        if "作者" in content and "▌" in content:
            structure_patterns["has_author"] += 1
        if "·END·" in content or "END" in content:
            structure_patterns["has_end"] += 1
        if "贴身定制方案" in content or "如果您所在的企业" in content:
            structure_patterns["has_cta"] += 1

    # Sort by frequency
    sorted_colors = sorted(color_counts.items(), key=lambda x: x[1], reverse=True)[:15]
    sorted_bg = sorted(bg_colors.items(), key=lambda x: x[1], reverse=True)[:10]
    sorted_border = sorted(border_colors.items(), key=lambda x: x[1], reverse=True)[:10]
    sorted_sizes = sorted(font_sizes.items(), key=lambda x: x[1], reverse=True)[:10]

    # Build per-type profiles
    per_type_profiles = {}
    for atype, arts in per_type_articles.items():
        per_type_profiles[atype] = {
            "count": len(arts),
            "titles": [a["title"] for a in arts[:5]],
            "example_urls": [a.get("url", "") for a in arts[:3] if a.get("url")],
        }

    profile = {
        "total_articles": len(articles),
        "common_text_colors": sorted_colors,
        "common_bg_colors": sorted_bg,
        "common_border_colors": sorted_border,
        "common_font_sizes": sorted_sizes,
        "section_patterns": section_patterns,
        "structure_patterns": structure_patterns,
        "per_type_profiles": per_type_profiles,
        "analysis_date": datetime.now().strftime("%Y-%m-%d %H:%M"),
    }

    return profile


# ============================================================
# Main CLI
# ============================================================

def main():
    parser = argparse.ArgumentParser(
        description="Fetch and analyze WeChat Official Account articles"
    )
    mode_group = parser.add_mutually_exclusive_group(required=True)
    mode_group.add_argument("--api", action="store_true",
                            help="Fetch all articles via WeChat API (requires credentials)")
    mode_group.add_argument("--urls", nargs="+",
                            help="Fetch specific article URLs")
    mode_group.add_argument("--url-file",
                            help="File containing article URLs (one per line)")

    parser.add_argument("--output", "-o", default="./articles/",
                        help="Output directory for articles")
    parser.add_argument("--config", help="WeChat config file path (for --api mode)")
    parser.add_argument("--analyze", action="store_true", default=True,
                        help="Analyze styles after fetching (default: True)")
    parser.add_argument("--max", type=int, default=100,
                        help="Max articles to fetch (for --api mode)")

    args = parser.parse_args()

    output_dir = Path(args.output)
    output_dir.mkdir(parents=True, exist_ok=True)

    articles = []

    if args.api:
        # Fetch via WeChat API
        config = load_config(args.config)
        if not config.get("app_id") or not config.get("app_secret"):
            print("Error: WeChat credentials not found. Configure via ~/.wechat/config.json "
                  "or environment variables WECHAT_APP_ID/WECHAT_APP_SECRET", file=sys.stderr)
            sys.exit(1)

        fetcher = WeChatFetcher(config["app_id"], config["app_secret"])
        print("=== Fetching articles via WeChat API ===")
        articles = fetcher.get_all_articles(max_count=args.max)

    elif args.urls:
        # Fetch specific URLs
        print(f"=== Fetching {len(args.urls)} article URLs ===")
        for url in args.urls:
            print(f"  Fetching: {url[:60]}...")
            article = fetch_article_from_url(url)
            if article:
                articles.append(article)
            time.sleep(2)  # Rate limiting

    elif args.url_file:
        # Fetch from URL file
        url_file = Path(args.url_file)
        if not url_file.exists():
            print(f"Error: URL file not found: {args.url_file}", file=sys.stderr)
            sys.exit(1)

        urls = [line.strip() for line in url_file.read_text(encoding="utf-8").splitlines()
                if line.strip() and not line.startswith("#")]
        print(f"=== Fetching {len(urls)} article URLs from {args.url_file} ===")
        for url in urls:
            print(f"  Fetching: {url[:60]}...")
            article = fetch_article_from_url(url)
            if article:
                articles.append(article)
            time.sleep(2)

    if not articles:
        print("No articles fetched.", file=sys.stderr)
        sys.exit(1)

    # Save articles
    print(f"\n=== Saving {len(articles)} articles to {output_dir} ===")
    for i, article in enumerate(articles):
        # Classify
        atype = classify_article(
            article.get("title", ""),
            article.get("content", ""),
            article.get("digest", "")
        )
        article["type"] = atype

        # Save metadata
        meta = {k: v for k, v in article.items()
                if k not in ("content", "raw_html")}
        meta_file = output_dir / f"{i+1:03d}_{atype}_meta.json"
        with open(meta_file, "w", encoding="utf-8") as f:
            json.dump(meta, f, ensure_ascii=False, indent=2)

        # Save content
        content = article.get("content", "")
        html_file = output_dir / f"{i+1:03d}_{atype}.html"
        with open(html_file, "w", encoding="utf-8") as f:
            f.write(content)

    # Analyze styles
    if args.analyze:
        print("\n=== Analyzing article styles ===")
        profile = analyze_styles(articles)

        # Save profile
        profile_file = output_dir / "style_profile.json"
        with open(profile_file, "w", encoding="utf-8") as f:
            json.dump(profile, f, ensure_ascii=False, indent=2)

        # Print summary
        print(f"\n--- Style Analysis Summary ---")
        print(f"Total articles: {profile['total_articles']}")
        print(f"\nArticle type distribution:")
        for atype, info in profile["per_type_profiles"].items():
            print(f"  {atype}: {info['count']} articles")

        print(f"\nTop text colors:")
        for color, count in profile["common_text_colors"][:5]:
            print(f"  {color}: {count} occurrences")

        print(f"\nTop background colors:")
        for color, count in profile["common_bg_colors"][:5]:
            print(f"  {color}: {count} occurrences")

        print(f"\nTop font sizes:")
        for size, count in profile["common_font_sizes"][:5]:
            print(f"  {size}px: {count} occurrences")

        print(f"\nSection patterns:")
        for pattern, count in profile["section_patterns"].items():
            print(f"  {pattern}: {count} articles")

        print(f"\nStructure elements:")
        for element, count in profile["structure_patterns"].items():
            print(f"  {element}: {count}/{len(articles)} articles")

        print(f"\nStyle profile saved to: {profile_file}")

    print(f"\n=== Done! {len(articles)} articles saved to {output_dir} ===")


if __name__ == "__main__":
    main()