import argparse
import html
import json
import re
import sys
import time
from datetime import datetime
from html.parser import HTMLParser
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen

import pandas as pd


API_URL = "https://openapi.naver.com/v1/search/news.json"
NAVER_NEWS_HOST = "n.news.naver.com"
DEFAULT_KEYWORDS_FILE = "woori_affiliates.md"
DEFAULT_CREDENTIALS_FILE = "naver_api_credentials.md"
OUTPUT_COLUMNS = [
    "keyword",
    "matched_keywords",
    "title",
    "originallink",
    "link",
    "description",
    "pubDate",
]
EXCEL_CELL_LIMIT = 32767


def read_keywords(path: Path) -> List[str]:
    if not path.exists():
        raise FileNotFoundError(f"키워드 파일을 찾을 수 없습니다: {path}")

    keywords: List[str] = []
    seen = set()
    for raw_line in path.read_text(encoding="utf-8").splitlines():
        line = raw_line.strip()
        if not line or line.startswith("#"):
            continue
        if line not in seen:
            keywords.append(line)
            seen.add(line)

    if not keywords:
        raise ValueError(f"키워드 파일에 유효한 키워드가 없습니다: {path}")
    return keywords


def read_api_credentials(path: Path) -> Dict[str, str]:
    if not path.exists():
        raise FileNotFoundError(f"Naver API credentials file not found: {path}")

    credentials: Dict[str, str] = {}
    for raw_line in path.read_text(encoding="utf-8").splitlines():
        line = raw_line.strip().strip("`")
        if not line or line.startswith("#") or line.startswith("|"):
            continue
        line = re.sub(r"^[-*]\s+", "", line)
        if "=" in line:
            key, value = line.split("=", 1)
        elif ":" in line:
            key, value = line.split(":", 1)
        else:
            continue
        key = key.strip()
        value = value.strip().strip('"').strip("'")
        if key in {"NAVER_CLIENT_ID", "NAVER_CLIENT_SECRET"}:
            credentials[key] = value

    missing = [key for key in ("NAVER_CLIENT_ID", "NAVER_CLIENT_SECRET") if not credentials.get(key)]
    placeholders = {"YOUR_CLIENT_ID", "YOUR_CLIENT_SECRET", "발급받은_CLIENT_ID", "발급받은_CLIENT_SECRET"}
    if missing or any(credentials.get(key) in placeholders for key in credentials):
        raise ValueError(
            f"{path} file must include real NAVER_CLIENT_ID and NAVER_CLIENT_SECRET values."
        )
    return credentials


def clean_text(value: Optional[str]) -> str:
    if value is None:
        return ""
    text = re.sub(r"</?b>", "", value)
    text = re.sub(r"<[^>]+>", " ", text)
    text = html.unescape(text)
    return re.sub(r"\s+", " ", text).strip()


def excel_safe_text(value: Optional[str]) -> str:
    text = value or ""
    if len(text) <= EXCEL_CELL_LIMIT:
        return text
    return text[: EXCEL_CELL_LIMIT - 20] + " [TRUNCATED]"


def safe_filename_part(value: str) -> str:
    cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "_", value).strip(" .")
    return cleaned or "keyword"


def request_json(url: str, headers: Dict[str, str], timeout: int, retries: int) -> Dict:
    last_error: Optional[Exception] = None
    for attempt in range(retries + 1):
        try:
            request = Request(url, headers=headers)
            with urlopen(request, timeout=timeout) as response:
                return json.loads(response.read().decode("utf-8"))
        except HTTPError as exc:
            body = exc.read().decode("utf-8", errors="replace")
            last_error = RuntimeError(f"HTTP {exc.code}: {body}")
            if exc.code < 500 and exc.code != 429:
                break
        except (URLError, TimeoutError, json.JSONDecodeError) as exc:
            last_error = exc

        if attempt < retries:
            time.sleep(1.5 * (attempt + 1))

    raise RuntimeError(f"요청 실패: {url} / {last_error}")


def search_news(
    keyword: str,
    client_id: str,
    client_secret: str,
    max_results: int,
    display: int,
    timeout: int,
    retries: int,
    delay: float,
) -> List[Dict[str, str]]:
    headers = {
        "X-Naver-Client-Id": client_id,
        "X-Naver-Client-Secret": client_secret,
        "User-Agent": "woori-news-monitor/1.0",
    }

    rows: List[Dict[str, str]] = []
    max_results = min(max_results, 1000)
    display = min(max(display, 1), 100)

    for start in range(1, max_results + 1, display):
        params = {
            "query": keyword,
            "display": min(display, max_results - start + 1),
            "start": start,
            "sort": "date",
        }
        url = f"{API_URL}?{urlencode(params)}"
        data = request_json(url, headers=headers, timeout=timeout, retries=retries)
        items = data.get("items", [])
        if not items:
            break

        for item in items:
            row = {
                "keyword": keyword,
                "matched_keywords": keyword,
                "title": clean_text(item.get("title")),
                "originallink": item.get("originallink", ""),
                "link": item.get("link", ""),
                "description": clean_text(item.get("description")),
                "pubDate": item.get("pubDate", ""),
            }
            rows.append(row)

        if len(items) < params["display"]:
            break
        if delay > 0:
            time.sleep(delay)

    return rows


def add_unique_news_row(merged: Dict[str, Dict[str, str]], row: Dict[str, str]) -> None:
    key = row.get("link") or row.get("originallink") or f"{row.get('title')}|{row.get('pubDate')}"
    if key not in merged:
        merged[key] = dict(row)
        return
    keywords = set(filter(None, merged[key].get("matched_keywords", "").split(",")))
    keywords.add(row.get("keyword", ""))
    merged[key]["matched_keywords"] = ",".join(sorted(keywords))


def collect_news_results(
    keywords: List[str],
    client_id: str,
    client_secret: str,
    max_results: int,
    display: int,
    timeout: int,
    retries: int,
    delay: float,
) -> List[Dict[str, str]]:
    merged: Dict[str, Dict[str, str]] = {}
    max_results = max(1, max_results)

    for keyword_index, keyword in enumerate(keywords, start=1):
        if len(merged) >= max_results:
            break
        print(f"[{keyword_index}/{len(keywords)}] 검색: {keyword}")
        remaining = max_results - len(merged)
        rows = search_news(
            keyword=keyword,
            client_id=client_id,
            client_secret=client_secret,
            max_results=remaining,
            display=display,
            timeout=timeout,
            retries=retries,
            delay=delay,
        )
        for row in rows:
            add_unique_news_row(merged, row)
            if len(merged) >= max_results:
                break
        print(f"누적 수집: {len(merged)}/{max_results}건")

    return list(merged.values())[:max_results]


def merge_duplicate_links(rows: Iterable[Dict[str, str]]) -> List[Dict[str, str]]:
    merged: Dict[str, Dict[str, str]] = {}
    for row in rows:
        add_unique_news_row(merged, row)
    return list(merged.values())


class NaverBodyParser(HTMLParser):
    BODY_IDS = {"dic_area", "articeBody"}
    FALLBACK_CLASS_HINTS = {"newsct_article"}
    SKIP_TAGS = {"script", "style", "noscript", "iframe", "button"}

    def __init__(self) -> None:
        super().__init__(convert_charrefs=True)
        self.primary_depth = 0
        self.fallback_depth = 0
        self.skip_depth = 0
        self.primary_parts: List[str] = []
        self.fallback_parts: List[str] = []

    def _active_parts(self) -> Optional[List[str]]:
        if self.primary_depth:
            return self.primary_parts
        if self.fallback_depth:
            return self.fallback_parts
        return None

    def handle_starttag(self, tag: str, attrs: List[tuple]) -> None:
        attr = dict(attrs)
        if tag in self.SKIP_TAGS:
            self.skip_depth += 1
            return

        tag_id = attr.get("id", "")
        tag_class = attr.get("class", "")
        if tag_id in self.BODY_IDS:
            self.primary_depth = 1
        elif self.primary_depth:
            self.primary_depth += 1
        elif any(hint in tag_class for hint in self.FALLBACK_CLASS_HINTS):
            self.fallback_depth = 1
        elif self.fallback_depth:
            self.fallback_depth += 1

        parts = self._active_parts()
        if parts is not None and tag in {"br", "p", "div", "section", "article"}:
            parts.append("\n")

    def handle_endtag(self, tag: str) -> None:
        if tag in self.SKIP_TAGS and self.skip_depth:
            self.skip_depth -= 1
            return

        parts = self._active_parts()
        if parts is not None:
            if tag in {"p", "div", "section", "article"}:
                parts.append("\n")
            if self.primary_depth:
                self.primary_depth -= 1
            elif self.fallback_depth:
                self.fallback_depth -= 1

    def handle_data(self, data: str) -> None:
        parts = self._active_parts()
        if parts is not None and not self.skip_depth:
            text = data.strip()
            if text:
                parts.append(text)

    def body(self) -> str:
        parts = self.primary_parts if self.primary_parts else self.fallback_parts
        return normalize_article_body(parts)


def normalize_article_body(parts: List[str]) -> str:
    lines = []
    for part in parts:
        text = clean_text(part)
        if not text:
            continue
        if text in {"기자", "구독", "본문 요약봇", "텍스트 음성 변환 서비스 사용하기"}:
            continue
        if re.search(r"(무단전재|재배포 금지|Copyright|ⓒ|▶|기사제보|제보는|구독해 주세요)", text):
            continue
        lines.append(text)
    body = " ".join(lines)
    body = re.sub(r"\s+", " ", body).strip()
    return body


def extract_body_from_html(page: str) -> str:
    parser = NaverBodyParser()
    parser.feed(page)
    body = parser.body()
    if body:
        return body

    meta_match = re.search(
        r'<meta\s+property=["\']og:description["\']\s+content=["\'](.*?)["\']',
        page,
        flags=re.IGNORECASE | re.DOTALL,
    )
    if meta_match:
        return clean_text(meta_match.group(1))
    return ""


def fetch_article_body(url: str, timeout: int, retries: int, delay: float) -> str:
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/124.0 Safari/537.36",
        "Accept-Language": "ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7",
    }

    last_error: Optional[Exception] = None
    for attempt in range(retries + 1):
        try:
            request = Request(url, headers=headers)
            with urlopen(request, timeout=timeout) as response:
                charset = response.headers.get_content_charset() or "utf-8"
                page = response.read().decode(charset, errors="replace")
            if delay > 0:
                time.sleep(delay)
            return extract_body_from_html(page)
        except (HTTPError, URLError, TimeoutError) as exc:
            last_error = exc
            if isinstance(exc, HTTPError) and exc.code < 500 and exc.code != 429:
                break
            if attempt < retries:
                time.sleep(1.5 * (attempt + 1))

    return f"[본문 수집 실패] {last_error}"


def save_excel(rows: List[Dict[str, str]], path: Path, columns: Optional[List[str]] = None) -> None:
    columns = columns or sorted({key for row in rows for key in row.keys()})
    safe_rows = [
        {key: excel_safe_text(value) if isinstance(value, str) else value for key, value in row.items()}
        for row in rows
    ]
    df = pd.DataFrame(safe_rows)
    for column in columns:
        if column not in df.columns:
            df[column] = ""
    df = df[columns]
    df.to_excel(path, index=False, engine="openpyxl")


def load_excel_rows(path: Path) -> List[Dict[str, str]]:
    df = pd.read_excel(path, dtype=str).fillna("")
    return df.to_dict(orient="records")


def timestamp_from_news_filename(path: Path) -> Optional[str]:
    match = re.search(r"news_(\d{8}_\d{6})-", path.name)
    return match.group(1) if match else None


def keyword_in_text(keyword: str, text: str) -> bool:
    return keyword.casefold() in (text or "").casefold()


def build_arg_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="네이버 뉴스 API 기반 우리금융그룹 뉴스 모니터링 에이전트")
    parser.add_argument("--keywords-file", default=DEFAULT_KEYWORDS_FILE, help="한 줄에 하나씩 키워드가 있는 md 파일")
    parser.add_argument("--credentials-file", default=DEFAULT_CREDENTIALS_FILE, help="네이버 API 인증정보 md 파일")
    parser.add_argument("--output-dir", default="outputs", help="엑셀 저장 디렉터리")
    parser.add_argument("--max-results", type=int, default=1000, help="전체 최대 수집 건수")
    parser.add_argument("--max-results-per-keyword", type=int, help=argparse.SUPPRESS)
    parser.add_argument("--display", type=int, default=100, help="API 호출당 결과 수, 최대 100")
    parser.add_argument("--timeout", type=int, default=20, help="HTTP 요청 타임아웃 초")
    parser.add_argument("--retries", type=int, default=2, help="요청 재시도 횟수")
    parser.add_argument("--api-delay", type=float, default=0.15, help="API 호출 간 대기 초")
    parser.add_argument("--body-delay", type=float, default=0.25, help="본문 수집 간 대기 초")
    parser.add_argument("--filtered-input", help="기존 naver-filtered 엑셀에서 본문 수집 단계만 실행")
    parser.add_argument("--checkpoint-every", type=int, default=50, help="본문 수집 중간 저장 주기")
    return parser


def main() -> int:
    args = build_arg_parser().parse_args()
    try:
        credentials = read_api_credentials(Path(args.credentials_file))
    except (FileNotFoundError, ValueError) as exc:
        print(exc, file=sys.stderr)
        return 2
    client_id = credentials["NAVER_CLIENT_ID"]
    client_secret = credentials["NAVER_CLIENT_SECRET"]

    keywords_path = Path(args.keywords_file)
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    keywords = read_keywords(keywords_path)
    timestamp = (
        timestamp_from_news_filename(Path(args.filtered_input))
        if args.filtered_input
        else datetime.now().strftime("%Y%m%d_%H%M%S")
    )
    if not timestamp:
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    base = output_dir / f"news_{timestamp}"

    if args.filtered_input:
        filtered_path = Path(args.filtered_input)
        filtered_rows = load_excel_rows(filtered_path)
        print(f"기존 필터 파일 사용: {filtered_path} ({len(filtered_rows)}건)")
    else:
        requested_max_results = args.max_results_per_keyword or args.max_results
        full_rows = collect_news_results(
            keywords=keywords,
            client_id=client_id,
            client_secret=client_secret,
            max_results=requested_max_results,
            display=args.display,
            timeout=args.timeout,
            retries=args.retries,
            delay=args.api_delay,
        )
        full_path = Path(f"{base}-full.xlsx")
        save_excel(full_rows, full_path, OUTPUT_COLUMNS)
        print(f"저장: {full_path} ({len(full_rows)}건)")

        filtered_rows = [row for row in full_rows if NAVER_NEWS_HOST in row.get("link", "")]
        filtered_path = Path(f"{base}-naver-filtered.xlsx")
        save_excel(filtered_rows, filtered_path, OUTPUT_COLUMNS)
        print(f"저장: {filtered_path} ({len(filtered_rows)}건)")

    body_rows: List[Dict[str, str]] = []
    body_columns = OUTPUT_COLUMNS + ["body"]
    for index, row in enumerate(filtered_rows, start=1):
        enriched = dict(row)
        if enriched.get("body"):
            print(f"[{index}/{len(filtered_rows)}] 본문 있음: {row.get('link')}")
        else:
            print(f"[{index}/{len(filtered_rows)}] 본문 수집: {row.get('link')}")
            enriched["body"] = fetch_article_body(
                row.get("link", ""),
                timeout=args.timeout,
                retries=args.retries,
                delay=args.body_delay,
            )
        body_rows.append(enriched)
        if args.checkpoint_every > 0 and index % args.checkpoint_every == 0:
            save_excel(body_rows, Path(f"{base}-body.xlsx"), body_columns)
            print(f"중간 저장: {base}-body.xlsx ({len(body_rows)}건)")

    body_path = Path(f"{base}-body.xlsx")
    save_excel(body_rows, body_path, body_columns)
    print(f"저장: {body_path} ({len(body_rows)}건)")

    for keyword in keywords:
        keyword_rows = [row for row in body_rows if keyword_in_text(keyword, row.get("body", ""))]
        keyword_path = Path(f"{base}-body-{safe_filename_part(keyword)}.xlsx")
        save_excel(keyword_rows, keyword_path, body_columns)
        print(f"저장: {keyword_path} ({len(keyword_rows)}건)")

    return 0


if __name__ == "__main__":
    raise SystemExit(main())