from __future__ import annotations

import time
from collections import Counter

import pandas as pd
import requests
from bs4 import BeautifulSoup
from tqdm.auto import tqdm

from .atlas import generate_atlas
from .catalog import refresh_anime_catalog
from .console import console
from .config import ProjectPaths, TrainingConfig
from .io import load_csv_optional, read_lines, write_csv, write_lines
from .mal import MalClient
from .training import derive_user_genre_features, train_model


BANNED_USERNAMES = {
    "Egeslean",
    "ElrondSeregon",
    "Momo9_",
    "Nantick",
    "Rayan_04",
    "Stormy_Watches",
    "TheFifouSlain1",
    "akimxmona",
    "amione",
    "drknoodles15",
    "spicarie",
    "spickspaegal",
}


def scrape_usernames(
    paths: ProjectPaths,
    *,
    num_pages: int = 300,
    delay_seconds: float = 1.0,
    max_usernames: int = 5000,
) -> dict:
    scraped_usernames: set[str] = set()
    console.info(
        f"Scraping MAL user listing pages | pages={num_pages} | max_usernames={max_usernames}"
    )

    retry_attempts = 3
    retry_backoff_seconds = 2.0
    rate_limit_wait = 30.0

    with tqdm(total=num_pages, desc="Scraping usernames", unit="page", dynamic_ncols=True) as bar:
        for page_number in range(num_pages):
            url = f"https://myanimelist.net/users.php?show={page_number}"

            while True:
                last_error = None
                for attempt in range(1, retry_attempts + 1):
                    try:
                        response = requests.get(
                            url,
                            headers={"User-Agent": "Mozilla/5.0"},
                            timeout=30,
                        )
                        response.raise_for_status()
                        last_error = None
                        break
                    except requests.RequestException as exc:
                        last_error = exc
                        if attempt < retry_attempts:
                            wait = retry_backoff_seconds * attempt
                            print(f"[scrape] Attempt {attempt}/{retry_attempts} failed for page {page_number}: {exc} — retrying in {wait}s")
                            time.sleep(wait)

                if last_error is None:
                    break
                print(f"[scrape] All {retry_attempts} attempts failed for page {page_number}, retrying in {rate_limit_wait}s ...")
                time.sleep(rate_limit_wait)

            soup = BeautifulSoup(response.text, "html.parser")
            for tag in soup.find_all("a", href=True):
                href = tag["href"]
                if href.startswith("/profile/") and len(href.split("/")) == 3:
                    scraped_usernames.add(href.split("/")[2])

            bar.update(1)
            bar.set_postfix(unique=len(scraped_usernames))

            if len(scraped_usernames) >= max_usernames:
                console.info(f"Reached {max_usernames} usernames, stopping early")
                break

            if delay_seconds:
                time.sleep(delay_seconds)

    # Take only up to max_usernames (deterministic via sorted order)
    final_usernames = sorted(scraped_usernames)[:max_usernames]

    write_lines(final_usernames, paths.usernames)
    console.success(f"Saved {len(final_usernames)} usernames to {paths.usernames} (fresh fetch, no appending)")
    return {
        "username_path": str(paths.usernames),
        "total_usernames": len(final_usernames),
    }


def fetch_user_interactions(
    paths: ProjectPaths,
    *,
    username_source=None,
    resume: bool = True,
    refresh_existing_users: bool = True,
    max_new_public_users: int | None = None,
    sleep_seconds: float = 0.0,
) -> dict:
    paths.ensure_directories()
    client = MalClient.from_env(sleep_seconds=sleep_seconds)

    output_path = paths.user_interactions
    existing_path = output_path if output_path.exists() else paths.resolve_user_interactions_input()

    existing_df = load_csv_optional(existing_path)
    existing_user_count = int(existing_df["username"].nunique()) if not existing_df.empty else 0
    if refresh_existing_users:
        records = []
        processed_usernames = set()
        public_usernames = set()
    else:
        records = existing_df.to_dict(orient="records") if not existing_df.empty else []
        processed_usernames = set(existing_df["username"].astype(str).unique()) if not existing_df.empty else set()
        public_usernames = set(processed_usernames)

    source_path = username_source or paths.usernames
    usernames = read_lines(source_path)
    console.info(
        f"Fetching MAL animelists from {source_path} | source usernames={len(usernames)} | "
        f"cached_users={existing_user_count} | refresh_existing_users={refresh_existing_users}"
    )

    new_public_users = 0
    skipped_private_or_missing = 0
    refreshed_users = 0

    with tqdm(total=len(usernames), desc="Fetching users", unit="user", dynamic_ncols=True) as bar:
        for username in usernames:
            if username in BANNED_USERNAMES:
                bar.update(1)
                bar.set_postfix(status="banned", public=new_public_users, private=skipped_private_or_missing, rows=len(records))
                continue
            if not refresh_existing_users and username in processed_usernames:
                bar.update(1)
                bar.set_postfix(status="skipped", public=new_public_users, private=skipped_private_or_missing, rows=len(records))
                continue

            if username in processed_usernames:
                refreshed_users += 1

            had_public_entries = False
            user_anime_count = 0
            for entry in client.iter_user_anime_list(username, fields=["list_status"]):
                anime = entry.get("node", {})
                list_status = entry.get("list_status", {})
                anime_id = anime.get("id")
                if anime_id is None:
                    continue

                had_public_entries = True
                user_anime_count += 1
                records.append(
                    {
                        "username": username,
                        "anime_id": int(anime_id),
                        "anime_title": anime.get("title", ""),
                        "status": list_status.get("status", ""),
                        "score": int(list_status.get("score", 0) or 0),
                    }
                )

            if had_public_entries:
                public_usernames.add(username)
                if username not in processed_usernames:
                    new_public_users += 1
            else:
                skipped_private_or_missing += 1

            processed_usernames.add(username)

            if len(processed_usernames) > 0 and len(processed_usernames) % 50 == 0:
                interim_df = pd.DataFrame(records)
                write_csv(interim_df, output_path)
                write_lines(sorted(public_usernames), paths.usernames_cleaned)

            bar.update(1)
            bar.set_postfix(
                user=username[:20],
                anime=user_anime_count,
                public=new_public_users,
                refreshed=refreshed_users,
                private=skipped_private_or_missing,
                rows=len(records),
            )

            if max_new_public_users is not None and new_public_users >= max_new_public_users:
                break

    interactions_df = pd.DataFrame(
        records,
        columns=["username", "anime_id", "anime_title", "status", "score"],
    )
    write_csv(interactions_df, output_path)
    write_lines(sorted(public_usernames), paths.usernames_cleaned)
    console.success(
        f"Saved {len(interactions_df)} interaction rows for {interactions_df['username'].nunique() if not interactions_df.empty else 0} users"
    )

    return {
        "interaction_path": str(output_path),
        "clean_username_path": str(paths.usernames_cleaned),
        "rows": int(len(interactions_df)),
        "public_users_total": int(interactions_df["username"].nunique()) if not interactions_df.empty else 0,
        "new_public_users": new_public_users,
        "refreshed_users": refreshed_users,
        "private_or_missing_users": skipped_private_or_missing,
    }


def _classify_relationship(client: MalClient, anime_id: int) -> str:
    relationship_chain = []
    seen_ids = set()
    current_id = int(anime_id)

    while current_id not in seen_ids:
        seen_ids.add(current_id)
        detail = client.get_anime_details(
            current_id,
            fields=["related_anime", "media_type", "num_episodes"],
            allow_missing=True,
        )
        if detail is None:
            break

        media_type = str(detail.get("media_type", "") or "").lower()
        num_episodes = int(detail.get("num_episodes") or 0)
        prequel_id = None
        parent_id = None
        alternative_id = None

        for relation in detail.get("related_anime", []):
            node = relation.get("node", {})
            related_id = node.get("id")
            if related_id is None:
                continue
            relation_type = relation.get("relation_type")
            if relation_type == "prequel":
                prequel_id = int(related_id)
            elif relation_type == "parent_story":
                parent_id = int(related_id)
            elif relation_type == "alternative_version":
                alternative_id = int(related_id)

        relationship_chain.append(
            {
                "id": current_id,
                "media_type": media_type,
                "parent": parent_id,
                "alternative": alternative_id,
                "num_episodes": num_episodes,
            }
        )

        if len(relationship_chain) == 1 and media_type in {"music", "special"}:
            break
        if (
            len(relationship_chain) > 1
            and relationship_chain[0]["media_type"] == "tv"
            and media_type == "tv"
        ):
            break
        if prequel_id is None:
            break

        current_id = prequel_id

    if not relationship_chain:
        return ""

    first = relationship_chain[0]
    last_tv_index = -1
    for index, entry in enumerate(relationship_chain):
        if entry["media_type"] == "tv":
            last_tv_index = index

    if first["media_type"] == "music":
        return "s1"
    if len(relationship_chain) == 1 and first["media_type"] == "tv":
        return "s1"
    if len(relationship_chain) == 1 and first["media_type"] == "movie" and first["parent"] is None:
        return "s1"
    if len(relationship_chain) == 1 and first["media_type"] == "ova" and first["parent"] is None and first["num_episodes"] >= 3:
        return "s1"
    if len(relationship_chain) == 1 and first["media_type"] == "ona" and first["num_episodes"] >= 6:
        return "s1"
    if len(relationship_chain) == 1 and first["num_episodes"] > 6:
        return "s1"
    if len(relationship_chain) > 1 and last_tv_index == 0:
        return "s1"
    if (
        len(relationship_chain) == 2
        and first["media_type"] == "ova"
        and relationship_chain[1]["media_type"] == "ova"
        and relationship_chain[1]["num_episodes"] <= 1
    ):
        return "s1"
    return "not_s1"


def refresh_relationships(
    paths: ProjectPaths,
    *,
    sleep_seconds: float = 0.0,
) -> dict:
    catalog_path = paths.resolve_anime_catalog_input()
    catalog_df = load_csv_optional(catalog_path)
    if catalog_df.empty:
        raise RuntimeError("Anime catalog is missing. Run refresh-anime-catalog first.")
    console.info(f"Refreshing relationship labels for catalog at {catalog_path}")

    existing_relationships_df = load_csv_optional(paths.resolve_anime_relationships_input())
    relationship_map = {}
    if not existing_relationships_df.empty:
        existing_relationships_df["anime_id"] = existing_relationships_df["anime_id"].astype(int)
        relationship_map.update(
            {
                int(row.anime_id): str(row.relationship)
                for row in existing_relationships_df.itertuples(index=False)
                if str(row.relationship).strip() and str(row.relationship).strip().lower() != "nan"
            }
        )

    if "relationship" in catalog_df.columns:
        for row in catalog_df.itertuples(index=False):
            relationship = getattr(row, "relationship", "")
            if str(relationship).strip() and str(relationship).strip().lower() != "nan":
                relationship_map[int(row.anime_id)] = str(relationship)

    client = MalClient.from_env(sleep_seconds=sleep_seconds)
    missing_ids = [
        int(anime_id)
        for anime_id in catalog_df["anime_id"].astype(int).tolist()
        if int(anime_id) not in relationship_map
    ]
    console.info(f"Relationship labels present={len(relationship_map)} | missing={len(missing_ids)}")

    with tqdm(total=len(missing_ids), desc="Classifying relationships", unit="anime", dynamic_ncols=True) as bar:
        for anime_id in missing_ids:
            relationship = _classify_relationship(client, anime_id)
            relationship_map[anime_id] = relationship
            bar.update(1)
            bar.set_postfix(anime_id=anime_id, relationship=relationship)
            if bar.n % 20 == 0:
                interim_df = pd.DataFrame(
                    [
                        {"anime_id": key, "relationship": value}
                        for key, value in sorted(relationship_map.items())
                    ],
                    columns=["anime_id", "relationship"],
                )
                write_csv(interim_df, paths.anime_relationships, paths.legacy_anime_relationships)

    relationships_df = pd.DataFrame(
        [
            {"anime_id": anime_id, "relationship": relationship}
            for anime_id, relationship in sorted(relationship_map.items())
        ],
        columns=["anime_id", "relationship"],
    )
    write_csv(relationships_df, paths.anime_relationships, paths.legacy_anime_relationships)

    merged_catalog_df = catalog_df.drop(columns=["relationship"], errors="ignore").merge(
        relationships_df,
        on="anime_id",
        how="left",
    )
    write_csv(merged_catalog_df, paths.anime_catalog, paths.legacy_anime_catalog)
    console.success(
        f"Saved relationship labels to {paths.anime_relationships} and merged them into {paths.anime_catalog}"
    )

    return {
        "relationship_path": str(paths.anime_relationships),
        "catalog_path": str(paths.anime_catalog),
        "relationship_count": int(len(relationships_df)),
        "new_relationships": int(len(missing_ids)),
    }


def build_user_genre_features(paths: ProjectPaths) -> dict:
    console.info("Building derived user genre features from interactions and the anime catalog")
    interactions_df = pd.read_csv(paths.resolve_user_interactions_input())
    interactions_df["anime_id"] = interactions_df["anime_id"].astype(str)
    catalog_df = pd.read_csv(paths.resolve_anime_catalog_input())
    catalog_df["anime_id"] = catalog_df["anime_id"].astype(str)

    feature_df = derive_user_genre_features(
        interactions_df[["username", "anime_id"]].copy(),
        catalog_df[["anime_id", "genres"]].copy(),
        output_path=paths.user_genre_features,
    )
    console.success(f"Wrote derived genre features to {paths.user_genre_features}")
    return {
        "feature_path": str(paths.user_genre_features),
        "feature_rows": int(len(feature_df)),
        "feature_users": int(feature_df["username"].nunique()) if not feature_df.empty else 0,
    }


def refresh_all(
    paths: ProjectPaths,
    *,
    username_pages: int = 300,
    years_back: int = 2,
    scrape_delay_seconds: float = 1.0,
    api_delay_seconds: float = 0.0,
    refresh_existing_catalog: bool = False,
    max_usernames: int = 5000,
    start_step: int = 1,
    training_config: TrainingConfig | None = None,
) -> dict:
    summary = {}
    steps = [
        ("scrape_usernames", "Scrape public MAL usernames", lambda: scrape_usernames(
            paths,
            num_pages=username_pages,
            delay_seconds=scrape_delay_seconds,
            max_usernames=max_usernames,
        ), f"pages={username_pages}, scrape_delay={scrape_delay_seconds}s, max_usernames={max_usernames}"),
        ("fetch_user_interactions", "Fetch user interaction data", lambda: fetch_user_interactions(
            paths,
            sleep_seconds=api_delay_seconds,
            refresh_existing_users=True,
        ), f"api_sleep={api_delay_seconds}s between MAL API requests"),
        ("refresh_anime_catalog", "Refresh anime catalog", lambda: refresh_anime_catalog(
            paths,
            years_back=years_back,
            refresh_existing=refresh_existing_catalog,
            sleep_seconds=api_delay_seconds,
        ), f"years_back={years_back}, refresh_existing={refresh_existing_catalog}"),
        ("refresh_relationships", "Refresh sequel and season relationships", lambda: refresh_relationships(
            paths,
            sleep_seconds=api_delay_seconds,
        ), None),
        ("train_model", "Train LightFM model", lambda: train_model(paths, training_config), None),
        ("build_atlas", "Build atlas coordinates", lambda: generate_atlas(paths), None),
    ]

    total_steps = len(steps)
    for index, (key, title, runner, detail) in enumerate(steps, start=1):
        if index < start_step:
            console.info(f"Skipping step {index}/{total_steps}: {title}")
            continue
        with console.step(title, index=index, total=total_steps, detail=detail):
            summary[key] = runner()
        console.summary(f"Step {index} Result", summary[key])

    return summary