"""Crawler integration tests with fake HTTP/browser and the real repository."""

from __future__ import annotations

import asyncio
import socket
from types import SimpleNamespace
from typing import Any

import pytest

from app.adapters.base import MediaCandidate, ParsedPage, SiteAdapter, VideoPageLink
from app.adapters.registry import AdapterRegistry
from app.database.models import Job, PageStatus
from app.errors import (
    AccessDeniedError,
    BrowserUnavailableError,
    MediaNotFoundError,
    NetworkError,
    NotFoundError,
    RobotsDeniedError,
)
from app.services.crawler import Crawler
from app.services.extractor import Extractor
from app.services.http_client import FetchedPage
from app.services.url_validator import NetworkGuard

PUBLIC_V4 = "93.184.216.34"


def _guard() -> NetworkGuard:
    async def resolver(host: str, port: int):
        return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (PUBLIC_V4, port))]

    return NetworkGuard(["example.com"], resolver=resolver)


class FakeHttp:
    def __init__(
        self,
        *,
        pages: dict[str, Any] | None = None,
        robots: bytes | BaseException | None = None,
    ) -> None:
        self.pages = dict(pages or {})
        self.robots = robots
        self.html_calls: list[str] = []
        self.robots_calls: list[str] = []

    async def fetch_html(self, url: str, *, max_bytes: int) -> FetchedPage:
        self.html_calls.append(url)
        outcome = self.pages.get(url)
        if isinstance(outcome, BaseException):
            raise outcome
        if outcome is None:
            raise AssertionError(f"unexpected page request: {url}")
        if isinstance(outcome, FetchedPage):
            return outcome
        return FetchedPage(
            requested_url=url,
            final_url=url,
            status=200,
            content_type="text/html",
            html=str(outcome),
            size=len(str(outcome)),
        )

    async def fetch_bytes(self, url: str, *, max_bytes: int) -> bytes:
        self.robots_calls.append(url)
        if isinstance(self.robots, BaseException):
            raise self.robots
        if self.robots is None:
            raise NotFoundError()
        return self.robots


class FakeBrowser:
    def __init__(self, rendered_html: str | None = None, error: BaseException | None = None) -> None:
        self.rendered_html = rendered_html
        self.error = error
        self.calls: list[str] = []

    async def render(self, url: str, adapter: SiteAdapter) -> Any:
        self.calls.append(url)
        if self.error is not None:
            raise self.error
        return SimpleNamespace(final_url=url, html=self.rendered_html or "")

    async def close(self) -> None:
        return None


class MappingAdapter(SiteAdapter):
    name = "test_site"

    def __init__(self, builder) -> None:
        self.builder = builder
        self.parsed_urls: list[str] = []

    def supports(self, url: str) -> bool:
        return True

    def parse(self, html: str, url: str) -> ParsedPage:
        self.parsed_urls.append(url)
        return self.builder(url, html, self)


def _media(
    url: str,
    *,
    page: str = "https://example.com/list",
    title: str = "Video",
    caption: str | None = None,
) -> MediaCandidate:
    return MediaCandidate(
        page_url=page,
        source_url=url,
        title=title,
        caption=caption or title,
        media_url=url,
        media_type="video/mp4",
    )


def _page(url: str, **kwargs: Any) -> ParsedPage:
    return ParsedPage(url=url, title="Test", caption="Test", **kwargs)


def _crawler(
    settings,
    repository,
    http: FakeHttp,
    builder,
    browser: FakeBrowser | None = None,
) -> tuple[Crawler, MappingAdapter, FakeBrowser]:
    adapter = MappingAdapter(builder)
    registry = AdapterRegistry(site_adapters=[adapter])
    browser = browser or FakeBrowser(error=BrowserUnavailableError("not installed"))
    guard = _guard()
    crawler = Crawler(
        settings,
        guard,
        http,
        browser,
        Extractor(guard),
        registry,
        repository,
    )
    return crawler, adapter, browser


async def _job(repository, source: str = "https://example.com/list") -> Job:
    job = Job.create(source, owner_id=1, owner_chat_id=2)
    await repository.create_job(job)
    return job


async def _noop() -> None:
    return None


@pytest.mark.asyncio
async def test_crawl_follows_explicit_pagination_and_stores_videos(settings, repository) -> None:
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                videos=[
                    _media("https://example.com/v1.mp4", title="One"),
                    _media("https://example.com/v2.mp4", title="Two"),
                ],
                next_page_url="https://example.com/page2",
                next_page_strategy="rel_next",
            ),
            "https://example.com/page2": _page(
                "https://example.com/page2",
                videos=[_media("https://example.com/v3.mp4", title="Three", page="https://example.com/page2")],
            ),
        }
    )
    crawler, _, _ = _crawler(
        settings,
        repository,
        http,
        lambda url, html, adapter: http.pages[url],
    )
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)

    assert result.pagination_pages == 2
    assert result.videos_found == 3
    assert result.stop_reason == "no_next_page"
    videos = await repository.list_videos(job.id)
    assert {video.source_url for video in videos} == {
        "https://example.com/v1.mp4",
        "https://example.com/v2.mp4",
        "https://example.com/v3.mp4",
    }
    pages = await repository.list_pages(job.id)
    assert all(page.status == PageStatus.PROCESSED for page in pages)
    assert len(pages) == 2


@pytest.mark.asyncio
async def test_crawl_counts_duplicate_video_urls(settings, repository) -> None:
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                videos=[_media("https://example.com/v1.mp4")],
                next_page_url="https://example.com/page2",
            ),
            "https://example.com/page2": _page(
                "https://example.com/page2",
                videos=[_media("https://example.com/v1.mp4")],
            ),
        }
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.videos_found == 1
    assert result.duplicate_videos == 1
    assert len(await repository.list_videos(job.id)) == 1


@pytest.mark.asyncio
async def test_crawl_processes_one_level_of_video_detail_pages(settings, repository) -> None:
    detail_url = "https://example.com/watch/42"
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                video_pages=[VideoPageLink(url=detail_url, title="Detail title")],
            ),
            detail_url: _page(
                detail_url,
                videos=[
                    _media(
                        "https://example.com/detail.mp4",
                        page=detail_url,
                        title="Detail title",
                    )
                ],
            ),
        }
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.detail_pages == 1
    assert result.videos_found == 1
    videos = await repository.list_videos(job.id)
    assert videos[0].page_url == detail_url
    assert videos[0].title == "Detail title"


@pytest.mark.asyncio
async def test_detail_page_without_direct_media_is_not_stored(settings, repository) -> None:
    detail_url = "https://example.com/watch/no-direct-media"
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                video_pages=[VideoPageLink(url=detail_url)],
            ),
            detail_url: _page(detail_url),
        }
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    with pytest.raises(MediaNotFoundError):
        await crawler.crawl(job, asyncio.Event(), _noop)
    assert await repository.list_videos(job.id) == []


@pytest.mark.asyncio
async def test_crawl_enforces_max_videos(settings, repository) -> None:
    settings.max_videos_per_job = 1
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                videos=[
                    _media("https://example.com/v1.mp4"),
                    _media("https://example.com/v2.mp4"),
                ],
                next_page_url="https://example.com/page2",
            )
        }
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.videos_found == 1
    assert result.stop_reason == "max_videos"
    assert len(await repository.list_videos(job.id)) == 1


@pytest.mark.asyncio
async def test_crawl_enforces_max_pages(settings, repository) -> None:
    settings.max_pages_per_job = 1
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                videos=[_media("https://example.com/v1.mp4")],
                next_page_url="https://example.com/page2",
            )
        }
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.pagination_pages == 1
    assert result.stop_reason == "max_pages"
    assert "https://example.com/page2" not in http.html_calls


@pytest.mark.asyncio
async def test_direct_media_source_url_needs_no_page_fetch(settings, repository) -> None:
    http = FakeHttp()
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: _page(url))
    job = await _job(repository, "https://example.com/media/My-Cool_Video.mp4")

    result = await crawler.crawl(job, asyncio.Event(), _noop)

    assert result.stop_reason == "direct_media"
    assert result.videos_found == 1
    assert http.html_calls == []
    video = (await repository.list_videos(job.id))[0]
    assert video.media_url == "https://example.com/media/My-Cool_Video.mp4"
    assert video.title == "My Cool Video"


@pytest.mark.asyncio
async def test_robots_disallow_stops_the_crawl(settings, repository) -> None:
    settings.respect_robots_txt = True
    http = FakeHttp(
        pages={"https://example.com/list": _page("https://example.com/list")},
        robots=b"User-agent: *\nDisallow: /\n",
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: _page(url))
    job = await _job(repository)

    with pytest.raises(RobotsDeniedError):
        await crawler.crawl(job, asyncio.Event(), _noop)
    assert http.html_calls == []


@pytest.mark.asyncio
async def test_robots_not_found_allows_crawl(settings, repository) -> None:
    settings.respect_robots_txt = True
    http = FakeHttp(
        pages={"https://example.com/list": _page("https://example.com/list", videos=[_media("https://example.com/v.mp4")])},
        robots=None,
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.videos_found == 1
    assert http.robots_calls == ["https://example.com/robots.txt"]


@pytest.mark.asyncio
async def test_access_denied_on_first_page_is_propagated(settings, repository) -> None:
    http = FakeHttp(pages={"https://example.com/list": AccessDeniedError(403)})
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: _page(url))
    job = await _job(repository)

    with pytest.raises(AccessDeniedError):
        await crawler.crawl(job, asyncio.Event(), _noop)


@pytest.mark.asyncio
async def test_pagination_error_keeps_already_discovered_videos(settings, repository) -> None:
    settings.retry_count = 0
    http = FakeHttp(
        pages={
            "https://example.com/list": _page(
                "https://example.com/list",
                videos=[_media("https://example.com/v1.mp4")],
                next_page_url="https://example.com/page2",
            ),
            "https://example.com/page2": NetworkError("down"),
        }
    )
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: http.pages[url])
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.videos_found == 1
    assert result.stop_reason == "permanent_page_error"
    failed_pages = [p for p in await repository.list_pages(job.id) if p.status == PageStatus.FAILED]
    assert len(failed_pages) == 1


@pytest.mark.asyncio
async def test_crawl_raises_when_no_video_is_found(settings, repository) -> None:
    http = FakeHttp(pages={"https://example.com/list": _page("https://example.com/list")})
    crawler, _, _ = _crawler(settings, repository, http, lambda url, html, adapter: _page(url))
    job = await _job(repository)

    with pytest.raises(MediaNotFoundError):
        await crawler.crawl(job, asyncio.Event(), _noop)


@pytest.mark.asyncio
async def test_browser_fallback_is_used_when_http_has_no_media(settings, repository) -> None:
    settings.browser_fallback_enabled = True
    http = FakeHttp(pages={"https://example.com/list": "<html>empty shell</html>"})

    def builder(url: str, html: str, adapter: MappingAdapter) -> ParsedPage:
        if html == "rendered":
            return _page(url, videos=[_media("https://example.com/v.mp4")])
        return _page(url)

    browser = FakeBrowser(rendered_html="rendered")
    crawler, _, browser = _crawler(settings, repository, http, builder, browser=browser)
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.videos_found == 1
    assert browser.calls == ["https://example.com/list"]
    pages = await repository.list_pages(job.id)
    assert pages[0].status == PageStatus.PROCESSED


@pytest.mark.asyncio
async def test_browser_fallback_unavailable_does_not_crash_http_result(settings, repository) -> None:
    settings.browser_fallback_enabled = True
    http = FakeHttp(pages={"https://example.com/list": "<html>empty shell</html>"})
    browser = FakeBrowser(error=BrowserUnavailableError("not installed"))
    crawler, _, _ = _crawler(
        settings, repository, http, lambda url, html, adapter: _page(url), browser=browser
    )
    job = await _job(repository)

    with pytest.raises(MediaNotFoundError):
        await crawler.crawl(job, asyncio.Event(), _noop)
    assert browser.calls == ["https://example.com/list"]


@pytest.mark.asyncio
async def test_redirect_is_tracked_as_skipped_page(settings, repository) -> None:
    redirected = FetchedPage(
        requested_url="https://example.com/list",
        final_url="https://example.com/final",
        status=200,
        content_type="text/html",
        html="final",
        size=5,
    )
    http = FakeHttp(pages={"https://example.com/list": redirected})
    crawler, _, _ = _crawler(
        settings,
        repository,
        http,
        lambda url, html, adapter: _page(url, videos=[_media("https://example.com/v.mp4", page=url)]),
    )
    job = await _job(repository)

    result = await crawler.crawl(job, asyncio.Event(), _noop)
    assert result.videos_found == 1
    pages = await repository.list_pages(job.id)
    by_url = {page.page_url: page for page in pages}
    assert by_url["https://example.com/list"].status == PageStatus.SKIPPED
    assert by_url["https://example.com/final"].status == PageStatus.PROCESSED