"""
scrape_subtitles.py
Navega el curso con Playwright, intercepta peticiones VTT del player
Vidalytics y descarga los subtitulos detectando idioma por CONTENIDO.

Correcciones v3:
- NO hace clear() de la lista de URLs - usa snapshot before/after para
  aislar las URLs de cada leccion sin perder las que llegaron durante la
  transicion entre lecciones.
- Espera activa: en vez de tiempo fijo, espera hasta detectar 2+ VTT nuevas
  o cumplir el timeout maximo.
- Saltar lecciones ya completas con navegacion rapida.

Idiomas: es (Español) | pt (Português) | en (English)
Salida:  public/subtitles/{num:02d}-{lang}.vtt
"""
import asyncio
import json
import re
import sys
import time
import urllib.request
from pathlib import Path

from playwright.async_api import async_playwright

sys.stdout.reconfigure(encoding="utf-8")

SCRIPTS_DIR    = Path(__file__).parent
BASE_DIR       = SCRIPTS_DIR.parent
COOKIES_FILE   = SCRIPTS_DIR / ".course_cookies.json"
STRUCTURE_FILE = SCRIPTS_DIR / "course_structure.json"
CONTENT_FILE   = SCRIPTS_DIR / "course_content.json"
MEMBERS_URL    = "https://kaleanders.clickfunnels.com/members-raio"
SUBS_DIR       = BASE_DIR / "public" / "subtitles"
SUBS_DIR.mkdir(parents=True, exist_ok=True)

MAX_WAIT_SECS   = 18   # max seconds to wait for VTT URLs per lesson
MIN_VTTS_NEEDED = 2    # stop waiting once this many new VTTs detected
POLL_MS         = 400  # polling interval while waiting

# ── Language detection ─────────────────────────────────────────────────────

PT_WORDS = re.compile(
    r'\b(você|não|olá|tudo|obrigado|obrigada|então|português|bem-vindo|bem-vindos|'
    r'esse|essa|isso|nosso|nossa|muito|porque|quando|onde|sempre|nunca|agora|'
    r'depois|antes|durante|enquanto|quero|posso|estou|este|esta)\b',
    re.I | re.U,
)
ES_WORDS = re.compile(
    r'\b(hola|usted|también|sería|tengo|tiene|pero|porque|cuando|donde|muy|'
    r'aquí|así|español|siempre|nunca|ahora|después|antes|durante|mientras|'
    r'vamos|quiero|puedo|está|esto|eso|señor|señora|gracias|buenos)\b',
    re.I | re.U,
)
EN_WORDS = re.compile(
    r'\b(the|and|that|this|with|have|from|they|would|could|should|about|'
    r'there|their|because|before|after|during|while|always|never|going|'
    r'hello|thank|please|sorry|really|actually|exactly)\b',
    re.I,
)


def detect_language(text: str) -> str:
    body = "\n".join(text.splitlines()[10:])[:4000]
    pt = len(PT_WORDS.findall(body))
    es = len(ES_WORDS.findall(body))
    en = len(EN_WORDS.findall(body))
    scores = {"pt": pt, "es": es, "en": en}
    top, top_score = max(scores.items(), key=lambda x: x[1])
    if top_score < 2:
        acc_es = len(re.findall(r'[áéíóúñ¿¡]', body, re.I))
        acc_pt = len(re.findall(r'[ãõâêîôûç]', body, re.I))
        acc = {"es": acc_es, "pt": acc_pt, "en": 0}
        top = max(acc, key=acc.get)
    return top


def log(msg: str, level: str = "INFO") -> None:
    icons = {"INFO": "   ", "OK": "[OK]", "SKIP": "[--]", "ERROR": "[!!]", "HEAD": "\n==="}
    print(f"{icons.get(level, '   ')} {msg}", flush=True)


def download_vtt(url: str) -> bytes | None:
    try:
        req = urllib.request.Request(url, headers={
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
            "Referer": "https://kaleanders.clickfunnels.com/",
        })
        with urllib.request.urlopen(req, timeout=20) as r:
            data = r.read()
        return data if len(data) > 80 else None
    except Exception as e:
        log(f"Error: {url[:60]}: {e}", "ERROR")
        return None


def save_vtt(num: int, data: bytes, content_entry: dict) -> str | None:
    try:
        text = data.decode("utf-8", errors="ignore")
    except Exception:
        return None
    if "WEBVTT" not in text[:20]:
        return None
    lang = detect_language(text)
    dest = SUBS_DIR / f"{num:02d}-{lang}.vtt"
    if dest.exists() and dest.stat().st_size >= len(data):
        return lang  # already have equal or better copy
    dest.write_bytes(data)
    if lang in ("es", "en"):
        content_entry[f"subtitle_{lang}"] = f"subtitles/{num:02d}-{lang}.vtt"
    return lang


async def click_next(page) -> bool:
    for sel in [
        "a[href='#next-lesson']", "#next-lesson",
        "a:has-text('Siguiente')", "a:has-text('Next')",
        "button:has-text('Siguiente')", "button:has-text('Next')",
    ]:
        try:
            btn = await page.query_selector(sel)
            if btn and await btn.is_visible():
                await btn.click()
                return True
        except Exception:
            pass
    return False


async def wait_for_vtts(vtt_log: list[str], seen_before: set[str]) -> list[str]:
    """
    Wait until MIN_VTTS_NEEDED new URLs appear (not in seen_before),
    or until MAX_WAIT_SECS seconds elapse.
    Returns the list of new URLs found.
    """
    deadline = time.monotonic() + MAX_WAIT_SECS
    while time.monotonic() < deadline:
        new = [u for u in vtt_log if u not in seen_before]
        if len(new) >= MIN_VTTS_NEEDED:
            # One extra short wait so remaining tracks can arrive
            await asyncio.sleep(1.5)
            break
        await asyncio.sleep(POLL_MS / 1000)
    return [u for u in vtt_log if u not in seen_before]


async def main():
    log("RAIO Subtitle Scraper v3", "HEAD")

    lessons        = json.loads(STRUCTURE_FILE.read_text(encoding="utf-8"))
    cookies        = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
    content        = json.loads(CONTENT_FILE.read_text(encoding="utf-8"))
    content_by_num = {l["num"]: l for l in content}

    already_done: set[int] = set()
    for l in content:
        num = l["num"]
        if (SUBS_DIR / f"{num:02d}-es.vtt").exists() and (SUBS_DIR / f"{num:02d}-en.vtt").exists():
            already_done.add(num)

    log(f"Lecciones ya completas (ES+EN): {len(already_done)}")
    log(f"Lecciones a procesar: {len(lessons) - len(already_done)}")

    # Global ordered log of all VTT URLs seen (never cleared)
    vtt_log: list[str] = []

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=True)
        ctx = await browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/124 Safari/537.36",
            viewport={"width": 1280, "height": 800},
        )
        await ctx.add_cookies(cookies)
        page = await ctx.new_page()

        def on_request(request):
            url = request.url
            lo = url.lower()
            if (".vtt" in lo or "subtitle" in lo or "caption" in lo) and url not in vtt_log:
                vtt_log.append(url)

        def on_response(response):
            url = response.url
            lo = url.lower()
            if (".vtt" in lo or "subtitle" in lo or "caption" in lo) and url not in vtt_log:
                vtt_log.append(url)

        page.on("request", on_request)
        page.on("response", on_response)

        log("Cargando página del curso...")
        await page.goto(MEMBERS_URL, wait_until="domcontentloaded", timeout=45000)

        found_total = 0
        skipped     = 0

        for i, lesson in enumerate(lessons):
            num          = lesson["num"]
            seen_before  = set(vtt_log)  # snapshot: URLs known BEFORE this lesson loaded

            if num in already_done:
                log(f"[{num:02d}] Ya tiene ES+EN — saltando", "SKIP")
                skipped += 1
                if i < len(lessons) - 1:
                    await click_next(page)
                    await asyncio.sleep(2)  # minimal wait — next iteration will wait properly
                continue

            # Wait actively for new VTT URLs (up to MAX_WAIT_SECS)
            new_urls = await wait_for_vtts(vtt_log, seen_before)

            # Also check <track> elements in DOM (Vidalytics sometimes uses those)
            try:
                track_srcs = await page.evaluate(
                    "() => Array.from(document.querySelectorAll('track')).map(t => t.src || t.getAttribute('src') || '')"
                ) or []
                for src in track_srcs:
                    if src and src not in vtt_log:
                        vtt_log.append(src)
                new_urls = [u for u in vtt_log if u not in seen_before]
            except Exception:
                pass

            lesson_content = content_by_num.get(num, {})
            langs_saved: set[str] = set()

            if new_urls:
                log(f"[{num:02d}] {len(new_urls)} VTT URL(s)")
                for url in new_urls:
                    data = download_vtt(url)
                    if not data:
                        continue
                    lang = save_vtt(num, data, lesson_content)
                    if lang and lang not in langs_saved:
                        langs_saved.add(lang)
                        log(f"   Guardado: {num:02d}-{lang}.vtt ({len(data):,} bytes)", "OK")
                    found_total += 1
            else:
                log(f"[{num:02d}] Sin subtítulos (timeout {MAX_WAIT_SECS}s)", "SKIP")

            if i < len(lessons) - 1:
                clicked = await click_next(page)
                if not clicked:
                    log("Sin botón siguiente. Fin.", "OK")
                    break
                # Do NOT sleep long here — next iteration's wait_for_vtts handles timing
                await asyncio.sleep(1)

        page.remove_listener("request", on_request)
        page.remove_listener("response", on_response)
        await browser.close()

    CONTENT_FILE.write_text(
        json.dumps(content, ensure_ascii=False, indent=2),
        encoding="utf-8",
    )

    total_es = len(list(SUBS_DIR.glob("*-es.vtt")))
    total_en = len(list(SUBS_DIR.glob("*-en.vtt")))
    total_pt = len(list(SUBS_DIR.glob("*-pt.vtt")))

    log(f"VTTs procesados: {found_total} | Saltadas: {skipped}", "OK")
    log(f"Totales → ES: {total_es} | EN: {total_en} | PT: {total_pt}", "OK")

    if found_total > 0:
        log("Actualizando DB...", "INFO")
        import subprocess
        subprocess.run(["python", str(SCRIPTS_DIR / "build_course_map.py")], check=True)
        subprocess.run(
            ["php", "artisan", "raio:import", "--force"],
            cwd=str(BASE_DIR),
            check=True,
        )
        log("Listo.", "OK")


if __name__ == "__main__":
    asyncio.run(main())