#!/usr/bin/env python3
"""
Site Cloner v2 — smarter resource extraction
"""
import os, re, time, json, hashlib, urllib.request, urllib.parse
from pathlib import Path

TARGET_URL  = "https://minutedrama.com/en"
OUTPUT_ROOT = Path("/www/wwwroot/www.q22.pro/dev/minutedrama")

MAX_RETRY    = 3
RETRY_DELAY  = 2
TIMEOUT      = 30
MAX_VIDEO_MB = 50

UA_LIST = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/17.0 Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0 Safari/537.36",
]

DIRS = {
    'css':    OUTPUT_ROOT / 'static/css',
    'js':     OUTPUT_ROOT / 'static/js',
    'images': OUTPUT_ROOT / 'static/images',
    'fonts':  OUTPUT_ROOT / 'static/fonts',
    'flash':  OUTPUT_ROOT / 'static/flash',
}
for d in DIRS.values():
    d.mkdir(parents=True, exist_ok=True)

EXT_MAP = {
    **{e: 'css'    for e in ['css']},
    **{e: 'js'     for e in ['js', 'mjs']},
    **{e: 'images' for e in ['jpg','jpeg','png','gif','webp','svg','ico','bmp','avif']},
    **{e: 'fonts'  for e in ['woff','woff2','ttf','eot','otf']},
    **{e: 'flash'  for e in ['mp4','webm','ogv','ogg','mp3','wav','swf','flv']},
}

IMAGE_EXTS = {'jpg','jpeg','png','gif','webp','svg','ico','bmp','avif'}
RESOURCE_EXTS = IMAGE_EXTS | {'css','js','mjs','woff','woff2','ttf','eot','otf','mp4','webm'}
RESOURCE_PATHS = ['/css2', '/css']

url_to_local  = {}
failed_urls   = []
skipped_large = []
queue         = []
queued_set    = set()
parsed_files  = set()

def normalize_url(url, base):
    url = url.strip().strip('"\'')
    if not url or url.startswith('data:') or url.startswith('javascript:') or url.startswith('#'):
        return None
    if not (url.startswith('http://') or url.startswith('https://') or url.startswith('//') or url.startswith('/')):
        return None
    abs_url = urllib.parse.urljoin(base, url)
    parsed  = urllib.parse.urlparse(abs_url)
    if parsed.scheme not in ('http', 'https'):
        return None
    return urllib.parse.urlunparse(parsed._replace(fragment=''))

def has_resource_ext(url):
    path = urllib.parse.urlparse(url).path
    ext  = path.rsplit('.', 1)[-1].lower().split('?')[0] if '.' in path else ''
    if ext in RESOURCE_EXTS:
        return True
    for rp in RESOURCE_PATHS:
        if path == rp or path.startswith(rp + '?'):
            return True
    return False

def get_category(url):
    path = urllib.parse.urlparse(url).path
    ext  = path.rsplit('.', 1)[-1].lower().split('?')[0] if '.' in path else ''
    return EXT_MAP.get(ext, 'images')

def safe_filename(url):
    path = urllib.parse.urlparse(url).path
    name = path.rsplit('/', 1)[-1].split('?')[0]
    name = re.sub(r'[<>:"/\\|?*]', '_', name)
    if len(name) > 100:
        ext = name.rsplit('.', 1)[-1] if '.' in name else ''
        name = hashlib.md5(name.encode()).hexdigest()[:16] + (f'.{ext}' if ext else '')
    return name or hashlib.md5(url.encode()).hexdigest()[:10]

def unique_dest(cat, fname):
    dest = DIRS[cat] / fname
    if not dest.exists():
        return dest
    stem = fname.rsplit('.', 1)[0] if '.' in fname else fname
    suf  = ('.' + fname.rsplit('.', 1)[1]) if '.' in fname else ''
    for i in range(2, 9999):
        c = DIRS[cat] / f"{stem}_{i}{suf}"
        if not c.exists():
            return c

def local_rel_path(p: Path) -> str:
    return str(p.relative_to(OUTPUT_ROOT)).replace('\\', '/')

def download_url(url) -> bytes | None:
    for attempt in range(MAX_RETRY):
        ua = UA_LIST[attempt % len(UA_LIST)]
        try:
            req = urllib.request.Request(url, headers={
                'User-Agent': ua,
                'Referer':    TARGET_URL,
                'Accept':     '*/*',
            })
            with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
                cl = r.headers.get('Content-Length')
                if cl and int(cl) > MAX_VIDEO_MB * 1024 * 1024:
                    if get_category(url) == 'flash':
                        print(f"  ⏭  Skip large({int(cl)//1024//1024}MB): {url.split('?')[0]}")
                        skipped_large.append((url, int(cl)))
                        return None
                return r.read()
        except UnicodeEncodeError:
            print(f"  ⚠️  Invalid URL (skipping): {url[:80]}...")
            return None
        except urllib.error.HTTPError as e:
            if e.code in (404, 403):
                print(f"  ⚠️  {e.code} skip: {url.split('?')[0]}")
                failed_urls.append(url)
                return None
            print(f"  ⚠️  HTTP {e.code} [{url.split('?')[0]}] attempt {attempt+1}")
            if attempt < MAX_RETRY - 1:
                time.sleep(RETRY_DELAY * (attempt + 1))
        except Exception as e:
            print(f"  ⚠️  {e.__class__.__name__} [{url.split('?')[0][:50]}] attempt {attempt+1}")
            if attempt < MAX_RETRY - 1:
                time.sleep(RETRY_DELAY * (attempt + 1))
    failed_urls.append(url)
    return None

def enqueue(url):
    if url and url not in queued_set:
        queued_set.add(url)
        queue.append(url)

def process_one(url):
    if url in url_to_local:
        return url_to_local[url]
    cat   = get_category(url)
    fname = safe_filename(url)
    dest  = unique_dest(cat, fname)
    data  = download_url(url)
    if not data:
        return None
    dest.write_bytes(data)
    lp = local_rel_path(dest)
    url_to_local[url] = lp
    print(f"  ✅ {lp}  ←  {url.split('?')[0]}")
    return lp

# ============== URL Extraction ==============

def extract_urls_from_html(html, base):
    """Extract resource URLs, filtering to only real resources"""
    found = set()

    # 1. Extract <script src="..."> BEFORE stripping script content
    for m in re.finditer(r'<script[^>]+src=["\']([^"\']+)["\']', html, re.I):
        u = normalize_url(m.group(1), base)
        if u and has_resource_ext(u): found.add(u)

    # 2. Strip inline script content to avoid false matches
    html_no_scripts = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.S | re.I)

    # 3. <link href="..."> (CSS, favicons, fonts, preload)
    for m in re.finditer(r'<link[^>]+href=["\']([^"\']+)["\']', html_no_scripts, re.I):
        u = normalize_url(m.group(1), base)
        if u and has_resource_ext(u): found.add(u)

    # 4. <img src="...">, <source src="...">, <video poster="...">, etc.
    for tag in ['img', 'source', 'video', 'audio', 'embed', 'object', 'track']:
        for m in re.finditer(rf'<{tag}[^>]+(?:src|poster)=["\']([^"\']+)["\']', html_no_scripts, re.I):
            u = normalize_url(m.group(1), base)
            if u and has_resource_ext(u): found.add(u)

    # 5. data-src, data-lazy, data-original, data-bg (lazy loading)
    for m in re.finditer(r'(?:data-src|data-lazy|data-original|data-bg|data-url)\s*=\s*["\']([^"\']+)["\']', html_no_scripts, re.I):
        u = normalize_url(m.group(1), base)
        if u and has_resource_ext(u): found.add(u)

    # 6. srcset
    for m in re.finditer(r'srcset\s*=\s*["\']([^"\']+)["\']', html_no_scripts, re.I):
        for seg in m.group(1).split(','):
            tok = seg.strip().split()
            if tok:
                u = normalize_url(tok[0], base)
                if u and has_resource_ext(u): found.add(u)

    # 7. CSS url() references inline
    for m in re.finditer(r'url\s*\(\s*["\']?([^"\')\s]+)["\']?\s*\)', html_no_scripts, re.I):
        u = normalize_url(m.group(1), base)
        if u and has_resource_ext(u): found.add(u)

    return list(found)

def extract_urls_from_css(css_text, base):
    found = set()
    for m in re.finditer(r'url\s*\(\s*["\']?([^"\')\s]+)["\']?\s*\)', css_text, re.I):
        u = normalize_url(m.group(1), base)
        if u and has_resource_ext(u): found.add(u)
    for m in re.finditer(r'@import\s+["\']([^"\']+)["\']', css_text, re.I):
        u = normalize_url(m.group(1), base)
        if u and has_resource_ext(u): found.add(u)
    return list(found)

def replace_paths_html(html: str, base_url: str) -> str:
    """Replace original URLs with local paths in HTML.
    Only replaces resources from the same domain (minutedrama.com).
    External CDN resources (Google Fonts, etc.) are kept as-is.
    """
    base_parsed = urllib.parse.urlparse(base_url)
    base_domain = f"{base_parsed.scheme}://{base_parsed.netloc}"

    # Only build mappings for same-domain resources
    replacements = []
    for orig_url, lp in url_to_local.items():
        parsed = urllib.parse.urlparse(orig_url)
        # Only replace if it's from the same domain
        if parsed.netloc == base_parsed.netloc:
            # Full URL version (e.g. https://minutedrama.com/path/to/file.js)
            replacements.append((orig_url, lp))
            # Relative path version (e.g. /path/to/file.js)
            if parsed.path:
                replacements.append((parsed.path, lp))
                if parsed.query:
                    replacements.append((f"{parsed.path}?{parsed.query}", lp))

    # Sort longest first to avoid partial substring matches
    replacements.sort(key=lambda x: -len(x[0]))

    for orig, lp in replacements:
        html = html.replace(orig, lp)

    return html

# ============================  MAIN  ============================
print(f"\n{'='*60}")
print(f"Target: {TARGET_URL}")
print(f"Output: {OUTPUT_ROOT}")
print(f"{'='*60}\n")

print("Fetching main page...")
raw_bytes = download_url(TARGET_URL)
if not raw_bytes:
    print("Failed to fetch main page"); exit(1)

import chardet
enc = chardet.detect(raw_bytes).get('encoding') or 'utf-8'
html_content = raw_bytes.decode(enc, errors='replace')
print(f"Encoding: {enc}, Length: {len(html_content):,} chars\n")

# Extract and enqueue resources
print("Parsing resource URLs...")
for u in extract_urls_from_html(html_content, TARGET_URL):
    enqueue(u)
print(f"Found {len(queue)} resources\n")

# Batch download (queue grows dynamically from CSS scanning)
print("Downloading resources...\n")
i = 0
while i < len(queue):
    url = queue[i]; i += 1
    lp  = process_one(url)
    if not lp or url in parsed_files:
        continue
    parsed_files.add(url)
    if lp.startswith('static/css'):
        css_text = (OUTPUT_ROOT / lp).read_text(errors='replace')
        added = sum(1 for u in extract_urls_from_css(css_text, url) if u not in queued_set and (enqueue(u) or True))
        if added: print(f"  📎 CSS scan +{added}: {lp}")
    if lp.startswith('static/js'):
        js_text = (OUTPUT_ROOT / lp).read_text(errors='replace')
        for m in re.finditer(r"""['"](https?://[^'"\s]*(?:\.(?:jpg|jpeg|png|gif|webp|svg|mp4|woff2?|css|js))[^'"\s]*)['"]""", js_text, re.I):
            u = normalize_url(m.group(1), TARGET_URL)
            if u and has_resource_ext(u): enqueue(u)

print(f"\nDone — Success:{len(url_to_local)} Failed:{len(failed_urls)} Skipped:{len(skipped_large)}\n")

# Replace paths in HTML
print("Replacing HTML paths...")
html_fixed = replace_paths_html(html_content, TARGET_URL)

print("Replacing CSS paths...")
for css_file in DIRS['css'].glob('*.css'):
    try:
        orig  = css_file.read_text(errors='replace')
        fixed = replace_paths_html(orig, str(css_file.resolve()))
        css_file.write_text(fixed, encoding='utf-8')
    except Exception as e:
        print(f"  ⚠️ {css_file.name}: {e}")

# Write index.html
(OUTPUT_ROOT / 'index.html').write_text(html_fixed, encoding='utf-8')
print("  ✅ index.html written\n")

# Resource manifest
lines = [
    f"# Resources — {TARGET_URL}",
    f"# Success: {len(url_to_local)}  Failed: {len(failed_urls)}  Skipped: {len(skipped_large)}",
    "", "## Success",
]
for url, lp in sorted(url_to_local.items(), key=lambda x: x[1]):
    sz = (OUTPUT_ROOT / lp).stat().st_size // 1024
    lines.append(f"  [{sz:>5}KB]  {lp}  ←  {url.split('?')[0]}")
if failed_urls:
    lines += ["", "## Failed"]
    lines += [f"  {u}" for u in failed_urls]
(OUTPUT_ROOT / 'resources.txt').write_text('\n'.join(lines), encoding='utf-8')

# Stats
print("Stats:")
total_f = total_s = 0
for cat, d in DIRS.items():
    files = [f for f in d.glob('*') if f.is_file()]
    size  = sum(f.stat().st_size for f in files)
    if files: print(f"  {cat:<8}: {len(files):>3} files  {size//1024:>6} KB")
    total_f += len(files); total_s += size
print(f"  {'Total':<8}: {total_f:>3} files  {total_s//1024:>6} KB\n")

print("Done! Open index.html in browser to check.")