#!/usr/bin/env python3
import os, re, time, json, hashlib, urllib.request, urllib.parse, chardet
from pathlib import Path

TARGET_URL = "https://media69.xyz/"
SITE_NAME  = "media69.xyz"
OUTPUT_ROOT = Path("/www/wwwroot/www.q22.pro/dev/siteclone") / SITE_NAME
RENDERED_HTML_PATH = "/tmp/rendered.html"

MAX_RETRY   = 3
RETRY_DELAY = 2
TIMEOUT     = 20
MAX_VIDEO_MB = 50

UA_LIST = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/17.0 Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/122.0 Safari/537.36",
]

DIRS = {
    'css':    OUTPUT_ROOT / 'static/css',
    'js':     OUTPUT_ROOT / 'static/js',
    'images': OUTPUT_ROOT / 'static/images',
    'fonts':  OUTPUT_ROOT / 'static/fonts',
    'flash':  OUTPUT_ROOT / 'static/flash',
}
for d in DIRS.values():
    d.mkdir(parents=True, exist_ok=True)

EXT_MAP = {
    **{e: 'css'    for e in ['css']},
    **{e: 'js'     for e in ['js', 'mjs']},
    **{e: 'images' for e in ['jpg','jpeg','png','gif','webp','svg','ico','bmp','avif']},
    **{e: 'fonts'  for e in ['woff','woff2','ttf','eot','otf']},
    **{e: 'flash'  for e in ['mp4','webm','ogv','ogg','mp3','wav','swf','flv']},
}

url_to_local  = {}
failed_urls   = []
skipped_large = []
queue         = []
queued_set    = set()
parsed_files  = set()

def normalize_url(url, base):
    url = url.strip().strip('"\'')
    if not url or url.startswith('data:') or url.startswith('javascript:') or url.startswith('#'):
        return None
    abs_url = urllib.parse.urljoin(base, url)
    parsed  = urllib.parse.urlparse(abs_url)
    if parsed.scheme not in ('http', 'https'):
        return None
    return urllib.parse.urlunparse(parsed._replace(fragment=''))

def get_category(url):
    path = urllib.parse.urlparse(url).path
    ext  = path.rsplit('.', 1)[-1].lower().split('?')[0] if '.' in path else ''
    return EXT_MAP.get(ext, 'images')

def safe_filename(url):
    path = urllib.parse.urlparse(url).path
    name = path.rsplit('/', 1)[-1].split('?')[0]
    name = re.sub(r'[<>:"/\\|?*]', '_', name)
    return name or hashlib.md5(url.encode()).hexdigest()[:10]

def unique_dest(cat, fname):
    dest = DIRS[cat] / fname
    if not dest.exists():
        return dest
    stem = fname.rsplit('.', 1)[0] if '.' in fname else fname
    suf  = ('.' + fname.rsplit('.', 1)[1]) if '.' in fname else ''
    for i in range(2, 9999):
        c = DIRS[cat] / f"{stem}_{i}{suf}"
        if not c.exists():
            return c

def local_rel_path(p):
    return str(p.relative_to(OUTPUT_ROOT)).replace('\\', '/')

def rel_from_css(css_dest, asset_local):
    return os.path.relpath(OUTPUT_ROOT / asset_local, css_dest.parent).replace('\\', '/')

def download_url(url):
    for attempt in range(MAX_RETRY):
        ua = UA_LIST[attempt % len(UA_LIST)]
        try:
            req = urllib.request.Request(url, headers={
                'User-Agent': ua,
                'Referer':    TARGET_URL,
                'Accept':     '*/*',
            })
            with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
                cl = r.headers.get('Content-Length')
                if cl and int(cl) > MAX_VIDEO_MB * 1024 * 1024:
                    if get_category(url) == 'flash':
                        print(f"⏭  跳过超50MB视频: {url}")
                        skipped_large.append((url, int(cl)))
                        return None
                return r.read()
        except Exception as e:
            print(f"  ⚠  第{attempt+1}次失败 [{e}]")
            if attempt < MAX_RETRY - 1:
                time.sleep(RETRY_DELAY * (attempt + 1))
    failed_urls.append(url)
    return None

def enqueue(url):
    if url and url not in queued_set:
        queued_set.add(url)
        queue.append(url)

def process_one(url):
    if url in url_to_local:
        return url_to_local[url]
    cat   = get_category(url)
    fname = safe_filename(url)
    dest  = unique_dest(cat, fname)
    data  = download_url(url)
    if not data:
        return None
    dest.write_bytes(data)
    lp = local_rel_path(dest)
    url_to_local[url] = lp
    print(f"  ✅ {lp}  <-  {url.split('?')[0]}")
    return lp

# --- URL提取 ---
URL_PATTERNS_HTML = [
    r'''(?:src|href|data-src|data-lazy|data-bg|data-original|data-url)\s*=\s*['"]([^'">\s]+)['"]''',
    r'''url\s*\(\s*['"]?([^'")>\s]+)['"]?\s*\)''',
]
URL_PATTERNS_CSS  = [
    r'''url\s*\(\s*['"]?([^'")>\s]+)['"]?\s*\)''',
    r'''@import\s+['"]([^'"]+)['"]''',
]

def extract_from_html(html, base):
    found = []
    for pat in URL_PATTERNS_HTML:
        for m in re.finditer(pat, html, re.I):
            u = normalize_url(m.group(1), base)
            if u: found.append(u)
    for m in re.finditer(r'''srcset\s*=\s*['"]([^'"]+)['"]''', html, re.I):
        for seg in m.group(1).split(','):
            tok = seg.strip().split()
            if tok and not re.match(r'^\d+(\.\d+)?[wx]$', tok[0], re.I):
                u = normalize_url(tok[0], base)
                if u: found.append(u)
    for m in re.finditer(r'<(?:video|audio|source|track)[^>]+>', html, re.I):
        for a in re.finditer(r'(?:src|poster)\s*=\s*[\'"]([^\'"]+)[\'"]', m.group(0)):
            u = normalize_url(a.group(1), base)
            if u: found.append(u)
    return list(set(found))

def extract_from_css(css, base):
    found = []
    for pat in URL_PATTERNS_CSS:
        for m in re.finditer(pat, css, re.I):
            u = normalize_url(m.group(1), base)
            if u: found.append(u)
    return list(set(found))

# --- 主流程 ---
print(f"\n{'='*60}")
print(f"目标: {TARGET_URL}")
print(f"输出: {OUTPUT_ROOT}")
print(f"{'='*60}\n")

if RENDERED_HTML_PATH and Path(RENDERED_HTML_PATH).exists():
    print("使用渲染后HTML（Playwright抓取）...")
    raw_bytes = Path(RENDERED_HTML_PATH).read_bytes()
else:
    print("抓取主页面...")
    req = urllib.request.Request(TARGET_URL, headers={
        'User-Agent': UA_LIST[0],
        'Accept': 'text/html,*/*',
    })
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
            raw_bytes = r.read()
    except Exception as e:
        print(f"失败: {e}")
        exit(1)

enc = chardet.detect(raw_bytes).get('encoding') or 'utf-8'
html_content = raw_bytes.decode(enc, errors='replace')
print(f"  编码: {enc}, 长度: {len(html_content):,} 字符\n")

print("解析资源URL...")
for u in extract_from_html(html_content, TARGET_URL):
    enqueue(u)
print(f"  发现 {len(queue)} 个资源\n")

print("下载资源...\n")
i = 0
while i < len(queue):
    url = queue[i]; i += 1
    lp  = process_one(url)
    if not lp or url in parsed_files:
        continue
    parsed_files.add(url)
    if lp.startswith('static/css'):
        css_text = (OUTPUT_ROOT / lp).read_text(errors='replace')
        added = sum(1 for u in extract_from_css(css_text, url) if u not in queued_set and (enqueue(u) or True))
        if added:
            print(f"  CSS二次扫描+{added}: {lp}")
    if lp.startswith('static/js'):
        js_text = (OUTPUT_ROOT / lp).read_text(errors='replace')
        for m in re.finditer(r"""['"]([^'"]*\.(?:jpg|jpeg|png|gif|webp|svg|mp4|woff2?))['"]""", js_text, re.I):
            u = normalize_url(m.group(1), TARGET_URL)
            if u: enqueue(u)

print(f"\n下载完成 — 成功:{len(url_to_local)} 失败:{len(failed_urls)} 跳过:{len(skipped_large)}\n")

print("替换HTML路径...")
html_fixed = html_content
for orig, lp in sorted(url_to_local.items(), key=lambda x: -len(x[0])):
    html_fixed = html_fixed.replace(orig, lp)
    html_fixed = html_fixed.replace(orig.replace('&', '&amp;'), lp)

print("替换CSS路径...")
for css_file in DIRS['css'].glob('*.css'):
    try:
        orig = css_file.read_text(errors='replace')
        fixed = orig
        def rep(m):
            raw = m.group(1).strip().strip('"\'')
            for orig_url, lp in url_to_local.items():
                if raw in orig_url or orig_url.endswith(raw.lstrip('./')):
                    rel = rel_from_css(css_file, lp)
                    return m.group(0).replace(m.group(1), f'"{rel}"')
            return m.group(0)
        fixed = re.sub(r"""url\s*\(\s*(['"]?[^'")]+['"]?)\s*\)""", rep, fixed)
        css_file.write_text(fixed, encoding='utf-8')
    except Exception as e:
        print(f"  {css_file.name}: {e}")

(OUTPUT_ROOT / 'index.html').write_text(html_fixed, encoding='utf-8')
print("  index.html 已写入\n")

# 统计
print("统计:")
total_f = total_s = 0
for cat, d in DIRS.items():
    files = [f for f in d.glob('*') if f.is_file()]
    size  = sum(f.stat().st_size for f in files)
    if files:
        print(f"  {cat:<7}: {len(files):>3} 个  {size//1024:>6} KB")
    total_f += len(files); total_s += size
print(f"  {'合计':<7}: {total_f:>3} 个  {total_s//1024:>6} KB\n")

print("完成！")