video/server/app/services/collection_service.py

"""短剧采集服务：从内容源拉取短剧列表/详情，导入为 series + videos（幂等）"""
import asyncio
import logging
from datetime import datetime
from urllib.parse import quote

import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.models.video import CollectionRecord, CollectionSource, Series, Video

logger = logging.getLogger(__name__)

# 内容源配置（域名轮换时只需改这里）
SOURCE_KEY = "mdcmai4"
UPSTREAM = "https://mdcmai4.xyz"
PRODUCT_ID = 1
CATEGORY_ID = 12          # 本站分类：AI短剧
AUTHOR_USER_ID = 1        # 导入内容归属用户
SELF_API_BASE = "https://vapi.256kb.top"  # 本站 API 对外地址

HEADERS = {"User-Agent": "Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36"}


async def _fetch_json(client: httpx.AsyncClient, url: str, params: dict | None = None) -> dict:
    resp = await client.get(url, params=params, headers=HEADERS, timeout=20)
    resp.raise_for_status()
    body = resp.json()
    if body.get("code") != 200 or not isinstance(body.get("data"), dict):
        raise RuntimeError(f"上游响应异常: {url} -> {str(body)[:200]}")
    return body["data"]


def _proxy_url(path: str) -> str:
    """把源站相对 m3u8 路径转为本站代理地址（域名轮换只改 SELF_API_BASE/上游常量）"""
    return f"{SELF_API_BASE}/api/v1/m3u8/proxy?path={quote(path, safe='')}"


async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: int) -> tuple[int, int]:
    """导入单部短剧，返回 (新增集数, 跳过集数)"""
    detail = await _fetch_json(client, f"{UPSTREAM}/api/v1/short-dramas/{drama_id}", {"productId": PRODUCT_ID})

    record = (
        await db.execute(
            select(CollectionRecord).where(
                CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == drama_id
            )
        )
    ).scalar_one_or_none()

    published_at = None
    try:
        published_at = datetime.fromisoformat(detail.get("publishedAt", "")) or None
    except ValueError:
        pass

    series = await db.get(Series, record.series_id) if record else None
    if series is None:
        series = Series(
            title=(detail.get("title") or f"短剧{drama_id}")[:100],
            cover_url=f"{UPSTREAM}{detail.get('coverUrl', '')}",
            description=(detail.get("description") or "")[:500],
            author_id=AUTHOR_USER_ID,
            category_id=CATEGORY_ID,
            total_episodes=0,
            view_count=int(detail.get("heatCount") or 0),
            status=1,
        )
        db.add(series)
        await db.flush()
        if record:  # 系列曾被删除：重挂映射
            record.series_id = series.id
        else:
            db.add(CollectionRecord(source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id))
        await db.flush()

    # 已有集数去重
    existing = set(
        (
            await db.execute(
                select(Video.episode_no).where(Video.series_id == series.id, Video.episode_no.is_not(None))
            )
        ).scalars().all()
    )

    created = skipped = 0
    for ep in detail.get("episodes", []):
        ep_no = int(ep.get("episodeNo") or 0)
        if not ep_no or ep_no in existing:
            skipped += 1
            continue
        db.add(
            Video(
                user_id=AUTHOR_USER_ID,
                category_id=CATEGORY_ID,
                series_id=series.id,
                episode_no=ep_no,
                title=(ep.get("titleOverride") or ep.get("title") or f"第{ep_no}集")[:100],
                description=(detail.get("description") or "")[:500],
                cover_url=f"{UPSTREAM}{detail.get('coverUrl', '')}",
                video_url=_proxy_url(ep.get("videoUrl", "")),
                duration=int(ep.get("durationSec") or 0),
                status=2,
                published_at=published_at,
            )
        )
        existing.add(ep_no)
        created += 1

    series.total_episodes = len(existing)
    return created, skipped


async def sync_drama_by_id(db: AsyncSession, drama_id: int) -> dict:
    """手动同步单部短剧（新增或补充缺失集数）；返回统计"""
    async with httpx.AsyncClient(follow_redirects=True) as client:
        created, skipped = await _import_drama(client, db, drama_id)
        await db.commit()
    return {"drama_id": drama_id, "episodes_created": created, "episodes_skipped": skipped}


async def sync_short_dramas(
    db: AsyncSession,
    max_pages: int = 0,
    page_size: int = 20,
    sort_by: str = "heat",
    delay: float = 0.25,
) -> dict:
    """按列表页增量同步短剧；返回统计"""
    stat = {"dramas_new": 0, "dramas_exist": 0, "episodes_created": 0, "episodes_skipped": 0, "pages": 0}
    async with httpx.AsyncClient(follow_redirects=True) as client:
        page = 1
        while True:
            data = await _fetch_json(
                client,
                f"{UPSTREAM}/api/v1/short-dramas",
                {"productId": PRODUCT_ID, "sortBy": sort_by, "page": page, "size": page_size},
            )
            items = data.get("items", [])
            stat["pages"] = page
            logger.info("列表第 %s/%s 页，%s 部", page, data.get("totalPages"), len(items))

            for item in items:
                drama_id = int(item.get("id") or 0)
                if not drama_id:
                    continue
                exists = (
                    await db.execute(
                        select(CollectionRecord.id).where(
                            CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == drama_id
                        )
                    )
                ).scalar_one_or_none()
                if exists:
                    stat["dramas_exist"] += 1
                    continue
                try:
                    created, skipped = await _import_drama(client, db, drama_id)
                    await db.commit()
                    stat["dramas_new"] += 1
                    stat["episodes_created"] += created
                    stat["episodes_skipped"] += skipped
                    logger.info("导入短剧 %s《%s》：新增 %s 集", drama_id, item.get("title"), created)
                except Exception:
                    await db.rollback()
                    logger.exception("导入短剧 %s 失败，跳过", drama_id)
                await asyncio.sleep(delay)

            total_pages = int(data.get("totalPages") or 1)
            if page >= total_pages or (max_pages and page >= max_pages):
                break
            page += 1
            await asyncio.sleep(delay)
    return stat


# ---------- 单视频采集（通用 URL 模板，后台自助配置的采集源）----------

def _abs_cover(item: dict) -> str:
    """封面转绝对地址：源站 /api/... 代理路径或 /uploads/... 静态路径"""
    u = item.get("coverUrl") or ""
    if u.startswith("http"):
        return u
    return f"{UPSTREAM}{u}" if u else ""


async def _import_video_item(db: AsyncSession, source: CollectionSource, item: dict) -> bool:
    """导入单条视频，已存在返回 False"""
    vid = int(item.get("id") or 0)
    if not vid or not (item.get("videoUrl") or "").endswith(".m3u8"):
        return False
    exists = (
        await db.execute(
            select(CollectionRecord.id).where(
                CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == vid
            )
        )
    ).scalar_one_or_none()
    if exists:
        return False

    published_at = None
    try:
        published_at = datetime.fromisoformat(item.get("publishedAt", "")) or None
    except ValueError:
        pass

    v = Video(
        user_id=AUTHOR_USER_ID,
        category_id=source.category_id,
        title=(item.get("title") or f"视频{vid}")[:100],
        description=(item.get("description") or "")[:500],
        cover_url=_abs_cover(item)[:500],
        video_url=_proxy_url(item["videoUrl"]),
        duration=int(item.get("durationSec") or 0),
        status=2,
        view_count=int(item.get("viewCount") or 0),
        like_count=int(item.get("likeCount") or 0),
        favorite_count=int(item.get("favoriteCount") or 0),
        published_at=published_at,
    )
    db.add(v)
    await db.flush()
    db.add(CollectionRecord(source_key=SOURCE_KEY, source_id=vid, video_id=v.id))
    return True


async def sync_source(
    db: AsyncSession,
    source: CollectionSource,
    max_pages: int = 0,
    stop_after_existing: int = 30,
    delay: float = 0.25,
) -> dict:
    """按源的 URL 模板翻页采集；连续 stop_after_existing 条已存在则提前停止（增量）"""
    stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False}
    consecutive_existing = 0
    async with httpx.AsyncClient(follow_redirects=True) as client:
        page = 1
        while True:
            url = source.list_url.replace("{{page}}", str(page))
            data = await _fetch_json(client, url)
            items = data.get("items", [])
            if not items:
                break
            stat["pages"] = page
            for item in items:
                if await _import_video_item(db, source, item):
                    stat["created"] += 1
                    consecutive_existing = 0
                else:
                    stat["skipped"] += 1
                    consecutive_existing += 1
                if consecutive_existing >= stop_after_existing:
                    stat["stopped_early"] = True
                    break
            await db.commit()
            if stat["stopped_early"]:
                break
            total_pages = int(data.get("totalPages") or page)
            if page >= total_pages or (max_pages and page >= max_pages):
                break
            page += 1
            await asyncio.sleep(delay)

    source.last_sync_at = datetime.now()
    await db.commit()
    return stat