"""短剧采集服务:从内容源拉取短剧列表/详情,导入为 series + videos(幂等)""" import asyncio import logging from datetime import datetime from urllib.parse import quote import httpx from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.models.video import CollectionRecord, CollectionSource, Series, Video logger = logging.getLogger(__name__) # 内容源配置(域名轮换时只需改这里) SOURCE_KEY = "mdcmai4" UPSTREAM = "https://mdcmai4.xyz" PRODUCT_ID = 1 CATEGORY_ID = 12 # 本站分类:AI短剧 AUTHOR_USER_ID = 1 # 导入内容归属用户 SELF_API_BASE = "https://vapi.256kb.top" # 本站 API 对外地址 HEADERS = {"User-Agent": "Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36"} async def _fetch_json(client: httpx.AsyncClient, url: str, params: dict | None = None) -> dict: resp = await client.get(url, params=params, headers=HEADERS, timeout=20) resp.raise_for_status() body = resp.json() if body.get("code") != 200 or not isinstance(body.get("data"), dict): raise RuntimeError(f"上游响应异常: {url} -> {str(body)[:200]}") return body["data"] def _proxy_url(path: str) -> str: """把源站相对 m3u8 路径转为本站代理地址(域名轮换只改 SELF_API_BASE/上游常量)""" return f"{SELF_API_BASE}/api/v1/m3u8/proxy?path={quote(path, safe='')}" async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: int) -> tuple[int, int]: """导入单部短剧,返回 (新增集数, 跳过集数)""" detail = await _fetch_json(client, f"{UPSTREAM}/api/v1/short-dramas/{drama_id}", {"productId": PRODUCT_ID}) record = ( await db.execute( select(CollectionRecord).where( CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == drama_id ) ) ).scalar_one_or_none() published_at = None try: published_at = datetime.fromisoformat(detail.get("publishedAt", "")) or None except ValueError: pass series = await db.get(Series, record.series_id) if record else None if series is None: series = Series( title=(detail.get("title") or f"短剧{drama_id}")[:100], cover_url=f"{UPSTREAM}{detail.get('coverUrl', '')}", description=(detail.get("description") or "")[:500], author_id=AUTHOR_USER_ID, category_id=CATEGORY_ID, total_episodes=0, view_count=int(detail.get("heatCount") or 0), status=1, ) db.add(series) await db.flush() if record: # 系列曾被删除:重挂映射 record.series_id = series.id else: db.add(CollectionRecord(source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id)) await db.flush() # 已有集数去重 existing = set( ( await db.execute( select(Video.episode_no).where(Video.series_id == series.id, Video.episode_no.is_not(None)) ) ).scalars().all() ) created = skipped = 0 for ep in detail.get("episodes", []): ep_no = int(ep.get("episodeNo") or 0) if not ep_no or ep_no in existing: skipped += 1 continue db.add( Video( user_id=AUTHOR_USER_ID, category_id=CATEGORY_ID, series_id=series.id, episode_no=ep_no, title=(ep.get("titleOverride") or ep.get("title") or f"第{ep_no}集")[:100], description=(detail.get("description") or "")[:500], cover_url=f"{UPSTREAM}{detail.get('coverUrl', '')}", video_url=_proxy_url(ep.get("videoUrl", "")), duration=int(ep.get("durationSec") or 0), status=2, published_at=published_at, ) ) existing.add(ep_no) created += 1 series.total_episodes = len(existing) return created, skipped async def sync_drama_by_id(db: AsyncSession, drama_id: int) -> dict: """手动同步单部短剧(新增或补充缺失集数);返回统计""" async with httpx.AsyncClient(follow_redirects=True) as client: created, skipped = await _import_drama(client, db, drama_id) await db.commit() return {"drama_id": drama_id, "episodes_created": created, "episodes_skipped": skipped} async def sync_short_dramas( db: AsyncSession, max_pages: int = 0, page_size: int = 20, sort_by: str = "heat", delay: float = 0.25, ) -> dict: """按列表页增量同步短剧;返回统计""" stat = {"dramas_new": 0, "dramas_exist": 0, "episodes_created": 0, "episodes_skipped": 0, "pages": 0} async with httpx.AsyncClient(follow_redirects=True) as client: page = 1 while True: data = await _fetch_json( client, f"{UPSTREAM}/api/v1/short-dramas", {"productId": PRODUCT_ID, "sortBy": sort_by, "page": page, "size": page_size}, ) items = data.get("items", []) stat["pages"] = page logger.info("列表第 %s/%s 页,%s 部", page, data.get("totalPages"), len(items)) for item in items: drama_id = int(item.get("id") or 0) if not drama_id: continue exists = ( await db.execute( select(CollectionRecord.id).where( CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == drama_id ) ) ).scalar_one_or_none() if exists: stat["dramas_exist"] += 1 continue try: created, skipped = await _import_drama(client, db, drama_id) await db.commit() stat["dramas_new"] += 1 stat["episodes_created"] += created stat["episodes_skipped"] += skipped logger.info("导入短剧 %s《%s》:新增 %s 集", drama_id, item.get("title"), created) except Exception: await db.rollback() logger.exception("导入短剧 %s 失败,跳过", drama_id) await asyncio.sleep(delay) total_pages = int(data.get("totalPages") or 1) if page >= total_pages or (max_pages and page >= max_pages): break page += 1 await asyncio.sleep(delay) return stat # ---------- 单视频采集(通用 URL 模板,后台自助配置的采集源)---------- def _abs_cover(item: dict) -> str: """封面转绝对地址:源站 /api/... 代理路径或 /uploads/... 静态路径""" u = item.get("coverUrl") or "" if u.startswith("http"): return u return f"{UPSTREAM}{u}" if u else "" async def _import_video_item(db: AsyncSession, source: CollectionSource, item: dict) -> bool: """导入单条视频,已存在返回 False""" vid = int(item.get("id") or 0) if not vid or not (item.get("videoUrl") or "").endswith(".m3u8"): return False exists = ( await db.execute( select(CollectionRecord.id).where( CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == vid ) ) ).scalar_one_or_none() if exists: return False published_at = None try: published_at = datetime.fromisoformat(item.get("publishedAt", "")) or None except ValueError: pass v = Video( user_id=AUTHOR_USER_ID, category_id=source.category_id, title=(item.get("title") or f"视频{vid}")[:100], description=(item.get("description") or "")[:500], cover_url=_abs_cover(item)[:500], video_url=_proxy_url(item["videoUrl"]), duration=int(item.get("durationSec") or 0), status=2, view_count=int(item.get("viewCount") or 0), like_count=int(item.get("likeCount") or 0), favorite_count=int(item.get("favoriteCount") or 0), published_at=published_at, ) db.add(v) await db.flush() db.add(CollectionRecord(source_key=SOURCE_KEY, source_id=vid, video_id=v.id)) return True async def sync_source( db: AsyncSession, source: CollectionSource, max_pages: int = 0, stop_after_existing: int = 30, delay: float = 0.25, ) -> dict: """按源的 URL 模板翻页采集;连续 stop_after_existing 条已存在则提前停止(增量)""" stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False} consecutive_existing = 0 async with httpx.AsyncClient(follow_redirects=True) as client: page = 1 while True: url = source.list_url.replace("{{page}}", str(page)) data = await _fetch_json(client, url) items = data.get("items", []) if not items: break stat["pages"] = page for item in items: if await _import_video_item(db, source, item): stat["created"] += 1 consecutive_existing = 0 else: stat["skipped"] += 1 consecutive_existing += 1 if consecutive_existing >= stop_after_existing: stat["stopped_early"] = True break await db.commit() if stat["stopped_early"]: break total_pages = int(data.get("totalPages") or page) if page >= total_pages or (max_pages and page >= max_pages): break page += 1 await asyncio.sleep(delay) source.last_sync_at = datetime.now() await db.commit() return stat