863 lines
32 KiB
Python
863 lines
32 KiB
Python
"""91 短剧（AI短剧）采集：加密接口客户端 + 系列/集数入库
|
||
|
||
上游协议：pwa.php 单入口，AES-256-CFB(segment 128) 加密请求体 + MD5(SHA256) 签名。
|
||
时间戳签名有效期约 4 小时，因此**不存原始 m3u8 地址**，改为存本站代理地址，
|
||
播放时再向上游换取新鲜签名（见 /api/v1/m3u8/aidrama）。
|
||
"""
|
||
import asyncio
|
||
import hashlib
|
||
import json
|
||
import logging
|
||
import random
|
||
import string
|
||
import time
|
||
from datetime import datetime
|
||
|
||
import httpx
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.models.video import (
|
||
Category,
|
||
CollectionRecord,
|
||
CollectionSource,
|
||
Series,
|
||
Tag,
|
||
Video,
|
||
VideoTag,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
KIND = "aidrama91"
|
||
VLOG_KIND = "aidrama91_vlog" # 91 非系列短视频
|
||
SOURCE_KEY = "91aidrama" # collection_records.source_key
|
||
VLOG_SOURCE_KEY = "91vlog" # 短视频采集记录 key
|
||
SOURCE_CODE = "91" # sources.code
|
||
SELF_API_BASE = "https://vapi.256kb.top"
|
||
AUTHOR_USER_ID = 1
|
||
VLOG_CATEGORY_NAME = "91短视频" # vlog 无真实分类，归到本地分类
|
||
|
||
# 协议常量（前端 JS 逆向）
|
||
AES_KEY = b"cc88ddc9357ff461e08f047aedee692b"
|
||
AES_IV = b"e89225cfbbimgkcu"
|
||
SIGN_KEY = "cc88ddc9357ff461e08f047aedee692b"
|
||
APP_VERSION = "7.5.0"
|
||
CLIENT_ORIGIN = "https://pwa4.jl8v0p5t.cc"
|
||
|
||
# 封面图解密：CDN 返回的是 AES-128-CBC 加密的图片字节（无 padding），
|
||
# 源站前端 DecryptImage 解密后展示 —— 本站通过 /api/v1/img/aidrama 代理解密
|
||
IMG_AES_KEY = b"f5d965df75336270"
|
||
IMG_AES_IV = b"97b60394abc2fbe1"
|
||
IMG_HOST_SUFFIX = ".drleao.cn" # 封面 CDN 域名白名单（防 SSRF）
|
||
|
||
HEADERS = {
|
||
"User-Agent": (
|
||
"Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
|
||
),
|
||
"Origin": CLIENT_ORIGIN,
|
||
"Referer": f"{CLIENT_ORIGIN}/",
|
||
"Accept": "application/json, text/plain, */*",
|
||
"Content-Type": "application/x-www-form-urlencoded",
|
||
}
|
||
|
||
OAUTH_CHARSET = string.ascii_letters + string.digits
|
||
MAX_PAGE_LIMIT = 20 # 上游单页上限（传 100 实际只返回 20）
|
||
|
||
|
||
# ---------- 加密 / 签名 ----------
|
||
|
||
def _cfb(key: bytes, iv: bytes):
|
||
try:
|
||
from cryptography.hazmat.decrepit.ciphers.modes import CFB
|
||
except ImportError: # cryptography < 43
|
||
from cryptography.hazmat.primitives.ciphers.modes import CFB
|
||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
|
||
|
||
return Cipher(algorithms.AES(key), CFB(iv))
|
||
|
||
|
||
def _encrypt(plaintext: str) -> str:
|
||
op = _cfb(AES_KEY, AES_IV).encryptor()
|
||
return (op.update(plaintext.encode()) + op.finalize()).hex().upper()
|
||
|
||
|
||
def _decrypt(hex_data: str) -> str:
|
||
op = _cfb(AES_KEY, AES_IV).decryptor()
|
||
return (op.update(bytes.fromhex(hex_data)) + op.finalize()).decode("utf-8", "replace")
|
||
|
||
|
||
def _new_oauth_id() -> str:
|
||
return "".join(random.choices(OAUTH_CHARSET, k=16)) + str(int(time.time() * 1000))
|
||
|
||
|
||
async def _post(client: httpx.AsyncClient, api_url: str, payload: dict) -> dict:
|
||
"""加密并发送，返回解密后的业务数据（已剥掉外层包装）"""
|
||
ts_ms = int(time.time() * 1000)
|
||
payload = dict(payload, timestamp=ts_ms)
|
||
data_hex = _encrypt(json.dumps(payload, separators=(",", ":")))
|
||
ts_sec = ts_ms // 1000
|
||
raw = f"client=pwa&data={data_hex}×tamp={ts_sec}{SIGN_KEY}"
|
||
sign = hashlib.md5(hashlib.sha256(raw.encode()).hexdigest().encode()).hexdigest()
|
||
|
||
resp = await client.post(
|
||
api_url,
|
||
content=f"client=pwa×tamp={ts_sec}&data={data_hex}&sign={sign}",
|
||
headers=HEADERS,
|
||
timeout=30,
|
||
)
|
||
if resp.status_code != 200:
|
||
raise RuntimeError(f"上游 HTTP {resp.status_code}（oauth_id 可能未注册或已被限流）")
|
||
body = resp.json()
|
||
if body.get("errcode") not in (0, None):
|
||
raise RuntimeError(f"上游错误: {str(body)[:200]}")
|
||
payload_text = body.get("data")
|
||
if not payload_text:
|
||
return body
|
||
return json.loads(_decrypt(payload_text))
|
||
|
||
|
||
def _base(oauth_id: str, token: str) -> dict:
|
||
return {
|
||
"oauth_type": "pwa", "oauth_id": oauth_id, "trace_id": "",
|
||
"aff_x_code": "", "version": APP_VERSION, "token": token,
|
||
}
|
||
|
||
|
||
# ---------- 会话 ----------
|
||
|
||
async def create_session(
|
||
client: httpx.AsyncClient, api_url: str, account: str, password: str
|
||
) -> tuple[str, str]:
|
||
"""注册 oauth_id 并登录，返回 (oauth_id, token)"""
|
||
oauth_id = _new_oauth_id()
|
||
# 上游对未注册的 oauth_id 直接 503，必须先 system/index
|
||
await _post(client, api_url, dict(_base(oauth_id, ""), mod="system", code="index", data={}))
|
||
result = await _post(client, api_url, dict(
|
||
_base(oauth_id, ""), mod="user", code="regOrLogin",
|
||
account=account, password=password, type="login",
|
||
))
|
||
inner = result.get("data", result) if isinstance(result, dict) else {}
|
||
token = inner.get("token") or ""
|
||
if not token:
|
||
raise RuntimeError(f"91 登录失败: {inner.get('msg') or str(result)[:150]}")
|
||
return oauth_id, token
|
||
|
||
|
||
# 进程内 token 缓存（2 个 uvicorn worker → 最多 2 次登录）；失效时自动重登
|
||
_TOKENS: dict[int, tuple[str, str, float]] = {}
|
||
TOKEN_TTL = 3600
|
||
|
||
|
||
async def open_session(
|
||
client: httpx.AsyncClient, source: CollectionSource, force: bool = False
|
||
) -> tuple[str, str]:
|
||
"""获取可用的 (oauth_id, token)：进程缓存 → DB 配置 → 重新登录"""
|
||
key = int(source.id or 0)
|
||
cfg = load_config(source)
|
||
if not force:
|
||
hit = _TOKENS.get(key)
|
||
if hit and hit[2] > time.time():
|
||
return hit[0], hit[1]
|
||
if cfg.get("oauth_id") and cfg.get("token"):
|
||
_TOKENS[key] = (cfg["oauth_id"], cfg["token"], time.time() + TOKEN_TTL)
|
||
return cfg["oauth_id"], cfg["token"]
|
||
|
||
oauth_id, token = await create_session(
|
||
client, source.list_url, cfg["account"], cfg["password"])
|
||
_TOKENS[key] = (oauth_id, token, time.time() + TOKEN_TTL)
|
||
return oauth_id, token
|
||
|
||
|
||
def load_config(source: CollectionSource) -> dict:
|
||
try:
|
||
cfg = json.loads(source.config or "{}")
|
||
except json.JSONDecodeError:
|
||
cfg = {}
|
||
cfg.setdefault("account", "")
|
||
cfg.setdefault("password", "")
|
||
cfg.setdefault("category_ids", [4])
|
||
cfg.setdefault("page_size", MAX_PAGE_LIMIT)
|
||
return cfg
|
||
|
||
|
||
# ---------- 业务接口 ----------
|
||
|
||
async def fetch_list(
|
||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
|
||
page: int, limit: int, category_ids: list[int],
|
||
) -> dict:
|
||
"""短剧列表；返回 {list, total, has_more}"""
|
||
data = await _post(client, api_url, dict(
|
||
_base(oauth_id, token), mod="aidrama", code="home",
|
||
page=page, limit=min(limit, MAX_PAGE_LIMIT), pay_type="all", sort="all",
|
||
category_ids=category_ids or [4],
|
||
))
|
||
inner = data.get("data") or {}
|
||
return {
|
||
"list": inner.get("list") or [],
|
||
"total": int(inner.get("total") or 0),
|
||
"has_more": bool(inner.get("has_more")),
|
||
}
|
||
|
||
|
||
async def fetch_detail(
|
||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
|
||
) -> dict:
|
||
data = await _post(client, api_url, dict(
|
||
_base(oauth_id, token), mod="aidrama", code="detail", drama_id=drama_id,
|
||
))
|
||
return (data.get("data") or {}).get("row") or {}
|
||
|
||
|
||
async def fetch_episodes(
|
||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
|
||
drama_id: int, page: int = 1, limit: int = MAX_PAGE_LIMIT,
|
||
) -> dict:
|
||
"""集数列表；返回 {list, has_more}"""
|
||
data = await _post(client, api_url, dict(
|
||
_base(oauth_id, token), mod="aidrama", code="episodes",
|
||
drama_id=drama_id, page=page, limit=min(limit, MAX_PAGE_LIMIT),
|
||
))
|
||
inner = data.get("data") or {}
|
||
return {"list": inner.get("list") or [], "has_more": bool(inner.get("has_more"))}
|
||
|
||
|
||
async def fetch_all_episodes(
|
||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
|
||
) -> list[dict]:
|
||
"""翻页取全部集数（每页 20 上限）"""
|
||
out: list[dict] = []
|
||
page = 1
|
||
while True:
|
||
part = await fetch_episodes(client, api_url, oauth_id, token, drama_id, page=page)
|
||
out.extend(part["list"])
|
||
if not part["has_more"] or not part["list"]:
|
||
break
|
||
page += 1
|
||
return out
|
||
|
||
|
||
# ---------- vlog 短视频接口 ----------
|
||
|
||
VLOG_MAX_LIMIT = 50 # 实测 vlog 单页上限 50（传 100 只返回 50）
|
||
|
||
|
||
async def fetch_vlog_list(
|
||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
|
||
page: int, limit: int = VLOG_MAX_LIMIT, vlog_type: str = "new",
|
||
) -> dict:
|
||
"""vlog 列表（非系列短视频）；返回 {list, has_more}"""
|
||
data = await _post(client, api_url, dict(
|
||
_base(oauth_id, token), mod="choice", code="vlog",
|
||
type=vlog_type, page=page, limit=min(limit, VLOG_MAX_LIMIT),
|
||
))
|
||
lst = data.get("data") or []
|
||
if not isinstance(lst, list):
|
||
lst = []
|
||
return {"list": lst, "has_more": len(lst) >= min(limit, VLOG_MAX_LIMIT)}
|
||
|
||
|
||
async def fetch_vlog_detail(
|
||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, vid: int
|
||
) -> dict:
|
||
"""vlog 详情（含 playUrl / tags）"""
|
||
data = await _post(client, api_url, dict(
|
||
_base(oauth_id, token), mod="index", code="detail", id=vid))
|
||
return data.get("data") or {}
|
||
|
||
|
||
# ---------- 字段解析 ----------
|
||
|
||
def _pick_url(raw: str | dict, prefer: str = "720") -> str:
|
||
"""封面字段是 JSON 字符串或 dict：{"ori","360","720"}，取指定档位"""
|
||
if not raw:
|
||
return ""
|
||
try:
|
||
obj = json.loads(raw) if isinstance(raw, str) else raw
|
||
except json.JSONDecodeError:
|
||
return raw if isinstance(raw, str) else ""
|
||
if not isinstance(obj, dict):
|
||
return ""
|
||
return obj.get(prefer) or obj.get("ori") or ""
|
||
|
||
|
||
def proxied_cover(url: str) -> str:
|
||
"""封面地址改写为本站解密代理（CDN 图是加密的，直连无法显示）"""
|
||
if not url or "/api/v1/img/aidrama" in url:
|
||
return url
|
||
if not url.startswith(("http://", "https://")):
|
||
return url
|
||
from urllib.parse import quote
|
||
|
||
return f"{SELF_API_BASE}/api/v1/img/aidrama?u={quote(url, safe='')}"
|
||
|
||
|
||
def decrypt_image(data: bytes) -> bytes:
|
||
"""AES-128-CBC(NoPadding) 解密 CDN 加密图片，返回原始 JPEG/PNG 字节"""
|
||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||
|
||
op = Cipher(algorithms.AES(IMG_AES_KEY), modes.CBC(IMG_AES_IV)).decryptor()
|
||
return op.update(data) + op.finalize()
|
||
|
||
|
||
def _play_url(drama_id: int, episode_id: int, episode_no: int | None) -> str:
|
||
"""本站代理地址（含 m3u8 字样，App 端会走本地缓存代理）"""
|
||
url = f"{SELF_API_BASE}/api/v1/m3u8/aidrama?d={drama_id}&e={episode_id}"
|
||
if episode_no:
|
||
url += f"&n={episode_no}"
|
||
return url
|
||
|
||
|
||
def _safe_dt(ts) -> datetime | None:
|
||
try:
|
||
n = int(ts)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
return datetime.fromtimestamp(n) if n > 0 else None
|
||
|
||
|
||
def _parse_count(v) -> int:
|
||
"""解析播放/点赞等计数，兼容 '143.7万' / '1.2万' / '34' / 34"""
|
||
if v is None:
|
||
return 0
|
||
if isinstance(v, (int, float)):
|
||
return int(v)
|
||
s = str(v).strip()
|
||
if not s:
|
||
return 0
|
||
if s.endswith("万"):
|
||
try:
|
||
return int(float(s[:-1]) * 10000)
|
||
except ValueError:
|
||
return 0
|
||
try:
|
||
return int(s)
|
||
except ValueError:
|
||
return 0
|
||
|
||
|
||
def _vlog_play_url(vid: int) -> str:
|
||
"""vlog 单视频播放代理地址（含 m3u8 字样，App 走本地缓存）"""
|
||
return f"{SELF_API_BASE}/api/v1/m3u8/vlog?vid={vid}"
|
||
|
||
|
||
# ---------- 入库 ----------
|
||
|
||
async def _upsert_tags(db: AsyncSession, names: list[str]) -> list[int]:
|
||
"""按名称取/建标签，返回 tag id 列表"""
|
||
ids: list[int] = []
|
||
for name in {n.strip() for n in names if n and n.strip()}:
|
||
name = name[:20]
|
||
tid = (await db.execute(select(Tag.id).where(Tag.name == name))).scalar_one_or_none()
|
||
if not tid:
|
||
t = Tag(name=name, video_count=0)
|
||
db.add(t)
|
||
await db.flush()
|
||
tid = t.id
|
||
ids.append(tid)
|
||
return ids
|
||
|
||
|
||
async def import_drama(
|
||
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
|
||
drama: dict, oauth_id: str, token: str,
|
||
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
|
||
) -> tuple[int, int]:
|
||
"""导入单部短剧（系列 + 集数视频），返回 (新增集数, 跳过集数)"""
|
||
drama_id = int(drama.get("id") or 0)
|
||
if not drama_id:
|
||
return 0, 0
|
||
api_url = api_url or source.list_url
|
||
cat = category_id if category_id is not None else source.category_id
|
||
sid = source_id if source_id is not None else source.source_id
|
||
|
||
cfg = load_config(source)
|
||
detail = drama
|
||
try:
|
||
full = await fetch_detail(client, api_url, oauth_id, token, drama_id)
|
||
if full:
|
||
detail = {**drama, **full}
|
||
except Exception:
|
||
logger.warning("短剧 %s 详情获取失败，使用列表字段", drama_id)
|
||
|
||
record = (await db.execute(
|
||
select(CollectionRecord).where(
|
||
CollectionRecord.source_key == SOURCE_KEY,
|
||
CollectionRecord.source_id == drama_id,
|
||
)
|
||
)).scalar_one_or_none()
|
||
|
||
cover = proxied_cover(_pick_url(detail.get("cover") or ""))
|
||
tags = list(detail.get("tag_list") or []) + list(detail.get("category_list") or [])
|
||
# 像既有采集一样：按标题拆分关键词作为 tag
|
||
from app.services.title_tags import extract_keywords
|
||
|
||
tags += extract_keywords(detail.get("title") or "")
|
||
total_declared = int(detail.get("total_episodes") or 0)
|
||
|
||
series = await db.get(Series, record.series_id) if record else None
|
||
if series is None:
|
||
series = Series(
|
||
title=(detail.get("title") or f"短剧{drama_id}")[:100],
|
||
cover_url=cover[:500],
|
||
description=(detail.get("description") or "")[:500],
|
||
author_id=AUTHOR_USER_ID,
|
||
category_id=cat,
|
||
source_id=sid,
|
||
total_episodes=total_declared,
|
||
status=1,
|
||
)
|
||
db.add(series)
|
||
await db.flush()
|
||
if record:
|
||
record.series_id = series.id
|
||
else:
|
||
db.add(CollectionRecord(
|
||
source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id))
|
||
await db.flush()
|
||
else:
|
||
# 已有系列：同步标题/封面/简介（源站可能改过）
|
||
series.title = (detail.get("title") or series.title)[:100]
|
||
series.cover_url = (cover or series.cover_url)[:500]
|
||
series.total_episodes = max(series.total_episodes, total_declared)
|
||
|
||
existing = set((await db.execute(
|
||
select(Video.episode_no).where(
|
||
Video.series_id == series.id, Video.episode_no.is_not(None))
|
||
)).scalars().all())
|
||
|
||
episodes = await fetch_all_episodes(client, api_url, oauth_id, token, drama_id)
|
||
tag_ids: list[int] | None = None
|
||
created = skipped = 0
|
||
now = datetime.now()
|
||
|
||
for ep in episodes:
|
||
ep_id = int(ep.get("id") or 0)
|
||
ep_no = int(ep.get("episode_number") or 0)
|
||
if not ep_id or not ep_no or ep_no in existing:
|
||
skipped += 1
|
||
continue
|
||
|
||
ep_cover = proxied_cover(_pick_url(ep.get("cover") or "")) or cover
|
||
video = Video(
|
||
user_id=AUTHOR_USER_ID,
|
||
category_id=cat,
|
||
source_id=sid,
|
||
series_id=series.id,
|
||
episode_no=ep_no,
|
||
title=(ep.get("title") or f"第{ep_no}集")[:100],
|
||
description=(detail.get("description") or "")[:500],
|
||
cover_url=ep_cover[:500],
|
||
video_url=_play_url(drama_id, ep_id, ep_no),
|
||
duration=int(ep.get("duration") or 0),
|
||
status=2,
|
||
like_count=int(ep.get("like") or 0),
|
||
favorite_count=int(ep.get("favorite_num") or 0),
|
||
comment_count=int(ep.get("comment_count") or 0),
|
||
published_at=_safe_dt(ep.get("created_at")) or now,
|
||
)
|
||
db.add(video)
|
||
await db.flush()
|
||
|
||
# 标签挂到每集视频上（系列标签来自 tag_list / category_list）
|
||
if tags:
|
||
if tag_ids is None:
|
||
tag_ids = await _upsert_tags(db, tags)
|
||
for tid in tag_ids:
|
||
db.add(VideoTag(video_id=video.id, tag_id=tid))
|
||
|
||
existing.add(ep_no)
|
||
created += 1
|
||
|
||
series.total_episodes = max(len(existing), total_declared)
|
||
return created, skipped
|
||
|
||
|
||
async def sync_source(
|
||
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
|
||
should_cancel=None, on_progress=None,
|
||
) -> dict:
|
||
"""按列表翻页增量采集（已导入的短剧跳过，仍会补同步新集数）
|
||
|
||
should_cancel: 可选回调，返回 True 时在每页/每条边界退出循环（用于手动取消）
|
||
on_progress: 可选回调，每页拉到列表后调用 on_progress(dict(stat))（用于后台展示进度）
|
||
"""
|
||
cfg = load_config(source)
|
||
# 每条/每页之间的间隔可由采集源 config 的 delay 自定义（秒），未配置时用默认值
|
||
if cfg.get("delay") is not None:
|
||
delay = float(cfg["delay"])
|
||
stat: dict = {
|
||
"kind": KIND, "dramas_new": 0, "dramas_exist": 0,
|
||
"episodes_created": 0, "episodes_skipped": 0, "pages": 0, "total": 0,
|
||
}
|
||
page_size = min(int(cfg.get("page_size") or MAX_PAGE_LIMIT), MAX_PAGE_LIMIT)
|
||
category_ids = cfg.get("category_ids") or [4]
|
||
|
||
def _cancelled() -> bool:
|
||
return bool(should_cancel and should_cancel())
|
||
|
||
for attempt in (1, 2):
|
||
try:
|
||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||
oauth_id, token = await open_session(client, source, force=attempt == 2)
|
||
api_url = source.list_url
|
||
cat = source.category_id
|
||
sid = source.source_id
|
||
page = 1
|
||
while True:
|
||
if _cancelled():
|
||
logger.info("91 短剧同步被取消（第 %s 页前）", page)
|
||
return stat
|
||
data = await fetch_list(
|
||
client, api_url, oauth_id, token, page, page_size, category_ids)
|
||
stat["pages"] = page
|
||
stat["total"] = data["total"]
|
||
items = data["list"]
|
||
logger.info("91 短剧第 %s 页：%s 部（共 %s）", page, len(items), data["total"])
|
||
if on_progress:
|
||
on_progress(dict(stat))
|
||
|
||
for drama in items:
|
||
if _cancelled():
|
||
logger.info("91 短剧同步被取消（处理 drama 中）")
|
||
return stat
|
||
drama_id = int(drama.get("id") or 0)
|
||
if not drama_id:
|
||
continue
|
||
try:
|
||
created, skipped = await import_drama(
|
||
db, client, source, drama, oauth_id, token,
|
||
api_url=api_url, category_id=cat, source_id=sid)
|
||
await db.commit()
|
||
if created > 0:
|
||
stat["dramas_new"] += 1
|
||
else:
|
||
stat["dramas_exist"] += 1
|
||
stat["episodes_created"] += created
|
||
stat["episodes_skipped"] += skipped
|
||
except Exception:
|
||
await db.rollback()
|
||
logger.exception("导入 91 短剧 %s 失败，跳过", drama_id)
|
||
await asyncio.sleep(delay)
|
||
|
||
if not data["has_more"] or not items:
|
||
break
|
||
if max_pages and page >= max_pages:
|
||
break
|
||
page += 1
|
||
await asyncio.sleep(delay)
|
||
|
||
cfg["oauth_id"], cfg["token"] = oauth_id, token
|
||
source.config = json.dumps(cfg, ensure_ascii=False)
|
||
break
|
||
except Exception:
|
||
_TOKENS.pop(int(source.id or 0), None)
|
||
if attempt == 2:
|
||
source.last_sync_at = datetime.now()
|
||
await db.commit()
|
||
raise
|
||
logger.warning("91 同步失败，重新登录后重试一次")
|
||
await asyncio.sleep(1)
|
||
|
||
source.last_sync_at = datetime.now()
|
||
await db.commit()
|
||
return stat
|
||
|
||
|
||
async def refresh_source_id(db: AsyncSession, source: CollectionSource) -> int:
|
||
"""确保采集源绑定的 sources 记录存在，返回来源 id"""
|
||
if source.source_id:
|
||
return source.source_id
|
||
from app.models.video import Source
|
||
|
||
sid = (await db.execute(
|
||
select(Source.id).where(Source.code == SOURCE_CODE)
|
||
)).scalar_one_or_none()
|
||
if not sid:
|
||
max_sort = 0
|
||
s = Source(name="91", code=SOURCE_CODE, sort=max_sort + 1, status=1)
|
||
db.add(s)
|
||
await db.flush()
|
||
sid = s.id
|
||
source.source_id = sid
|
||
return sid
|
||
|
||
|
||
async def ensure_local_category(db: AsyncSession, name: str) -> int:
|
||
"""按采集源名称找/建本站分类"""
|
||
cid = (await db.execute(
|
||
select(Category.id).where(Category.name == name)
|
||
)).scalar_one_or_none()
|
||
if cid:
|
||
return cid
|
||
c = Category(name=name, sort=99, status=1)
|
||
db.add(c)
|
||
await db.flush()
|
||
return c.id
|
||
|
||
|
||
async def fetch_playlist(source: CollectionSource, drama_id: int, episode_id: int,
|
||
episode_no: int | None = None) -> str:
|
||
"""换取某集的最新 m3u8 文本（签名有时效，每次播放现取；token 失效自动重登）"""
|
||
for attempt in (1, 2):
|
||
try:
|
||
return await _fetch_playlist_once(
|
||
source, drama_id, episode_id, episode_no, force=attempt == 2)
|
||
except Exception:
|
||
_TOKENS.pop(int(source.id or 0), None)
|
||
if attempt == 2:
|
||
raise
|
||
logger.warning("91 播放列表获取失败，重新登录后重试一次")
|
||
await asyncio.sleep(0.5)
|
||
raise RuntimeError("播放列表获取失败")
|
||
|
||
|
||
async def _fetch_playlist_once(
|
||
source: CollectionSource, drama_id: int, episode_id: int,
|
||
episode_no: int | None, force: bool,
|
||
) -> str:
|
||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||
oauth_id, token = await open_session(client, source, force=force)
|
||
|
||
target: dict | None = None
|
||
# 有集号就直接算所在页；否则从第 1 页起扫（最多 5 页兜底）
|
||
pages: list[int] = []
|
||
if episode_no and episode_no > 0:
|
||
pages.append((episode_no - 1) // MAX_PAGE_LIMIT + 1)
|
||
pages.extend(p for p in (1, 2, 3, 4, 5) if p not in pages)
|
||
|
||
for page in pages:
|
||
part = await fetch_episodes(
|
||
client, source.list_url, oauth_id, token, drama_id, page=page)
|
||
for ep in part["list"]:
|
||
if int(ep.get("id") or 0) == episode_id:
|
||
target = ep
|
||
break
|
||
if target or not part["has_more"]:
|
||
break
|
||
|
||
if not target:
|
||
raise RuntimeError("源站未找到该集")
|
||
|
||
m3u8 = target.get("full_m3u8") or ""
|
||
if not m3u8:
|
||
raise RuntimeError("该集暂无可播放地址（可能需付费或未解锁）")
|
||
|
||
resp = await client.get(
|
||
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
|
||
resp.raise_for_status()
|
||
if not resp.text.lstrip().startswith("#EXTM3U"):
|
||
raise RuntimeError("源站播放列表异常")
|
||
return resp.text
|
||
|
||
|
||
async def get_collection_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
|
||
"""取 91 采集源（播放代理用）"""
|
||
stmt = select(CollectionSource).where(CollectionSource.kind == KIND, CollectionSource.status == 1)
|
||
if source_id:
|
||
stmt = stmt.where(CollectionSource.id == source_id)
|
||
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
|
||
if not row:
|
||
raise RuntimeError("未配置 91 采集源")
|
||
return row
|
||
|
||
|
||
async def get_vlog_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
|
||
"""取 91 短视频采集源（播放代理用）"""
|
||
stmt = select(CollectionSource).where(
|
||
CollectionSource.kind == VLOG_KIND, CollectionSource.status == 1)
|
||
if source_id:
|
||
stmt = stmt.where(CollectionSource.id == source_id)
|
||
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
|
||
if not row:
|
||
raise RuntimeError("未配置 91 短视频采集源")
|
||
return row
|
||
|
||
|
||
# ---------- vlog 入库 ----------
|
||
|
||
async def import_vlog_video(
|
||
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
|
||
item: dict, oauth_id: str, token: str,
|
||
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
|
||
) -> bool:
|
||
"""导入单条 vlog 短视频，返回是否新建"""
|
||
vid = int(item.get("id") or 0)
|
||
if not vid:
|
||
return False
|
||
api_url = api_url or source.list_url
|
||
cat = category_id if category_id is not None else source.category_id
|
||
sid = source_id if source_id is not None else source.source_id
|
||
|
||
record = (await db.execute(select(CollectionRecord).where(
|
||
CollectionRecord.source_key == VLOG_SOURCE_KEY,
|
||
CollectionRecord.source_id == vid,
|
||
))).scalar_one_or_none()
|
||
if record:
|
||
v = await db.get(Video, record.video_id) if record.video_id else None
|
||
if v:
|
||
return False
|
||
|
||
# 详情取 playUrl + tags
|
||
detail = item
|
||
try:
|
||
full = await fetch_vlog_detail(client, api_url, oauth_id, token, vid)
|
||
if full:
|
||
detail = {**item, **full}
|
||
except Exception:
|
||
logger.warning("vlog %s 详情获取失败，使用列表字段", vid)
|
||
|
||
cover = proxied_cover(_pick_url(detail.get("thumbImg") or detail.get("cover_thumb") or ""))
|
||
title = (detail.get("title") or f"短视频{vid}")[:100]
|
||
tags = list(detail.get("tags") or [])
|
||
from app.services.title_tags import extract_keywords
|
||
|
||
tags += extract_keywords(title)
|
||
now = datetime.now()
|
||
|
||
video = Video(
|
||
user_id=AUTHOR_USER_ID,
|
||
category_id=cat,
|
||
source_id=sid,
|
||
title=title,
|
||
description=(str(detail.get("descriptions") or "")[:500]),
|
||
cover_url=cover[:500],
|
||
video_url=_vlog_play_url(vid),
|
||
duration=int(detail.get("duration") or 0),
|
||
status=2,
|
||
like_count=_parse_count(detail.get("like")),
|
||
favorite_count=0,
|
||
comment_count=_parse_count(detail.get("comment")),
|
||
view_count=_parse_count(detail.get("play_count")),
|
||
published_at=now,
|
||
)
|
||
db.add(video)
|
||
await db.flush()
|
||
|
||
if tags:
|
||
tag_ids = await _upsert_tags(db, tags)
|
||
for tid in tag_ids:
|
||
db.add(VideoTag(video_id=video.id, tag_id=tid))
|
||
|
||
if record:
|
||
record.video_id = video.id
|
||
else:
|
||
db.add(CollectionRecord(source_key=VLOG_SOURCE_KEY, source_id=vid, video_id=video.id))
|
||
return True
|
||
|
||
|
||
async def sync_vlog_source(
|
||
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
|
||
should_cancel=None, on_progress=None,
|
||
) -> dict:
|
||
"""vlog 短视频翻页增量采集。should_cancel 返回 True 时每页/每条边界退出；
|
||
on_progress 在每页拉到列表后回调 dict(stat)（用于后台展示进度）"""
|
||
cfg = load_config(source)
|
||
# 每条/每页之间的间隔可由采集源 config 的 delay 自定义（秒），未配置时用默认值
|
||
if cfg.get("delay") is not None:
|
||
delay = float(cfg["delay"])
|
||
stat: dict = {
|
||
"kind": VLOG_KIND, "videos_new": 0, "videos_exist": 0,
|
||
"pages": 0,
|
||
}
|
||
page_size = min(int(cfg.get("page_size") or VLOG_MAX_LIMIT), VLOG_MAX_LIMIT)
|
||
|
||
def _cancelled() -> bool:
|
||
return bool(should_cancel and should_cancel())
|
||
|
||
for attempt in (1, 2):
|
||
try:
|
||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||
oauth_id, token = await open_session(client, source, force=attempt == 2)
|
||
# 缓存源基本字段，避免 rollback 后 ORM 对象过期导致属性访问失败
|
||
api_url = source.list_url
|
||
category_id = source.category_id
|
||
source_id = source.source_id
|
||
page = 1
|
||
while True:
|
||
if _cancelled():
|
||
logger.info("91 vlog 同步被取消（第 %s 页前）", page)
|
||
return stat
|
||
data = await fetch_vlog_list(
|
||
client, api_url, oauth_id, token, page, page_size)
|
||
stat["pages"] = page
|
||
items = data["list"]
|
||
logger.info("91 vlog 第 %s 页：%s 条", page, len(items))
|
||
if on_progress:
|
||
on_progress(dict(stat))
|
||
|
||
for item in items:
|
||
if _cancelled():
|
||
logger.info("91 vlog 同步被取消（处理 item 中）")
|
||
return stat
|
||
vid = int(item.get("id") or 0)
|
||
if not vid:
|
||
continue
|
||
try:
|
||
created = await import_vlog_video(
|
||
db, client, source, item, oauth_id, token,
|
||
api_url=api_url, category_id=category_id, source_id=source_id)
|
||
await db.commit()
|
||
if created:
|
||
stat["videos_new"] += 1
|
||
else:
|
||
stat["videos_exist"] += 1
|
||
except Exception:
|
||
await db.rollback()
|
||
logger.exception("导入 91 vlog %s 失败，跳过", vid)
|
||
await asyncio.sleep(delay)
|
||
|
||
if not data["has_more"] or not items:
|
||
break
|
||
if max_pages and page >= max_pages:
|
||
break
|
||
page += 1
|
||
await asyncio.sleep(delay)
|
||
|
||
cfg["oauth_id"], cfg["token"] = oauth_id, token
|
||
source.config = json.dumps(cfg, ensure_ascii=False)
|
||
break
|
||
except Exception:
|
||
_TOKENS.pop(int(source.id or 0), None)
|
||
if attempt == 2:
|
||
source.last_sync_at = datetime.now()
|
||
await db.commit()
|
||
raise
|
||
logger.warning("91 vlog 同步失败，重新登录后重试一次")
|
||
await asyncio.sleep(1)
|
||
|
||
source.last_sync_at = datetime.now()
|
||
await db.commit()
|
||
return stat
|
||
|
||
|
||
async def fetch_vlog_playlist(source: CollectionSource, vid: int) -> str:
|
||
"""换取 vlog 播放列表（playUrl 带时效签名，播放时现取）"""
|
||
for attempt in (1, 2):
|
||
try:
|
||
return await _fetch_vlog_playlist_once(source, vid, force=attempt == 2)
|
||
except Exception:
|
||
_TOKENS.pop(int(source.id or 0), None)
|
||
if attempt == 2:
|
||
raise
|
||
logger.warning("91 vlog 播放列表获取失败，重新登录后重试一次")
|
||
await asyncio.sleep(0.5)
|
||
raise RuntimeError("播放列表获取失败")
|
||
|
||
|
||
async def _fetch_vlog_playlist_once(source: CollectionSource, vid: int, force: bool) -> str:
|
||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||
oauth_id, token = await open_session(client, source, force=force)
|
||
detail = await fetch_vlog_detail(client, source.list_url, oauth_id, token, vid)
|
||
m3u8 = detail.get("playUrl") or ""
|
||
if not m3u8:
|
||
raise RuntimeError("该视频暂无可播放地址（可能需付费或未解锁）")
|
||
resp = await client.get(
|
||
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
|
||
resp.raise_for_status()
|
||
if not resp.text.lstrip().startswith("#EXTM3U"):
|
||
raise RuntimeError("源站播放列表异常")
|
||
return resp.text
|