This commit is contained in:
lz_db
2026-10-25 11:46:44 +08:00
parent 893dfed0ed
commit 494f667719
63 changed files with 5213 additions and 239 deletions
+20 -3
View File
@@ -155,7 +155,7 @@ async def dashboard(db: AsyncSession) -> dict:
# ---------- 视频审核 / 管理 ----------
def _video_item(video: Video, author: User | None) -> dict:
def _video_item(video: Video, author: User | None, source_name: str = "") -> dict:
return {
"video_id": video.id,
"title": video.title,
@@ -164,6 +164,8 @@ def _video_item(video: Video, author: User | None) -> dict:
"video_url": video.video_url,
"duration": video.duration,
"category_id": video.category_id or 0,
"source_id": video.source_id or 0,
"source_name": source_name,
"series_id": video.series_id,
"episode_no": video.episode_no,
"status": video.status,
@@ -207,7 +209,7 @@ async def list_pending_videos(db: AsyncSession, page: int, page_size: int) -> di
async def list_videos(
db: AsyncSession, keyword: str | None, status: int | None, author_id: int | None,
page: int, page_size: int,
page: int, page_size: int, source_id: int | None = None,
) -> dict:
"""视频列表(11.25)"""
stmt = select(Video)
@@ -218,12 +220,27 @@ async def list_videos(
stmt = stmt.where(Video.status == status)
if author_id is not None:
stmt = stmt.where(Video.user_id == author_id)
if source_id is not None:
stmt = stmt.where(Video.source_id == source_id)
total = (await db.execute(select(func.count()).select_from(stmt.subquery()))).scalar() or 0
rows = (await db.execute(
stmt.order_by(Video.id.desc()).offset((page - 1) * page_size).limit(page_size)
)).scalars().all()
authors = await _authors_map(db, rows)
return {"total": total, "list": [_video_item(v, authors.get(v.user_id)) for v in rows]}
names = await source_names_map(db, {v.source_id for v in rows})
return {"total": total,
"list": [_video_item(v, authors.get(v.user_id), names.get(v.source_id, "")) for v in rows]}
async def source_names_map(db: AsyncSession, ids: set[int]) -> dict[int, str]:
"""来源 id → 名称"""
from app.models.video import Source
ids = {i for i in ids if i}
if not ids:
return {}
rows = (await db.execute(select(Source.id, Source.name).where(Source.id.in_(ids)))).all()
return {r.id: r.name for r in rows}
async def review_video(
+850
View File
@@ -0,0 +1,850 @@
"""91 短剧(AI短剧)采集:加密接口客户端 + 系列/集数入库
上游协议:pwa.php 单入口,AES-256-CFB(segment 128) 加密请求体 + MD5(SHA256) 签名。
时间戳签名有效期约 4 小时,因此**不存原始 m3u8 地址**,改为存本站代理地址,
播放时再向上游换取新鲜签名(见 /api/v1/m3u8/aidrama)。
"""
import asyncio
import hashlib
import json
import logging
import random
import string
import time
from datetime import datetime
import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.video import (
Category,
CollectionRecord,
CollectionSource,
Series,
Tag,
Video,
VideoTag,
)
logger = logging.getLogger(__name__)
KIND = "aidrama91"
VLOG_KIND = "aidrama91_vlog" # 91 非系列短视频
SOURCE_KEY = "91aidrama" # collection_records.source_key
VLOG_SOURCE_KEY = "91vlog" # 短视频采集记录 key
SOURCE_CODE = "91" # sources.code
SELF_API_BASE = "https://vapi.256kb.top"
AUTHOR_USER_ID = 1
VLOG_CATEGORY_NAME = "91短视频" # vlog 无真实分类,归到本地分类
# 协议常量(前端 JS 逆向)
AES_KEY = b"cc88ddc9357ff461e08f047aedee692b"
AES_IV = b"e89225cfbbimgkcu"
SIGN_KEY = "cc88ddc9357ff461e08f047aedee692b"
APP_VERSION = "7.5.0"
CLIENT_ORIGIN = "https://pwa4.jl8v0p5t.cc"
# 封面图解密:CDN 返回的是 AES-128-CBC 加密的图片字节(无 padding),
# 源站前端 DecryptImage 解密后展示 —— 本站通过 /api/v1/img/aidrama 代理解密
IMG_AES_KEY = b"f5d965df75336270"
IMG_AES_IV = b"97b60394abc2fbe1"
IMG_HOST_SUFFIX = ".drleao.cn" # 封面 CDN 域名白名单(防 SSRF)
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
),
"Origin": CLIENT_ORIGIN,
"Referer": f"{CLIENT_ORIGIN}/",
"Accept": "application/json, text/plain, */*",
"Content-Type": "application/x-www-form-urlencoded",
}
OAUTH_CHARSET = string.ascii_letters + string.digits
MAX_PAGE_LIMIT = 20 # 上游单页上限(传 100 实际只返回 20)
# ---------- 加密 / 签名 ----------
def _cfb(key: bytes, iv: bytes):
try:
from cryptography.hazmat.decrepit.ciphers.modes import CFB
except ImportError: # cryptography < 43
from cryptography.hazmat.primitives.ciphers.modes import CFB
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
return Cipher(algorithms.AES(key), CFB(iv))
def _encrypt(plaintext: str) -> str:
op = _cfb(AES_KEY, AES_IV).encryptor()
return (op.update(plaintext.encode()) + op.finalize()).hex().upper()
def _decrypt(hex_data: str) -> str:
op = _cfb(AES_KEY, AES_IV).decryptor()
return (op.update(bytes.fromhex(hex_data)) + op.finalize()).decode("utf-8", "replace")
def _new_oauth_id() -> str:
return "".join(random.choices(OAUTH_CHARSET, k=16)) + str(int(time.time() * 1000))
async def _post(client: httpx.AsyncClient, api_url: str, payload: dict) -> dict:
"""加密并发送,返回解密后的业务数据(已剥掉外层包装)"""
ts_ms = int(time.time() * 1000)
payload = dict(payload, timestamp=ts_ms)
data_hex = _encrypt(json.dumps(payload, separators=(",", ":")))
ts_sec = ts_ms // 1000
raw = f"client=pwa&data={data_hex}&timestamp={ts_sec}{SIGN_KEY}"
sign = hashlib.md5(hashlib.sha256(raw.encode()).hexdigest().encode()).hexdigest()
resp = await client.post(
api_url,
content=f"client=pwa&timestamp={ts_sec}&data={data_hex}&sign={sign}",
headers=HEADERS,
timeout=30,
)
if resp.status_code != 200:
raise RuntimeError(f"上游 HTTP {resp.status_code}(oauth_id 可能未注册或已被限流)")
body = resp.json()
if body.get("errcode") not in (0, None):
raise RuntimeError(f"上游错误: {str(body)[:200]}")
payload_text = body.get("data")
if not payload_text:
return body
return json.loads(_decrypt(payload_text))
def _base(oauth_id: str, token: str) -> dict:
return {
"oauth_type": "pwa", "oauth_id": oauth_id, "trace_id": "",
"aff_x_code": "", "version": APP_VERSION, "token": token,
}
# ---------- 会话 ----------
async def create_session(
client: httpx.AsyncClient, api_url: str, account: str, password: str
) -> tuple[str, str]:
"""注册 oauth_id 并登录,返回 (oauth_id, token)"""
oauth_id = _new_oauth_id()
# 上游对未注册的 oauth_id 直接 503,必须先 system/index
await _post(client, api_url, dict(_base(oauth_id, ""), mod="system", code="index", data={}))
result = await _post(client, api_url, dict(
_base(oauth_id, ""), mod="user", code="regOrLogin",
account=account, password=password, type="login",
))
inner = result.get("data", result) if isinstance(result, dict) else {}
token = inner.get("token") or ""
if not token:
raise RuntimeError(f"91 登录失败: {inner.get('msg') or str(result)[:150]}")
return oauth_id, token
# 进程内 token 缓存(2 个 uvicorn worker → 最多 2 次登录);失效时自动重登
_TOKENS: dict[int, tuple[str, str, float]] = {}
TOKEN_TTL = 3600
async def open_session(
client: httpx.AsyncClient, source: CollectionSource, force: bool = False
) -> tuple[str, str]:
"""获取可用的 (oauth_id, token):进程缓存 → DB 配置 → 重新登录"""
key = int(source.id or 0)
cfg = load_config(source)
if not force:
hit = _TOKENS.get(key)
if hit and hit[2] > time.time():
return hit[0], hit[1]
if cfg.get("oauth_id") and cfg.get("token"):
_TOKENS[key] = (cfg["oauth_id"], cfg["token"], time.time() + TOKEN_TTL)
return cfg["oauth_id"], cfg["token"]
oauth_id, token = await create_session(
client, source.list_url, cfg["account"], cfg["password"])
_TOKENS[key] = (oauth_id, token, time.time() + TOKEN_TTL)
return oauth_id, token
def load_config(source: CollectionSource) -> dict:
try:
cfg = json.loads(source.config or "{}")
except json.JSONDecodeError:
cfg = {}
cfg.setdefault("account", "")
cfg.setdefault("password", "")
cfg.setdefault("category_ids", [4])
cfg.setdefault("page_size", MAX_PAGE_LIMIT)
return cfg
# ---------- 业务接口 ----------
async def fetch_list(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
page: int, limit: int, category_ids: list[int],
) -> dict:
"""短剧列表;返回 {list, total, has_more}"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="aidrama", code="home",
page=page, limit=min(limit, MAX_PAGE_LIMIT), pay_type="all", sort="all",
category_ids=category_ids or [4],
))
inner = data.get("data") or {}
return {
"list": inner.get("list") or [],
"total": int(inner.get("total") or 0),
"has_more": bool(inner.get("has_more")),
}
async def fetch_detail(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
) -> dict:
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="aidrama", code="detail", drama_id=drama_id,
))
return (data.get("data") or {}).get("row") or {}
async def fetch_episodes(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
drama_id: int, page: int = 1, limit: int = MAX_PAGE_LIMIT,
) -> dict:
"""集数列表;返回 {list, has_more}"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="aidrama", code="episodes",
drama_id=drama_id, page=page, limit=min(limit, MAX_PAGE_LIMIT),
))
inner = data.get("data") or {}
return {"list": inner.get("list") or [], "has_more": bool(inner.get("has_more"))}
async def fetch_all_episodes(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
) -> list[dict]:
"""翻页取全部集数(每页 20 上限)"""
out: list[dict] = []
page = 1
while True:
part = await fetch_episodes(client, api_url, oauth_id, token, drama_id, page=page)
out.extend(part["list"])
if not part["has_more"] or not part["list"]:
break
page += 1
return out
# ---------- vlog 短视频接口 ----------
VLOG_MAX_LIMIT = 50 # 实测 vlog 单页上限 50(传 100 只返回 50)
async def fetch_vlog_list(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
page: int, limit: int = VLOG_MAX_LIMIT, vlog_type: str = "new",
) -> dict:
"""vlog 列表(非系列短视频);返回 {list, has_more}"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="choice", code="vlog",
type=vlog_type, page=page, limit=min(limit, VLOG_MAX_LIMIT),
))
lst = data.get("data") or []
if not isinstance(lst, list):
lst = []
return {"list": lst, "has_more": len(lst) >= min(limit, VLOG_MAX_LIMIT)}
async def fetch_vlog_detail(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, vid: int
) -> dict:
"""vlog 详情(含 playUrl / tags)"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="index", code="detail", id=vid))
return data.get("data") or {}
# ---------- 字段解析 ----------
def _pick_url(raw: str | dict, prefer: str = "720") -> str:
"""封面字段是 JSON 字符串或 dict:{"ori","360","720"},取指定档位"""
if not raw:
return ""
try:
obj = json.loads(raw) if isinstance(raw, str) else raw
except json.JSONDecodeError:
return raw if isinstance(raw, str) else ""
if not isinstance(obj, dict):
return ""
return obj.get(prefer) or obj.get("ori") or ""
def proxied_cover(url: str) -> str:
"""封面地址改写为本站解密代理(CDN 图是加密的,直连无法显示)"""
if not url or "/api/v1/img/aidrama" in url:
return url
if not url.startswith(("http://", "https://")):
return url
from urllib.parse import quote
return f"{SELF_API_BASE}/api/v1/img/aidrama?u={quote(url, safe='')}"
def decrypt_image(data: bytes) -> bytes:
"""AES-128-CBC(NoPadding) 解密 CDN 加密图片,返回原始 JPEG/PNG 字节"""
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
op = Cipher(algorithms.AES(IMG_AES_KEY), modes.CBC(IMG_AES_IV)).decryptor()
return op.update(data) + op.finalize()
def _play_url(drama_id: int, episode_id: int, episode_no: int | None) -> str:
"""本站代理地址(含 m3u8 字样,App 端会走本地缓存代理)"""
url = f"{SELF_API_BASE}/api/v1/m3u8/aidrama?d={drama_id}&e={episode_id}"
if episode_no:
url += f"&n={episode_no}"
return url
def _safe_dt(ts) -> datetime | None:
try:
n = int(ts)
except (TypeError, ValueError):
return None
return datetime.fromtimestamp(n) if n > 0 else None
def _parse_count(v) -> int:
"""解析播放/点赞等计数,兼容 '143.7万' / '1.2万' / '34' / 34"""
if v is None:
return 0
if isinstance(v, (int, float)):
return int(v)
s = str(v).strip()
if not s:
return 0
if s.endswith("万"):
try:
return int(float(s[:-1]) * 10000)
except ValueError:
return 0
try:
return int(s)
except ValueError:
return 0
def _vlog_play_url(vid: int) -> str:
"""vlog 单视频播放代理地址(含 m3u8 字样,App 走本地缓存)"""
return f"{SELF_API_BASE}/api/v1/m3u8/vlog?vid={vid}"
# ---------- 入库 ----------
async def _upsert_tags(db: AsyncSession, names: list[str]) -> list[int]:
"""按名称取/建标签,返回 tag id 列表"""
ids: list[int] = []
for name in {n.strip() for n in names if n and n.strip()}:
name = name[:20]
tid = (await db.execute(select(Tag.id).where(Tag.name == name))).scalar_one_or_none()
if not tid:
t = Tag(name=name, video_count=0)
db.add(t)
await db.flush()
tid = t.id
ids.append(tid)
return ids
async def import_drama(
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
drama: dict, oauth_id: str, token: str,
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
) -> tuple[int, int]:
"""导入单部短剧(系列 + 集数视频),返回 (新增集数, 跳过集数)"""
drama_id = int(drama.get("id") or 0)
if not drama_id:
return 0, 0
api_url = api_url or source.list_url
cat = category_id if category_id is not None else source.category_id
sid = source_id if source_id is not None else source.source_id
cfg = load_config(source)
detail = drama
try:
full = await fetch_detail(client, api_url, oauth_id, token, drama_id)
if full:
detail = {**drama, **full}
except Exception:
logger.warning("短剧 %s 详情获取失败,使用列表字段", drama_id)
record = (await db.execute(
select(CollectionRecord).where(
CollectionRecord.source_key == SOURCE_KEY,
CollectionRecord.source_id == drama_id,
)
)).scalar_one_or_none()
cover = proxied_cover(_pick_url(detail.get("cover") or ""))
tags = list(detail.get("tag_list") or []) + list(detail.get("category_list") or [])
# 像既有采集一样:按标题拆分关键词作为 tag
from app.services.title_tags import extract_keywords
tags += extract_keywords(detail.get("title") or "")
total_declared = int(detail.get("total_episodes") or 0)
series = await db.get(Series, record.series_id) if record else None
if series is None:
series = Series(
title=(detail.get("title") or f"短剧{drama_id}")[:100],
cover_url=cover[:500],
description=(detail.get("description") or "")[:500],
author_id=AUTHOR_USER_ID,
category_id=cat,
source_id=sid,
total_episodes=total_declared,
status=1,
)
db.add(series)
await db.flush()
if record:
record.series_id = series.id
else:
db.add(CollectionRecord(
source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id))
await db.flush()
else:
# 已有系列:同步标题/封面/简介(源站可能改过)
series.title = (detail.get("title") or series.title)[:100]
series.cover_url = (cover or series.cover_url)[:500]
series.total_episodes = max(series.total_episodes, total_declared)
existing = set((await db.execute(
select(Video.episode_no).where(
Video.series_id == series.id, Video.episode_no.is_not(None))
)).scalars().all())
episodes = await fetch_all_episodes(client, api_url, oauth_id, token, drama_id)
tag_ids: list[int] | None = None
created = skipped = 0
now = datetime.now()
for ep in episodes:
ep_id = int(ep.get("id") or 0)
ep_no = int(ep.get("episode_number") or 0)
if not ep_id or not ep_no or ep_no in existing:
skipped += 1
continue
ep_cover = proxied_cover(_pick_url(ep.get("cover") or "")) or cover
video = Video(
user_id=AUTHOR_USER_ID,
category_id=cat,
source_id=sid,
series_id=series.id,
episode_no=ep_no,
title=(ep.get("title") or f"第{ep_no}集")[:100],
description=(detail.get("description") or "")[:500],
cover_url=ep_cover[:500],
video_url=_play_url(drama_id, ep_id, ep_no),
duration=int(ep.get("duration") or 0),
status=2,
like_count=int(ep.get("like") or 0),
favorite_count=int(ep.get("favorite_num") or 0),
comment_count=int(ep.get("comment_count") or 0),
published_at=_safe_dt(ep.get("created_at")) or now,
)
db.add(video)
await db.flush()
# 标签挂到每集视频上(系列标签来自 tag_list / category_list)
if tags:
if tag_ids is None:
tag_ids = await _upsert_tags(db, tags)
for tid in tag_ids:
db.add(VideoTag(video_id=video.id, tag_id=tid))
existing.add(ep_no)
created += 1
series.total_episodes = max(len(existing), total_declared)
return created, skipped
async def sync_source(
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
should_cancel=None,
) -> dict:
"""按列表翻页增量采集(已导入的短剧跳过,仍会补同步新集数)
should_cancel: 可选回调,返回 True 时在每页/每条边界退出循环(用于手动取消)
"""
cfg = load_config(source)
stat: dict = {
"kind": KIND, "dramas_new": 0, "dramas_exist": 0,
"episodes_created": 0, "episodes_skipped": 0, "pages": 0, "total": 0,
}
page_size = min(int(cfg.get("page_size") or MAX_PAGE_LIMIT), MAX_PAGE_LIMIT)
category_ids = cfg.get("category_ids") or [4]
def _cancelled() -> bool:
return bool(should_cancel and should_cancel())
for attempt in (1, 2):
try:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=attempt == 2)
api_url = source.list_url
cat = source.category_id
sid = source.source_id
page = 1
while True:
if _cancelled():
logger.info("91 短剧同步被取消(第 %s 页前)", page)
return stat
data = await fetch_list(
client, api_url, oauth_id, token, page, page_size, category_ids)
stat["pages"] = page
stat["total"] = data["total"]
items = data["list"]
logger.info("91 短剧第 %s 页:%s 部(共 %s)", page, len(items), data["total"])
for drama in items:
if _cancelled():
logger.info("91 短剧同步被取消(处理 drama 中)")
return stat
drama_id = int(drama.get("id") or 0)
if not drama_id:
continue
try:
created, skipped = await import_drama(
db, client, source, drama, oauth_id, token,
api_url=api_url, category_id=cat, source_id=sid)
await db.commit()
if created > 0:
stat["dramas_new"] += 1
else:
stat["dramas_exist"] += 1
stat["episodes_created"] += created
stat["episodes_skipped"] += skipped
except Exception:
await db.rollback()
logger.exception("导入 91 短剧 %s 失败,跳过", drama_id)
await asyncio.sleep(delay)
if not data["has_more"] or not items:
break
if max_pages and page >= max_pages:
break
page += 1
await asyncio.sleep(delay)
cfg["oauth_id"], cfg["token"] = oauth_id, token
source.config = json.dumps(cfg, ensure_ascii=False)
break
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
source.last_sync_at = datetime.now()
await db.commit()
raise
logger.warning("91 同步失败,重新登录后重试一次")
await asyncio.sleep(1)
source.last_sync_at = datetime.now()
await db.commit()
return stat
async def refresh_source_id(db: AsyncSession, source: CollectionSource) -> int:
"""确保采集源绑定的 sources 记录存在,返回来源 id"""
if source.source_id:
return source.source_id
from app.models.video import Source
sid = (await db.execute(
select(Source.id).where(Source.code == SOURCE_CODE)
)).scalar_one_or_none()
if not sid:
max_sort = 0
s = Source(name="91", code=SOURCE_CODE, sort=max_sort + 1, status=1)
db.add(s)
await db.flush()
sid = s.id
source.source_id = sid
return sid
async def ensure_local_category(db: AsyncSession, name: str) -> int:
"""按采集源名称找/建本站分类"""
cid = (await db.execute(
select(Category.id).where(Category.name == name)
)).scalar_one_or_none()
if cid:
return cid
c = Category(name=name, sort=99, status=1)
db.add(c)
await db.flush()
return c.id
async def fetch_playlist(source: CollectionSource, drama_id: int, episode_id: int,
episode_no: int | None = None) -> str:
"""换取某集的最新 m3u8 文本(签名有时效,每次播放现取;token 失效自动重登)"""
for attempt in (1, 2):
try:
return await _fetch_playlist_once(
source, drama_id, episode_id, episode_no, force=attempt == 2)
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
raise
logger.warning("91 播放列表获取失败,重新登录后重试一次")
await asyncio.sleep(0.5)
raise RuntimeError("播放列表获取失败")
async def _fetch_playlist_once(
source: CollectionSource, drama_id: int, episode_id: int,
episode_no: int | None, force: bool,
) -> str:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=force)
target: dict | None = None
# 有集号就直接算所在页;否则从第 1 页起扫(最多 5 页兜底)
pages: list[int] = []
if episode_no and episode_no > 0:
pages.append((episode_no - 1) // MAX_PAGE_LIMIT + 1)
pages.extend(p for p in (1, 2, 3, 4, 5) if p not in pages)
for page in pages:
part = await fetch_episodes(
client, source.list_url, oauth_id, token, drama_id, page=page)
for ep in part["list"]:
if int(ep.get("id") or 0) == episode_id:
target = ep
break
if target or not part["has_more"]:
break
if not target:
raise RuntimeError("源站未找到该集")
m3u8 = target.get("full_m3u8") or ""
if not m3u8:
raise RuntimeError("该集暂无可播放地址(可能需付费或未解锁)")
resp = await client.get(
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
resp.raise_for_status()
if not resp.text.lstrip().startswith("#EXTM3U"):
raise RuntimeError("源站播放列表异常")
return resp.text
async def get_collection_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
"""取 91 采集源(播放代理用)"""
stmt = select(CollectionSource).where(CollectionSource.kind == KIND, CollectionSource.status == 1)
if source_id:
stmt = stmt.where(CollectionSource.id == source_id)
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
if not row:
raise RuntimeError("未配置 91 采集源")
return row
async def get_vlog_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
"""取 91 短视频采集源(播放代理用)"""
stmt = select(CollectionSource).where(
CollectionSource.kind == VLOG_KIND, CollectionSource.status == 1)
if source_id:
stmt = stmt.where(CollectionSource.id == source_id)
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
if not row:
raise RuntimeError("未配置 91 短视频采集源")
return row
# ---------- vlog 入库 ----------
async def import_vlog_video(
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
item: dict, oauth_id: str, token: str,
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
) -> bool:
"""导入单条 vlog 短视频,返回是否新建"""
vid = int(item.get("id") or 0)
if not vid:
return False
api_url = api_url or source.list_url
cat = category_id if category_id is not None else source.category_id
sid = source_id if source_id is not None else source.source_id
record = (await db.execute(select(CollectionRecord).where(
CollectionRecord.source_key == VLOG_SOURCE_KEY,
CollectionRecord.source_id == vid,
))).scalar_one_or_none()
if record:
v = await db.get(Video, record.video_id) if record.video_id else None
if v:
return False
# 详情取 playUrl + tags
detail = item
try:
full = await fetch_vlog_detail(client, api_url, oauth_id, token, vid)
if full:
detail = {**item, **full}
except Exception:
logger.warning("vlog %s 详情获取失败,使用列表字段", vid)
cover = proxied_cover(_pick_url(detail.get("thumbImg") or detail.get("cover_thumb") or ""))
title = (detail.get("title") or f"短视频{vid}")[:100]
tags = list(detail.get("tags") or [])
from app.services.title_tags import extract_keywords
tags += extract_keywords(title)
now = datetime.now()
video = Video(
user_id=AUTHOR_USER_ID,
category_id=cat,
source_id=sid,
title=title,
description=(str(detail.get("descriptions") or "")[:500]),
cover_url=cover[:500],
video_url=_vlog_play_url(vid),
duration=int(detail.get("duration") or 0),
status=2,
like_count=_parse_count(detail.get("like")),
favorite_count=0,
comment_count=_parse_count(detail.get("comment")),
view_count=_parse_count(detail.get("play_count")),
published_at=now,
)
db.add(video)
await db.flush()
if tags:
tag_ids = await _upsert_tags(db, tags)
for tid in tag_ids:
db.add(VideoTag(video_id=video.id, tag_id=tid))
if record:
record.video_id = video.id
else:
db.add(CollectionRecord(source_key=VLOG_SOURCE_KEY, source_id=vid, video_id=video.id))
return True
async def sync_vlog_source(
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
should_cancel=None,
) -> dict:
"""vlog 短视频翻页增量采集。should_cancel 返回 True 时每页/每条边界退出"""
cfg = load_config(source)
stat: dict = {
"kind": VLOG_KIND, "videos_new": 0, "videos_exist": 0,
"pages": 0,
}
page_size = min(int(cfg.get("page_size") or VLOG_MAX_LIMIT), VLOG_MAX_LIMIT)
def _cancelled() -> bool:
return bool(should_cancel and should_cancel())
for attempt in (1, 2):
try:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=attempt == 2)
# 缓存源基本字段,避免 rollback 后 ORM 对象过期导致属性访问失败
api_url = source.list_url
category_id = source.category_id
source_id = source.source_id
page = 1
while True:
if _cancelled():
logger.info("91 vlog 同步被取消(第 %s 页前)", page)
return stat
data = await fetch_vlog_list(
client, api_url, oauth_id, token, page, page_size)
stat["pages"] = page
items = data["list"]
logger.info("91 vlog 第 %s 页:%s 条", page, len(items))
for item in items:
if _cancelled():
logger.info("91 vlog 同步被取消(处理 item 中)")
return stat
vid = int(item.get("id") or 0)
if not vid:
continue
try:
created = await import_vlog_video(
db, client, source, item, oauth_id, token,
api_url=api_url, category_id=category_id, source_id=source_id)
await db.commit()
if created:
stat["videos_new"] += 1
else:
stat["videos_exist"] += 1
except Exception:
await db.rollback()
logger.exception("导入 91 vlog %s 失败,跳过", vid)
await asyncio.sleep(delay)
if not data["has_more"] or not items:
break
if max_pages and page >= max_pages:
break
page += 1
await asyncio.sleep(delay)
cfg["oauth_id"], cfg["token"] = oauth_id, token
source.config = json.dumps(cfg, ensure_ascii=False)
break
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
source.last_sync_at = datetime.now()
await db.commit()
raise
logger.warning("91 vlog 同步失败,重新登录后重试一次")
await asyncio.sleep(1)
source.last_sync_at = datetime.now()
await db.commit()
return stat
async def fetch_vlog_playlist(source: CollectionSource, vid: int) -> str:
"""换取 vlog 播放列表(playUrl 带时效签名,播放时现取)"""
for attempt in (1, 2):
try:
return await _fetch_vlog_playlist_once(source, vid, force=attempt == 2)
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
raise
logger.warning("91 vlog 播放列表获取失败,重新登录后重试一次")
await asyncio.sleep(0.5)
raise RuntimeError("播放列表获取失败")
async def _fetch_vlog_playlist_once(source: CollectionSource, vid: int, force: bool) -> str:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=force)
detail = await fetch_vlog_detail(client, source.list_url, oauth_id, token, vid)
m3u8 = detail.get("playUrl") or ""
if not m3u8:
raise RuntimeError("该视频暂无可播放地址(可能需付费或未解锁)")
resp = await client.get(
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
resp.raise_for_status()
if not resp.text.lstrip().startswith("#EXTM3U"):
raise RuntimeError("源站播放列表异常")
return resp.text
+35 -2
View File
@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
# 内容源配置(域名轮换时只需改这里)
SOURCE_KEY = "mdcmai4"
SOURCE_CODE = "madou" # sources 表中的 code,用于给导入内容打「来源」标
UPSTREAM = "https://mdcmai4.xyz"
PRODUCT_ID = 1
CATEGORY_ID = 12 # 本站分类:AI短剧
@@ -37,6 +38,21 @@ def _proxy_url(path: str) -> str:
return f"{SELF_API_BASE}/api/v1/m3u8/proxy?path={quote(path, safe='')}"
async def default_source_id(db: AsyncSession) -> int:
"""麻豆等内容源的来源 id(sources.code 匹配;缺失时取第一个启用来源)"""
from app.models.video import Source
sid = (
await db.execute(select(Source.id).where(Source.code == SOURCE_CODE))
).scalar_one_or_none()
if sid:
return sid
sid = (
await db.execute(select(Source.id).where(Source.status == 1).order_by(Source.sort, Source.id))
).scalars().first()
return sid or 0
async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: int) -> tuple[int, int]:
"""导入单部短剧,返回 (新增集数, 跳过集数)"""
detail = await _fetch_json(client, f"{UPSTREAM}/api/v1/short-dramas/{drama_id}", {"productId": PRODUCT_ID})
@@ -63,6 +79,7 @@ async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: i
description=(detail.get("description") or "")[:500],
author_id=AUTHOR_USER_ID,
category_id=CATEGORY_ID,
source_id=await default_source_id(db),
total_episodes=0,
view_count=int(detail.get("heatCount") or 0),
status=1,
@@ -94,6 +111,7 @@ async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: i
Video(
user_id=AUTHOR_USER_ID,
category_id=CATEGORY_ID,
source_id=series.source_id,
series_id=series.id,
episode_no=ep_no,
title=(ep.get("titleOverride") or ep.get("title") or f"第{ep_no}集")[:100],
@@ -209,6 +227,7 @@ async def _import_video_item(db: AsyncSession, source: CollectionSource, item: d
v = Video(
user_id=AUTHOR_USER_ID,
category_id=source.category_id,
source_id=source.source_id or await default_source_id(db),
title=(item.get("title") or f"视频{vid}")[:100],
description=(item.get("description") or "")[:500],
cover_url=_abs_cover(item)[:500],
@@ -232,13 +251,23 @@ async def sync_source(
max_pages: int = 0,
stop_after_existing: int = 30,
delay: float = 0.25,
should_cancel=None,
) -> dict:
"""按源的 URL 模板翻页采集;连续 stop_after_existing 条已存在则提前停止(增量)"""
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False}
"""按源的 URL 模板翻页采集;连续 stop_after_existing 条已存在则提前停止(增量)。
should_cancel 返回 True 时每页/每条边界退出(用于手动取消)"""
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False, "cancelled": False}
consecutive_existing = 0
def _cancelled() -> bool:
return bool(should_cancel and should_cancel())
async with httpx.AsyncClient(follow_redirects=True) as client:
page = 1
while True:
if _cancelled():
stat["cancelled"] = True
logger.info("通用采集源 %s 同步被取消(第 %s 页前)", source.id, page)
return stat
url = source.list_url.replace("{{page}}", str(page))
data = await _fetch_json(client, url)
items = data.get("items", [])
@@ -246,6 +275,10 @@ async def sync_source(
break
stat["pages"] = page
for item in items:
if _cancelled():
stat["cancelled"] = True
logger.info("通用采集源 %s 同步被取消(处理 item 中)", source.id)
return stat
if await _import_video_item(db, source, item):
stat["created"] += 1
consecutive_existing = 0
+62
View File
@@ -0,0 +1,62 @@
"""标题关键词提取:从视频/系列标题拆分出词作为 tag。
提取规则(原 scripts/build_tags_from_titles.py,抽成公共模块供采集入库复用):
1. 番号(如 CAWD-601)及其系列前缀(CAWD)
2. 书名号/方括号内容(≤12 字,去掉「无码破解」「中文字幕」等标记词)
3. jieba TF-IDF 关键词 topK=6(过滤停用词、纯数字、单字)
"""
import re
import jieba.analyse
STOP = set(
"第 集 部 篇 全集 完结 中文字幕 无码 破解 无码破解 字幕 高清 最新 精选 视频 在线 观看 "
"一个 一起 没有 为了 开始 结果 只能 直接 就是 没想到 真爽".split()
)
CODE_RE = re.compile(r"[A-Za-z]{2,6}-\d{2,4}")
EPISODE_RE = re.compile(r"第[0-9一二三四五六七八九十百千零]+[集部期话卷章]")
PUNCT_RE = re.compile(r"[《》「」『』\[\]【】()()_\-::,,.。!!??…、/\\|~~\s]+")
BRACKET_RE = re.compile(r"[《「『\[【((]([^《》「」『』\[\]【】()()]{2,30})[》」』\]】))]")
def extract_keywords(title: str) -> list[str]:
if not title:
return []
kws: list[str] = []
# 1. 番号 + 系列前缀
for code in CODE_RE.findall(title.upper()):
kws.append(code)
prefix = code.split("-")[0]
if len(prefix) >= 2:
kws.append(prefix)
# 2. 书名号/括号内容(剧名/主题)
for inner in BRACKET_RE.findall(title):
inner = inner.strip()
if 2 <= len(inner) <= 12 and inner not in STOP and not CODE_RE.search(inner):
kws.append(inner)
# 3. jieba TF-IDF
t = EPISODE_RE.sub(" ", title)
t = PUNCT_RE.sub(" ", t)
for w in jieba.analyse.extract_tags(t, topK=6):
w = w.strip()
if not (2 <= len(w) <= 12):
continue
if w in STOP or re.fullmatch(r"\d+", w):
continue
kws.append(w)
# 去重保序(按 casefold 键,避免 MySQL 不区分大小写的唯一键冲突)
seen: set[str] = set()
out: list[str] = []
for k in kws:
if k.isascii():
k = k.upper()
key = k.casefold()
if key not in seen:
seen.add(key)
out.append(k)
return out
+141 -11
View File
@@ -20,7 +20,7 @@ from app.models.billing import WatchRecord
from app.models.incentive import VideoEarning
from app.models.interaction import Favorite, Like
from app.models.user import User
from app.models.video import Category, Tag, Video, VideoTag
from app.models.video import Category, Series, Tag, Video, VideoTag
from app.schemas.video import PublishVideoReq, UploadTokenReq
from app.services import membership_service, sensitive_service, storage_service, wallet_service
@@ -79,6 +79,18 @@ async def serialize_videos(
tags_map = await get_tags_map(db, video_ids)
# 内容来源(麻豆 / 91 ...)
from app.models.video import Source
source_ids = {v.source_id for v in videos if v.source_id}
source_map = {}
if source_ids:
source_map = {
r.id: r.name for r in (
await db.execute(select(Source.id, Source.name).where(Source.id.in_(source_ids)))
).all()
}
liked: set[int] = set()
favorited: set[int] = set()
if viewer_user_id:
@@ -101,6 +113,8 @@ async def serialize_videos(
"video_url": v.video_url,
"duration": v.duration,
"category_id": v.category_id,
"source_id": v.source_id or 0,
"source_name": source_map.get(v.source_id, ""),
"author": _author_brief(authors.get(v.user_id)),
"view_count": v.view_count,
"like_count": v.like_count,
@@ -114,6 +128,16 @@ async def serialize_videos(
]
async def list_sources(db: AsyncSession) -> dict:
"""启用中的内容来源列表(App 分类页来源筛选)"""
from app.models.video import Source
rows = (await db.execute(
select(Source).where(Source.status == 1).order_by(Source.sort, Source.id)
)).scalars().all()
return {"list": [{"source_id": s.id, "name": s.name, "code": s.code} for s in rows]}
async def get_tags_map(db: AsyncSession, video_ids: list[int]) -> dict[int, list[str]]:
if not video_ids:
return {}
@@ -157,21 +181,110 @@ async def get_feed(
cursor: str | None,
limit: int,
category_id: int | None,
source_id: int | None = None,
group: str | None = None,
) -> dict:
# group=series:系列型分类(如 AI短剧)按「剧」聚合,每部剧只返回首集一条
if group == "series":
return await _feed_series(db, identity, cursor, limit, category_id, source_id)
if feed_type == "latest":
return await _feed_latest(db, identity, cursor, limit, category_id)
return await _feed_latest(db, identity, cursor, limit, category_id, source_id)
if feed_type in ("liked", "viewed"):
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id)
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id)
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id, source_id)
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id, source_id)
async def _feed_series(
db: AsyncSession,
identity: Identity,
cursor: str | None,
limit: int,
category_id: int | None,
source_id: int | None = None,
) -> dict:
"""系列聚合 Feed:一部剧一条(视频 id 为首集),按剧集播放量倒序。
卡片封面/标题取 Series(回退首集封面),点击后客户端拿到的 video_id 即首集,
播放页会按 /videos/{id}/series 拉全集列表,从而支持选集。
"""
if not category_id:
return {"next_cursor": None, "list": []}
# 仅返回至少有一集已发布的系列
live_series_ids = select(Video.series_id).where(
Video.status == STATUS_PUBLISHED, Video.series_id.isnot(None)
).distinct()
stmt = select(Series).where(
Series.status == 1,
Series.category_id == category_id,
Series.id.in_(live_series_ids),
)
if source_id:
stmt = stmt.where(Series.source_id == source_id)
offset = int(_decode_cursor(cursor).get("o", 0))
rows = list(
(await db.execute(
stmt.order_by(Series.view_count.desc(), Series.id.desc())
.offset(offset)
.limit(limit + 1)
)).scalars()
)
page, rows = (len(rows) > limit), rows[:limit]
if not rows:
return {"next_cursor": None, "list": []}
# 取这些剧的全部已发布集数,按 episode_no + id 找首集,同时统计实际集数
series_ids = [s.id for s in rows]
episodes = list(
(await db.execute(
select(Video).where(
Video.series_id.in_(series_ids), Video.status == STATUS_PUBLISHED
)
)).scalars()
)
first_ep: dict[int, Video] = {}
ep_count: dict[int, int] = {}
for v in episodes:
ep_count[v.series_id] = ep_count.get(v.series_id, 0) + 1
cur = first_ep.get(v.series_id)
if cur is None or _ep_key(v) < _ep_key(cur):
first_ep[v.series_id] = v
first_videos = [first_ep[s.id] for s in rows if s.id in first_ep]
serialized = await serialize_videos(db, first_videos, identity.user_id)
by_id = {it["video_id"]: it for it in serialized}
items = []
for s in rows:
v = first_ep.get(s.id)
if v is None:
continue
it = by_id[v.id]
# 系列封面/标题优先,空值回退首集自身字段
it["title"] = s.title or it["title"]
it["cover_url"] = s.cover_url or it["cover_url"]
it["view_count"] = s.view_count
it["series_id"] = s.id
it["total_episodes"] = ep_count.get(s.id, s.total_episodes)
items.append(it)
next_cursor = _encode_cursor({"o": offset + limit}) if page else None
return {"next_cursor": next_cursor, "list": items}
def _ep_key(v: Video) -> tuple[int, int]:
"""首集排序键:episode_no 缺失视为最大,再按 id 兜底"""
return (v.episode_no if v.episode_no is not None else 1_000_000, v.id)
async def _feed_ranked(
db: AsyncSession, identity: Identity, feed_type: str, cursor: str | None,
limit: int, category_id: int | None,
limit: int, category_id: int | None, source_id: int | None = None,
) -> dict:
"""liked / viewed:按点赞量 / 播放量严格倒序(值 + id 打破并列游标)"""
col = Video.like_count if feed_type == "liked" else Video.view_count
stmt = await _base_pool_stmt(category_id)
stmt = _base_pool_stmt(category_id, source_id)
c = _decode_cursor(cursor)
if c:
val, vid = int(c["n"]), int(c["i"])
@@ -191,18 +304,21 @@ async def _feed_ranked(
}
async def _base_pool_stmt(category_id: int | None):
def _base_pool_stmt(category_id: int | None, source_id: int | None = None):
stmt = select(Video).where(Video.status == STATUS_PUBLISHED, Video.published_at.isnot(None))
if category_id:
stmt = stmt.where(Video.category_id == category_id)
if source_id:
stmt = stmt.where(Video.source_id == source_id)
return stmt
async def _feed_latest(
db: AsyncSession, identity: Identity, cursor: str | None, limit: int, category_id: int | None
db: AsyncSession, identity: Identity, cursor: str | None, limit: int, category_id: int | None,
source_id: int | None = None,
) -> dict:
"""latest:published_at 游标倒序(ts + id 打破并列)"""
stmt = await _base_pool_stmt(category_id)
stmt = _base_pool_stmt(category_id, source_id)
c = _decode_cursor(cursor)
if c:
ts, vid = datetime.fromtimestamp(c["p"]), int(c["i"])
@@ -260,6 +376,7 @@ async def _feed_scored(
cursor: str | None,
limit: int,
category_id: int | None,
source_id: int | None = None,
) -> dict:
"""recommend / hot:候选池打分 → 当日种子确定性加权抽样 → 当日去重(不足兜底)→ 偏移游标"""
now = datetime.now()
@@ -267,7 +384,7 @@ async def _feed_scored(
subject = identity.subject or "anon"
# 1. 候选池:近 30 天已发布,封顶 FEED_POOL_SIZE
stmt = (await _base_pool_stmt(category_id)).where(
stmt = _base_pool_stmt(category_id, source_id).where(
Video.published_at > now - timedelta(days=30)
).order_by(Video.published_at.desc()).limit(FEED_POOL_SIZE)
pool = list((await db.execute(stmt)).scalars())
@@ -505,7 +622,20 @@ async def list_categories(db: AsyncSession) -> list[dict]:
rows = (await db.execute(
select(Category).where(Category.status == 1).order_by(Category.sort, Category.id)
)).scalars().all()
return [{"category_id": c.id, "name": c.name} for c in rows]
# 系列型分类标记(该分类下存在正常状态的系列),App 据此按「剧」聚合展示
cnt_rows = (await db.execute(
select(Series.category_id, func.count()).where(Series.status == 1)
.group_by(Series.category_id)
)).all()
series_cnt = {cid: n for cid, n in cnt_rows}
return [
{
"category_id": c.id,
"name": c.name,
"has_series": series_cnt.get(c.id, 0) > 0,
}
for c in rows
]
# ---------- 上传发布(5.5 / 5.6 / 5.7) ----------