1
This commit is contained in:
Binary file not shown.
Binary file not shown.
@@ -155,7 +155,7 @@ async def dashboard(db: AsyncSession) -> dict:
|
||||
|
||||
# ---------- 视频审核 / 管理 ----------
|
||||
|
||||
def _video_item(video: Video, author: User | None) -> dict:
|
||||
def _video_item(video: Video, author: User | None, source_name: str = "") -> dict:
|
||||
return {
|
||||
"video_id": video.id,
|
||||
"title": video.title,
|
||||
@@ -164,6 +164,8 @@ def _video_item(video: Video, author: User | None) -> dict:
|
||||
"video_url": video.video_url,
|
||||
"duration": video.duration,
|
||||
"category_id": video.category_id or 0,
|
||||
"source_id": video.source_id or 0,
|
||||
"source_name": source_name,
|
||||
"series_id": video.series_id,
|
||||
"episode_no": video.episode_no,
|
||||
"status": video.status,
|
||||
@@ -207,7 +209,7 @@ async def list_pending_videos(db: AsyncSession, page: int, page_size: int) -> di
|
||||
|
||||
async def list_videos(
|
||||
db: AsyncSession, keyword: str | None, status: int | None, author_id: int | None,
|
||||
page: int, page_size: int,
|
||||
page: int, page_size: int, source_id: int | None = None,
|
||||
) -> dict:
|
||||
"""视频列表(11.25)"""
|
||||
stmt = select(Video)
|
||||
@@ -218,12 +220,27 @@ async def list_videos(
|
||||
stmt = stmt.where(Video.status == status)
|
||||
if author_id is not None:
|
||||
stmt = stmt.where(Video.user_id == author_id)
|
||||
if source_id is not None:
|
||||
stmt = stmt.where(Video.source_id == source_id)
|
||||
total = (await db.execute(select(func.count()).select_from(stmt.subquery()))).scalar() or 0
|
||||
rows = (await db.execute(
|
||||
stmt.order_by(Video.id.desc()).offset((page - 1) * page_size).limit(page_size)
|
||||
)).scalars().all()
|
||||
authors = await _authors_map(db, rows)
|
||||
return {"total": total, "list": [_video_item(v, authors.get(v.user_id)) for v in rows]}
|
||||
names = await source_names_map(db, {v.source_id for v in rows})
|
||||
return {"total": total,
|
||||
"list": [_video_item(v, authors.get(v.user_id), names.get(v.source_id, "")) for v in rows]}
|
||||
|
||||
|
||||
async def source_names_map(db: AsyncSession, ids: set[int]) -> dict[int, str]:
|
||||
"""来源 id → 名称"""
|
||||
from app.models.video import Source
|
||||
|
||||
ids = {i for i in ids if i}
|
||||
if not ids:
|
||||
return {}
|
||||
rows = (await db.execute(select(Source.id, Source.name).where(Source.id.in_(ids)))).all()
|
||||
return {r.id: r.name for r in rows}
|
||||
|
||||
|
||||
async def review_video(
|
||||
|
||||
@@ -0,0 +1,850 @@
|
||||
"""91 短剧(AI短剧)采集:加密接口客户端 + 系列/集数入库
|
||||
|
||||
上游协议:pwa.php 单入口,AES-256-CFB(segment 128) 加密请求体 + MD5(SHA256) 签名。
|
||||
时间戳签名有效期约 4 小时,因此**不存原始 m3u8 地址**,改为存本站代理地址,
|
||||
播放时再向上游换取新鲜签名(见 /api/v1/m3u8/aidrama)。
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import string
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.video import (
|
||||
Category,
|
||||
CollectionRecord,
|
||||
CollectionSource,
|
||||
Series,
|
||||
Tag,
|
||||
Video,
|
||||
VideoTag,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KIND = "aidrama91"
|
||||
VLOG_KIND = "aidrama91_vlog" # 91 非系列短视频
|
||||
SOURCE_KEY = "91aidrama" # collection_records.source_key
|
||||
VLOG_SOURCE_KEY = "91vlog" # 短视频采集记录 key
|
||||
SOURCE_CODE = "91" # sources.code
|
||||
SELF_API_BASE = "https://vapi.256kb.top"
|
||||
AUTHOR_USER_ID = 1
|
||||
VLOG_CATEGORY_NAME = "91短视频" # vlog 无真实分类,归到本地分类
|
||||
|
||||
# 协议常量(前端 JS 逆向)
|
||||
AES_KEY = b"cc88ddc9357ff461e08f047aedee692b"
|
||||
AES_IV = b"e89225cfbbimgkcu"
|
||||
SIGN_KEY = "cc88ddc9357ff461e08f047aedee692b"
|
||||
APP_VERSION = "7.5.0"
|
||||
CLIENT_ORIGIN = "https://pwa4.jl8v0p5t.cc"
|
||||
|
||||
# 封面图解密:CDN 返回的是 AES-128-CBC 加密的图片字节(无 padding),
|
||||
# 源站前端 DecryptImage 解密后展示 —— 本站通过 /api/v1/img/aidrama 代理解密
|
||||
IMG_AES_KEY = b"f5d965df75336270"
|
||||
IMG_AES_IV = b"97b60394abc2fbe1"
|
||||
IMG_HOST_SUFFIX = ".drleao.cn" # 封面 CDN 域名白名单(防 SSRF)
|
||||
|
||||
HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
|
||||
),
|
||||
"Origin": CLIENT_ORIGIN,
|
||||
"Referer": f"{CLIENT_ORIGIN}/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Content-Type": "application/x-www-form-urlencoded",
|
||||
}
|
||||
|
||||
OAUTH_CHARSET = string.ascii_letters + string.digits
|
||||
MAX_PAGE_LIMIT = 20 # 上游单页上限(传 100 实际只返回 20)
|
||||
|
||||
|
||||
# ---------- 加密 / 签名 ----------
|
||||
|
||||
def _cfb(key: bytes, iv: bytes):
|
||||
try:
|
||||
from cryptography.hazmat.decrepit.ciphers.modes import CFB
|
||||
except ImportError: # cryptography < 43
|
||||
from cryptography.hazmat.primitives.ciphers.modes import CFB
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
|
||||
|
||||
return Cipher(algorithms.AES(key), CFB(iv))
|
||||
|
||||
|
||||
def _encrypt(plaintext: str) -> str:
|
||||
op = _cfb(AES_KEY, AES_IV).encryptor()
|
||||
return (op.update(plaintext.encode()) + op.finalize()).hex().upper()
|
||||
|
||||
|
||||
def _decrypt(hex_data: str) -> str:
|
||||
op = _cfb(AES_KEY, AES_IV).decryptor()
|
||||
return (op.update(bytes.fromhex(hex_data)) + op.finalize()).decode("utf-8", "replace")
|
||||
|
||||
|
||||
def _new_oauth_id() -> str:
|
||||
return "".join(random.choices(OAUTH_CHARSET, k=16)) + str(int(time.time() * 1000))
|
||||
|
||||
|
||||
async def _post(client: httpx.AsyncClient, api_url: str, payload: dict) -> dict:
|
||||
"""加密并发送,返回解密后的业务数据(已剥掉外层包装)"""
|
||||
ts_ms = int(time.time() * 1000)
|
||||
payload = dict(payload, timestamp=ts_ms)
|
||||
data_hex = _encrypt(json.dumps(payload, separators=(",", ":")))
|
||||
ts_sec = ts_ms // 1000
|
||||
raw = f"client=pwa&data={data_hex}×tamp={ts_sec}{SIGN_KEY}"
|
||||
sign = hashlib.md5(hashlib.sha256(raw.encode()).hexdigest().encode()).hexdigest()
|
||||
|
||||
resp = await client.post(
|
||||
api_url,
|
||||
content=f"client=pwa×tamp={ts_sec}&data={data_hex}&sign={sign}",
|
||||
headers=HEADERS,
|
||||
timeout=30,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"上游 HTTP {resp.status_code}(oauth_id 可能未注册或已被限流)")
|
||||
body = resp.json()
|
||||
if body.get("errcode") not in (0, None):
|
||||
raise RuntimeError(f"上游错误: {str(body)[:200]}")
|
||||
payload_text = body.get("data")
|
||||
if not payload_text:
|
||||
return body
|
||||
return json.loads(_decrypt(payload_text))
|
||||
|
||||
|
||||
def _base(oauth_id: str, token: str) -> dict:
|
||||
return {
|
||||
"oauth_type": "pwa", "oauth_id": oauth_id, "trace_id": "",
|
||||
"aff_x_code": "", "version": APP_VERSION, "token": token,
|
||||
}
|
||||
|
||||
|
||||
# ---------- 会话 ----------
|
||||
|
||||
async def create_session(
|
||||
client: httpx.AsyncClient, api_url: str, account: str, password: str
|
||||
) -> tuple[str, str]:
|
||||
"""注册 oauth_id 并登录,返回 (oauth_id, token)"""
|
||||
oauth_id = _new_oauth_id()
|
||||
# 上游对未注册的 oauth_id 直接 503,必须先 system/index
|
||||
await _post(client, api_url, dict(_base(oauth_id, ""), mod="system", code="index", data={}))
|
||||
result = await _post(client, api_url, dict(
|
||||
_base(oauth_id, ""), mod="user", code="regOrLogin",
|
||||
account=account, password=password, type="login",
|
||||
))
|
||||
inner = result.get("data", result) if isinstance(result, dict) else {}
|
||||
token = inner.get("token") or ""
|
||||
if not token:
|
||||
raise RuntimeError(f"91 登录失败: {inner.get('msg') or str(result)[:150]}")
|
||||
return oauth_id, token
|
||||
|
||||
|
||||
# 进程内 token 缓存(2 个 uvicorn worker → 最多 2 次登录);失效时自动重登
|
||||
_TOKENS: dict[int, tuple[str, str, float]] = {}
|
||||
TOKEN_TTL = 3600
|
||||
|
||||
|
||||
async def open_session(
|
||||
client: httpx.AsyncClient, source: CollectionSource, force: bool = False
|
||||
) -> tuple[str, str]:
|
||||
"""获取可用的 (oauth_id, token):进程缓存 → DB 配置 → 重新登录"""
|
||||
key = int(source.id or 0)
|
||||
cfg = load_config(source)
|
||||
if not force:
|
||||
hit = _TOKENS.get(key)
|
||||
if hit and hit[2] > time.time():
|
||||
return hit[0], hit[1]
|
||||
if cfg.get("oauth_id") and cfg.get("token"):
|
||||
_TOKENS[key] = (cfg["oauth_id"], cfg["token"], time.time() + TOKEN_TTL)
|
||||
return cfg["oauth_id"], cfg["token"]
|
||||
|
||||
oauth_id, token = await create_session(
|
||||
client, source.list_url, cfg["account"], cfg["password"])
|
||||
_TOKENS[key] = (oauth_id, token, time.time() + TOKEN_TTL)
|
||||
return oauth_id, token
|
||||
|
||||
|
||||
def load_config(source: CollectionSource) -> dict:
|
||||
try:
|
||||
cfg = json.loads(source.config or "{}")
|
||||
except json.JSONDecodeError:
|
||||
cfg = {}
|
||||
cfg.setdefault("account", "")
|
||||
cfg.setdefault("password", "")
|
||||
cfg.setdefault("category_ids", [4])
|
||||
cfg.setdefault("page_size", MAX_PAGE_LIMIT)
|
||||
return cfg
|
||||
|
||||
|
||||
# ---------- 业务接口 ----------
|
||||
|
||||
async def fetch_list(
|
||||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
|
||||
page: int, limit: int, category_ids: list[int],
|
||||
) -> dict:
|
||||
"""短剧列表;返回 {list, total, has_more}"""
|
||||
data = await _post(client, api_url, dict(
|
||||
_base(oauth_id, token), mod="aidrama", code="home",
|
||||
page=page, limit=min(limit, MAX_PAGE_LIMIT), pay_type="all", sort="all",
|
||||
category_ids=category_ids or [4],
|
||||
))
|
||||
inner = data.get("data") or {}
|
||||
return {
|
||||
"list": inner.get("list") or [],
|
||||
"total": int(inner.get("total") or 0),
|
||||
"has_more": bool(inner.get("has_more")),
|
||||
}
|
||||
|
||||
|
||||
async def fetch_detail(
|
||||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
|
||||
) -> dict:
|
||||
data = await _post(client, api_url, dict(
|
||||
_base(oauth_id, token), mod="aidrama", code="detail", drama_id=drama_id,
|
||||
))
|
||||
return (data.get("data") or {}).get("row") or {}
|
||||
|
||||
|
||||
async def fetch_episodes(
|
||||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
|
||||
drama_id: int, page: int = 1, limit: int = MAX_PAGE_LIMIT,
|
||||
) -> dict:
|
||||
"""集数列表;返回 {list, has_more}"""
|
||||
data = await _post(client, api_url, dict(
|
||||
_base(oauth_id, token), mod="aidrama", code="episodes",
|
||||
drama_id=drama_id, page=page, limit=min(limit, MAX_PAGE_LIMIT),
|
||||
))
|
||||
inner = data.get("data") or {}
|
||||
return {"list": inner.get("list") or [], "has_more": bool(inner.get("has_more"))}
|
||||
|
||||
|
||||
async def fetch_all_episodes(
|
||||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
|
||||
) -> list[dict]:
|
||||
"""翻页取全部集数(每页 20 上限)"""
|
||||
out: list[dict] = []
|
||||
page = 1
|
||||
while True:
|
||||
part = await fetch_episodes(client, api_url, oauth_id, token, drama_id, page=page)
|
||||
out.extend(part["list"])
|
||||
if not part["has_more"] or not part["list"]:
|
||||
break
|
||||
page += 1
|
||||
return out
|
||||
|
||||
|
||||
# ---------- vlog 短视频接口 ----------
|
||||
|
||||
VLOG_MAX_LIMIT = 50 # 实测 vlog 单页上限 50(传 100 只返回 50)
|
||||
|
||||
|
||||
async def fetch_vlog_list(
|
||||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
|
||||
page: int, limit: int = VLOG_MAX_LIMIT, vlog_type: str = "new",
|
||||
) -> dict:
|
||||
"""vlog 列表(非系列短视频);返回 {list, has_more}"""
|
||||
data = await _post(client, api_url, dict(
|
||||
_base(oauth_id, token), mod="choice", code="vlog",
|
||||
type=vlog_type, page=page, limit=min(limit, VLOG_MAX_LIMIT),
|
||||
))
|
||||
lst = data.get("data") or []
|
||||
if not isinstance(lst, list):
|
||||
lst = []
|
||||
return {"list": lst, "has_more": len(lst) >= min(limit, VLOG_MAX_LIMIT)}
|
||||
|
||||
|
||||
async def fetch_vlog_detail(
|
||||
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, vid: int
|
||||
) -> dict:
|
||||
"""vlog 详情(含 playUrl / tags)"""
|
||||
data = await _post(client, api_url, dict(
|
||||
_base(oauth_id, token), mod="index", code="detail", id=vid))
|
||||
return data.get("data") or {}
|
||||
|
||||
|
||||
# ---------- 字段解析 ----------
|
||||
|
||||
def _pick_url(raw: str | dict, prefer: str = "720") -> str:
|
||||
"""封面字段是 JSON 字符串或 dict:{"ori","360","720"},取指定档位"""
|
||||
if not raw:
|
||||
return ""
|
||||
try:
|
||||
obj = json.loads(raw) if isinstance(raw, str) else raw
|
||||
except json.JSONDecodeError:
|
||||
return raw if isinstance(raw, str) else ""
|
||||
if not isinstance(obj, dict):
|
||||
return ""
|
||||
return obj.get(prefer) or obj.get("ori") or ""
|
||||
|
||||
|
||||
def proxied_cover(url: str) -> str:
|
||||
"""封面地址改写为本站解密代理(CDN 图是加密的,直连无法显示)"""
|
||||
if not url or "/api/v1/img/aidrama" in url:
|
||||
return url
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return url
|
||||
from urllib.parse import quote
|
||||
|
||||
return f"{SELF_API_BASE}/api/v1/img/aidrama?u={quote(url, safe='')}"
|
||||
|
||||
|
||||
def decrypt_image(data: bytes) -> bytes:
|
||||
"""AES-128-CBC(NoPadding) 解密 CDN 加密图片,返回原始 JPEG/PNG 字节"""
|
||||
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
||||
|
||||
op = Cipher(algorithms.AES(IMG_AES_KEY), modes.CBC(IMG_AES_IV)).decryptor()
|
||||
return op.update(data) + op.finalize()
|
||||
|
||||
|
||||
def _play_url(drama_id: int, episode_id: int, episode_no: int | None) -> str:
|
||||
"""本站代理地址(含 m3u8 字样,App 端会走本地缓存代理)"""
|
||||
url = f"{SELF_API_BASE}/api/v1/m3u8/aidrama?d={drama_id}&e={episode_id}"
|
||||
if episode_no:
|
||||
url += f"&n={episode_no}"
|
||||
return url
|
||||
|
||||
|
||||
def _safe_dt(ts) -> datetime | None:
|
||||
try:
|
||||
n = int(ts)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return datetime.fromtimestamp(n) if n > 0 else None
|
||||
|
||||
|
||||
def _parse_count(v) -> int:
|
||||
"""解析播放/点赞等计数,兼容 '143.7万' / '1.2万' / '34' / 34"""
|
||||
if v is None:
|
||||
return 0
|
||||
if isinstance(v, (int, float)):
|
||||
return int(v)
|
||||
s = str(v).strip()
|
||||
if not s:
|
||||
return 0
|
||||
if s.endswith("万"):
|
||||
try:
|
||||
return int(float(s[:-1]) * 10000)
|
||||
except ValueError:
|
||||
return 0
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
|
||||
def _vlog_play_url(vid: int) -> str:
|
||||
"""vlog 单视频播放代理地址(含 m3u8 字样,App 走本地缓存)"""
|
||||
return f"{SELF_API_BASE}/api/v1/m3u8/vlog?vid={vid}"
|
||||
|
||||
|
||||
# ---------- 入库 ----------
|
||||
|
||||
async def _upsert_tags(db: AsyncSession, names: list[str]) -> list[int]:
|
||||
"""按名称取/建标签,返回 tag id 列表"""
|
||||
ids: list[int] = []
|
||||
for name in {n.strip() for n in names if n and n.strip()}:
|
||||
name = name[:20]
|
||||
tid = (await db.execute(select(Tag.id).where(Tag.name == name))).scalar_one_or_none()
|
||||
if not tid:
|
||||
t = Tag(name=name, video_count=0)
|
||||
db.add(t)
|
||||
await db.flush()
|
||||
tid = t.id
|
||||
ids.append(tid)
|
||||
return ids
|
||||
|
||||
|
||||
async def import_drama(
|
||||
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
|
||||
drama: dict, oauth_id: str, token: str,
|
||||
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""导入单部短剧(系列 + 集数视频),返回 (新增集数, 跳过集数)"""
|
||||
drama_id = int(drama.get("id") or 0)
|
||||
if not drama_id:
|
||||
return 0, 0
|
||||
api_url = api_url or source.list_url
|
||||
cat = category_id if category_id is not None else source.category_id
|
||||
sid = source_id if source_id is not None else source.source_id
|
||||
|
||||
cfg = load_config(source)
|
||||
detail = drama
|
||||
try:
|
||||
full = await fetch_detail(client, api_url, oauth_id, token, drama_id)
|
||||
if full:
|
||||
detail = {**drama, **full}
|
||||
except Exception:
|
||||
logger.warning("短剧 %s 详情获取失败,使用列表字段", drama_id)
|
||||
|
||||
record = (await db.execute(
|
||||
select(CollectionRecord).where(
|
||||
CollectionRecord.source_key == SOURCE_KEY,
|
||||
CollectionRecord.source_id == drama_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
cover = proxied_cover(_pick_url(detail.get("cover") or ""))
|
||||
tags = list(detail.get("tag_list") or []) + list(detail.get("category_list") or [])
|
||||
# 像既有采集一样:按标题拆分关键词作为 tag
|
||||
from app.services.title_tags import extract_keywords
|
||||
|
||||
tags += extract_keywords(detail.get("title") or "")
|
||||
total_declared = int(detail.get("total_episodes") or 0)
|
||||
|
||||
series = await db.get(Series, record.series_id) if record else None
|
||||
if series is None:
|
||||
series = Series(
|
||||
title=(detail.get("title") or f"短剧{drama_id}")[:100],
|
||||
cover_url=cover[:500],
|
||||
description=(detail.get("description") or "")[:500],
|
||||
author_id=AUTHOR_USER_ID,
|
||||
category_id=cat,
|
||||
source_id=sid,
|
||||
total_episodes=total_declared,
|
||||
status=1,
|
||||
)
|
||||
db.add(series)
|
||||
await db.flush()
|
||||
if record:
|
||||
record.series_id = series.id
|
||||
else:
|
||||
db.add(CollectionRecord(
|
||||
source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id))
|
||||
await db.flush()
|
||||
else:
|
||||
# 已有系列:同步标题/封面/简介(源站可能改过)
|
||||
series.title = (detail.get("title") or series.title)[:100]
|
||||
series.cover_url = (cover or series.cover_url)[:500]
|
||||
series.total_episodes = max(series.total_episodes, total_declared)
|
||||
|
||||
existing = set((await db.execute(
|
||||
select(Video.episode_no).where(
|
||||
Video.series_id == series.id, Video.episode_no.is_not(None))
|
||||
)).scalars().all())
|
||||
|
||||
episodes = await fetch_all_episodes(client, api_url, oauth_id, token, drama_id)
|
||||
tag_ids: list[int] | None = None
|
||||
created = skipped = 0
|
||||
now = datetime.now()
|
||||
|
||||
for ep in episodes:
|
||||
ep_id = int(ep.get("id") or 0)
|
||||
ep_no = int(ep.get("episode_number") or 0)
|
||||
if not ep_id or not ep_no or ep_no in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
ep_cover = proxied_cover(_pick_url(ep.get("cover") or "")) or cover
|
||||
video = Video(
|
||||
user_id=AUTHOR_USER_ID,
|
||||
category_id=cat,
|
||||
source_id=sid,
|
||||
series_id=series.id,
|
||||
episode_no=ep_no,
|
||||
title=(ep.get("title") or f"第{ep_no}集")[:100],
|
||||
description=(detail.get("description") or "")[:500],
|
||||
cover_url=ep_cover[:500],
|
||||
video_url=_play_url(drama_id, ep_id, ep_no),
|
||||
duration=int(ep.get("duration") or 0),
|
||||
status=2,
|
||||
like_count=int(ep.get("like") or 0),
|
||||
favorite_count=int(ep.get("favorite_num") or 0),
|
||||
comment_count=int(ep.get("comment_count") or 0),
|
||||
published_at=_safe_dt(ep.get("created_at")) or now,
|
||||
)
|
||||
db.add(video)
|
||||
await db.flush()
|
||||
|
||||
# 标签挂到每集视频上(系列标签来自 tag_list / category_list)
|
||||
if tags:
|
||||
if tag_ids is None:
|
||||
tag_ids = await _upsert_tags(db, tags)
|
||||
for tid in tag_ids:
|
||||
db.add(VideoTag(video_id=video.id, tag_id=tid))
|
||||
|
||||
existing.add(ep_no)
|
||||
created += 1
|
||||
|
||||
series.total_episodes = max(len(existing), total_declared)
|
||||
return created, skipped
|
||||
|
||||
|
||||
async def sync_source(
|
||||
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
|
||||
should_cancel=None,
|
||||
) -> dict:
|
||||
"""按列表翻页增量采集(已导入的短剧跳过,仍会补同步新集数)
|
||||
|
||||
should_cancel: 可选回调,返回 True 时在每页/每条边界退出循环(用于手动取消)
|
||||
"""
|
||||
cfg = load_config(source)
|
||||
stat: dict = {
|
||||
"kind": KIND, "dramas_new": 0, "dramas_exist": 0,
|
||||
"episodes_created": 0, "episodes_skipped": 0, "pages": 0, "total": 0,
|
||||
}
|
||||
page_size = min(int(cfg.get("page_size") or MAX_PAGE_LIMIT), MAX_PAGE_LIMIT)
|
||||
category_ids = cfg.get("category_ids") or [4]
|
||||
|
||||
def _cancelled() -> bool:
|
||||
return bool(should_cancel and should_cancel())
|
||||
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
oauth_id, token = await open_session(client, source, force=attempt == 2)
|
||||
api_url = source.list_url
|
||||
cat = source.category_id
|
||||
sid = source.source_id
|
||||
page = 1
|
||||
while True:
|
||||
if _cancelled():
|
||||
logger.info("91 短剧同步被取消(第 %s 页前)", page)
|
||||
return stat
|
||||
data = await fetch_list(
|
||||
client, api_url, oauth_id, token, page, page_size, category_ids)
|
||||
stat["pages"] = page
|
||||
stat["total"] = data["total"]
|
||||
items = data["list"]
|
||||
logger.info("91 短剧第 %s 页:%s 部(共 %s)", page, len(items), data["total"])
|
||||
|
||||
for drama in items:
|
||||
if _cancelled():
|
||||
logger.info("91 短剧同步被取消(处理 drama 中)")
|
||||
return stat
|
||||
drama_id = int(drama.get("id") or 0)
|
||||
if not drama_id:
|
||||
continue
|
||||
try:
|
||||
created, skipped = await import_drama(
|
||||
db, client, source, drama, oauth_id, token,
|
||||
api_url=api_url, category_id=cat, source_id=sid)
|
||||
await db.commit()
|
||||
if created > 0:
|
||||
stat["dramas_new"] += 1
|
||||
else:
|
||||
stat["dramas_exist"] += 1
|
||||
stat["episodes_created"] += created
|
||||
stat["episodes_skipped"] += skipped
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
logger.exception("导入 91 短剧 %s 失败,跳过", drama_id)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
if not data["has_more"] or not items:
|
||||
break
|
||||
if max_pages and page >= max_pages:
|
||||
break
|
||||
page += 1
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
cfg["oauth_id"], cfg["token"] = oauth_id, token
|
||||
source.config = json.dumps(cfg, ensure_ascii=False)
|
||||
break
|
||||
except Exception:
|
||||
_TOKENS.pop(int(source.id or 0), None)
|
||||
if attempt == 2:
|
||||
source.last_sync_at = datetime.now()
|
||||
await db.commit()
|
||||
raise
|
||||
logger.warning("91 同步失败,重新登录后重试一次")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
source.last_sync_at = datetime.now()
|
||||
await db.commit()
|
||||
return stat
|
||||
|
||||
|
||||
async def refresh_source_id(db: AsyncSession, source: CollectionSource) -> int:
|
||||
"""确保采集源绑定的 sources 记录存在,返回来源 id"""
|
||||
if source.source_id:
|
||||
return source.source_id
|
||||
from app.models.video import Source
|
||||
|
||||
sid = (await db.execute(
|
||||
select(Source.id).where(Source.code == SOURCE_CODE)
|
||||
)).scalar_one_or_none()
|
||||
if not sid:
|
||||
max_sort = 0
|
||||
s = Source(name="91", code=SOURCE_CODE, sort=max_sort + 1, status=1)
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
sid = s.id
|
||||
source.source_id = sid
|
||||
return sid
|
||||
|
||||
|
||||
async def ensure_local_category(db: AsyncSession, name: str) -> int:
|
||||
"""按采集源名称找/建本站分类"""
|
||||
cid = (await db.execute(
|
||||
select(Category.id).where(Category.name == name)
|
||||
)).scalar_one_or_none()
|
||||
if cid:
|
||||
return cid
|
||||
c = Category(name=name, sort=99, status=1)
|
||||
db.add(c)
|
||||
await db.flush()
|
||||
return c.id
|
||||
|
||||
|
||||
async def fetch_playlist(source: CollectionSource, drama_id: int, episode_id: int,
|
||||
episode_no: int | None = None) -> str:
|
||||
"""换取某集的最新 m3u8 文本(签名有时效,每次播放现取;token 失效自动重登)"""
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
return await _fetch_playlist_once(
|
||||
source, drama_id, episode_id, episode_no, force=attempt == 2)
|
||||
except Exception:
|
||||
_TOKENS.pop(int(source.id or 0), None)
|
||||
if attempt == 2:
|
||||
raise
|
||||
logger.warning("91 播放列表获取失败,重新登录后重试一次")
|
||||
await asyncio.sleep(0.5)
|
||||
raise RuntimeError("播放列表获取失败")
|
||||
|
||||
|
||||
async def _fetch_playlist_once(
|
||||
source: CollectionSource, drama_id: int, episode_id: int,
|
||||
episode_no: int | None, force: bool,
|
||||
) -> str:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
oauth_id, token = await open_session(client, source, force=force)
|
||||
|
||||
target: dict | None = None
|
||||
# 有集号就直接算所在页;否则从第 1 页起扫(最多 5 页兜底)
|
||||
pages: list[int] = []
|
||||
if episode_no and episode_no > 0:
|
||||
pages.append((episode_no - 1) // MAX_PAGE_LIMIT + 1)
|
||||
pages.extend(p for p in (1, 2, 3, 4, 5) if p not in pages)
|
||||
|
||||
for page in pages:
|
||||
part = await fetch_episodes(
|
||||
client, source.list_url, oauth_id, token, drama_id, page=page)
|
||||
for ep in part["list"]:
|
||||
if int(ep.get("id") or 0) == episode_id:
|
||||
target = ep
|
||||
break
|
||||
if target or not part["has_more"]:
|
||||
break
|
||||
|
||||
if not target:
|
||||
raise RuntimeError("源站未找到该集")
|
||||
|
||||
m3u8 = target.get("full_m3u8") or ""
|
||||
if not m3u8:
|
||||
raise RuntimeError("该集暂无可播放地址(可能需付费或未解锁)")
|
||||
|
||||
resp = await client.get(
|
||||
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
|
||||
resp.raise_for_status()
|
||||
if not resp.text.lstrip().startswith("#EXTM3U"):
|
||||
raise RuntimeError("源站播放列表异常")
|
||||
return resp.text
|
||||
|
||||
|
||||
async def get_collection_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
|
||||
"""取 91 采集源(播放代理用)"""
|
||||
stmt = select(CollectionSource).where(CollectionSource.kind == KIND, CollectionSource.status == 1)
|
||||
if source_id:
|
||||
stmt = stmt.where(CollectionSource.id == source_id)
|
||||
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
|
||||
if not row:
|
||||
raise RuntimeError("未配置 91 采集源")
|
||||
return row
|
||||
|
||||
|
||||
async def get_vlog_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
|
||||
"""取 91 短视频采集源(播放代理用)"""
|
||||
stmt = select(CollectionSource).where(
|
||||
CollectionSource.kind == VLOG_KIND, CollectionSource.status == 1)
|
||||
if source_id:
|
||||
stmt = stmt.where(CollectionSource.id == source_id)
|
||||
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
|
||||
if not row:
|
||||
raise RuntimeError("未配置 91 短视频采集源")
|
||||
return row
|
||||
|
||||
|
||||
# ---------- vlog 入库 ----------
|
||||
|
||||
async def import_vlog_video(
|
||||
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
|
||||
item: dict, oauth_id: str, token: str,
|
||||
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
|
||||
) -> bool:
|
||||
"""导入单条 vlog 短视频,返回是否新建"""
|
||||
vid = int(item.get("id") or 0)
|
||||
if not vid:
|
||||
return False
|
||||
api_url = api_url or source.list_url
|
||||
cat = category_id if category_id is not None else source.category_id
|
||||
sid = source_id if source_id is not None else source.source_id
|
||||
|
||||
record = (await db.execute(select(CollectionRecord).where(
|
||||
CollectionRecord.source_key == VLOG_SOURCE_KEY,
|
||||
CollectionRecord.source_id == vid,
|
||||
))).scalar_one_or_none()
|
||||
if record:
|
||||
v = await db.get(Video, record.video_id) if record.video_id else None
|
||||
if v:
|
||||
return False
|
||||
|
||||
# 详情取 playUrl + tags
|
||||
detail = item
|
||||
try:
|
||||
full = await fetch_vlog_detail(client, api_url, oauth_id, token, vid)
|
||||
if full:
|
||||
detail = {**item, **full}
|
||||
except Exception:
|
||||
logger.warning("vlog %s 详情获取失败,使用列表字段", vid)
|
||||
|
||||
cover = proxied_cover(_pick_url(detail.get("thumbImg") or detail.get("cover_thumb") or ""))
|
||||
title = (detail.get("title") or f"短视频{vid}")[:100]
|
||||
tags = list(detail.get("tags") or [])
|
||||
from app.services.title_tags import extract_keywords
|
||||
|
||||
tags += extract_keywords(title)
|
||||
now = datetime.now()
|
||||
|
||||
video = Video(
|
||||
user_id=AUTHOR_USER_ID,
|
||||
category_id=cat,
|
||||
source_id=sid,
|
||||
title=title,
|
||||
description=(str(detail.get("descriptions") or "")[:500]),
|
||||
cover_url=cover[:500],
|
||||
video_url=_vlog_play_url(vid),
|
||||
duration=int(detail.get("duration") or 0),
|
||||
status=2,
|
||||
like_count=_parse_count(detail.get("like")),
|
||||
favorite_count=0,
|
||||
comment_count=_parse_count(detail.get("comment")),
|
||||
view_count=_parse_count(detail.get("play_count")),
|
||||
published_at=now,
|
||||
)
|
||||
db.add(video)
|
||||
await db.flush()
|
||||
|
||||
if tags:
|
||||
tag_ids = await _upsert_tags(db, tags)
|
||||
for tid in tag_ids:
|
||||
db.add(VideoTag(video_id=video.id, tag_id=tid))
|
||||
|
||||
if record:
|
||||
record.video_id = video.id
|
||||
else:
|
||||
db.add(CollectionRecord(source_key=VLOG_SOURCE_KEY, source_id=vid, video_id=video.id))
|
||||
return True
|
||||
|
||||
|
||||
async def sync_vlog_source(
|
||||
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
|
||||
should_cancel=None,
|
||||
) -> dict:
|
||||
"""vlog 短视频翻页增量采集。should_cancel 返回 True 时每页/每条边界退出"""
|
||||
cfg = load_config(source)
|
||||
stat: dict = {
|
||||
"kind": VLOG_KIND, "videos_new": 0, "videos_exist": 0,
|
||||
"pages": 0,
|
||||
}
|
||||
page_size = min(int(cfg.get("page_size") or VLOG_MAX_LIMIT), VLOG_MAX_LIMIT)
|
||||
|
||||
def _cancelled() -> bool:
|
||||
return bool(should_cancel and should_cancel())
|
||||
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
oauth_id, token = await open_session(client, source, force=attempt == 2)
|
||||
# 缓存源基本字段,避免 rollback 后 ORM 对象过期导致属性访问失败
|
||||
api_url = source.list_url
|
||||
category_id = source.category_id
|
||||
source_id = source.source_id
|
||||
page = 1
|
||||
while True:
|
||||
if _cancelled():
|
||||
logger.info("91 vlog 同步被取消(第 %s 页前)", page)
|
||||
return stat
|
||||
data = await fetch_vlog_list(
|
||||
client, api_url, oauth_id, token, page, page_size)
|
||||
stat["pages"] = page
|
||||
items = data["list"]
|
||||
logger.info("91 vlog 第 %s 页:%s 条", page, len(items))
|
||||
|
||||
for item in items:
|
||||
if _cancelled():
|
||||
logger.info("91 vlog 同步被取消(处理 item 中)")
|
||||
return stat
|
||||
vid = int(item.get("id") or 0)
|
||||
if not vid:
|
||||
continue
|
||||
try:
|
||||
created = await import_vlog_video(
|
||||
db, client, source, item, oauth_id, token,
|
||||
api_url=api_url, category_id=category_id, source_id=source_id)
|
||||
await db.commit()
|
||||
if created:
|
||||
stat["videos_new"] += 1
|
||||
else:
|
||||
stat["videos_exist"] += 1
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
logger.exception("导入 91 vlog %s 失败,跳过", vid)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
if not data["has_more"] or not items:
|
||||
break
|
||||
if max_pages and page >= max_pages:
|
||||
break
|
||||
page += 1
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
cfg["oauth_id"], cfg["token"] = oauth_id, token
|
||||
source.config = json.dumps(cfg, ensure_ascii=False)
|
||||
break
|
||||
except Exception:
|
||||
_TOKENS.pop(int(source.id or 0), None)
|
||||
if attempt == 2:
|
||||
source.last_sync_at = datetime.now()
|
||||
await db.commit()
|
||||
raise
|
||||
logger.warning("91 vlog 同步失败,重新登录后重试一次")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
source.last_sync_at = datetime.now()
|
||||
await db.commit()
|
||||
return stat
|
||||
|
||||
|
||||
async def fetch_vlog_playlist(source: CollectionSource, vid: int) -> str:
|
||||
"""换取 vlog 播放列表(playUrl 带时效签名,播放时现取)"""
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
return await _fetch_vlog_playlist_once(source, vid, force=attempt == 2)
|
||||
except Exception:
|
||||
_TOKENS.pop(int(source.id or 0), None)
|
||||
if attempt == 2:
|
||||
raise
|
||||
logger.warning("91 vlog 播放列表获取失败,重新登录后重试一次")
|
||||
await asyncio.sleep(0.5)
|
||||
raise RuntimeError("播放列表获取失败")
|
||||
|
||||
|
||||
async def _fetch_vlog_playlist_once(source: CollectionSource, vid: int, force: bool) -> str:
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
oauth_id, token = await open_session(client, source, force=force)
|
||||
detail = await fetch_vlog_detail(client, source.list_url, oauth_id, token, vid)
|
||||
m3u8 = detail.get("playUrl") or ""
|
||||
if not m3u8:
|
||||
raise RuntimeError("该视频暂无可播放地址(可能需付费或未解锁)")
|
||||
resp = await client.get(
|
||||
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
|
||||
resp.raise_for_status()
|
||||
if not resp.text.lstrip().startswith("#EXTM3U"):
|
||||
raise RuntimeError("源站播放列表异常")
|
||||
return resp.text
|
||||
@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# 内容源配置(域名轮换时只需改这里)
|
||||
SOURCE_KEY = "mdcmai4"
|
||||
SOURCE_CODE = "madou" # sources 表中的 code,用于给导入内容打「来源」标
|
||||
UPSTREAM = "https://mdcmai4.xyz"
|
||||
PRODUCT_ID = 1
|
||||
CATEGORY_ID = 12 # 本站分类:AI短剧
|
||||
@@ -37,6 +38,21 @@ def _proxy_url(path: str) -> str:
|
||||
return f"{SELF_API_BASE}/api/v1/m3u8/proxy?path={quote(path, safe='')}"
|
||||
|
||||
|
||||
async def default_source_id(db: AsyncSession) -> int:
|
||||
"""麻豆等内容源的来源 id(sources.code 匹配;缺失时取第一个启用来源)"""
|
||||
from app.models.video import Source
|
||||
|
||||
sid = (
|
||||
await db.execute(select(Source.id).where(Source.code == SOURCE_CODE))
|
||||
).scalar_one_or_none()
|
||||
if sid:
|
||||
return sid
|
||||
sid = (
|
||||
await db.execute(select(Source.id).where(Source.status == 1).order_by(Source.sort, Source.id))
|
||||
).scalars().first()
|
||||
return sid or 0
|
||||
|
||||
|
||||
async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: int) -> tuple[int, int]:
|
||||
"""导入单部短剧,返回 (新增集数, 跳过集数)"""
|
||||
detail = await _fetch_json(client, f"{UPSTREAM}/api/v1/short-dramas/{drama_id}", {"productId": PRODUCT_ID})
|
||||
@@ -63,6 +79,7 @@ async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: i
|
||||
description=(detail.get("description") or "")[:500],
|
||||
author_id=AUTHOR_USER_ID,
|
||||
category_id=CATEGORY_ID,
|
||||
source_id=await default_source_id(db),
|
||||
total_episodes=0,
|
||||
view_count=int(detail.get("heatCount") or 0),
|
||||
status=1,
|
||||
@@ -94,6 +111,7 @@ async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: i
|
||||
Video(
|
||||
user_id=AUTHOR_USER_ID,
|
||||
category_id=CATEGORY_ID,
|
||||
source_id=series.source_id,
|
||||
series_id=series.id,
|
||||
episode_no=ep_no,
|
||||
title=(ep.get("titleOverride") or ep.get("title") or f"第{ep_no}集")[:100],
|
||||
@@ -209,6 +227,7 @@ async def _import_video_item(db: AsyncSession, source: CollectionSource, item: d
|
||||
v = Video(
|
||||
user_id=AUTHOR_USER_ID,
|
||||
category_id=source.category_id,
|
||||
source_id=source.source_id or await default_source_id(db),
|
||||
title=(item.get("title") or f"视频{vid}")[:100],
|
||||
description=(item.get("description") or "")[:500],
|
||||
cover_url=_abs_cover(item)[:500],
|
||||
@@ -232,13 +251,23 @@ async def sync_source(
|
||||
max_pages: int = 0,
|
||||
stop_after_existing: int = 30,
|
||||
delay: float = 0.25,
|
||||
should_cancel=None,
|
||||
) -> dict:
|
||||
"""按源的 URL 模板翻页采集;连续 stop_after_existing 条已存在则提前停止(增量)"""
|
||||
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False}
|
||||
"""按源的 URL 模板翻页采集;连续 stop_after_existing 条已存在则提前停止(增量)。
|
||||
should_cancel 返回 True 时每页/每条边界退出(用于手动取消)"""
|
||||
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False, "cancelled": False}
|
||||
consecutive_existing = 0
|
||||
|
||||
def _cancelled() -> bool:
|
||||
return bool(should_cancel and should_cancel())
|
||||
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
page = 1
|
||||
while True:
|
||||
if _cancelled():
|
||||
stat["cancelled"] = True
|
||||
logger.info("通用采集源 %s 同步被取消(第 %s 页前)", source.id, page)
|
||||
return stat
|
||||
url = source.list_url.replace("{{page}}", str(page))
|
||||
data = await _fetch_json(client, url)
|
||||
items = data.get("items", [])
|
||||
@@ -246,6 +275,10 @@ async def sync_source(
|
||||
break
|
||||
stat["pages"] = page
|
||||
for item in items:
|
||||
if _cancelled():
|
||||
stat["cancelled"] = True
|
||||
logger.info("通用采集源 %s 同步被取消(处理 item 中)", source.id)
|
||||
return stat
|
||||
if await _import_video_item(db, source, item):
|
||||
stat["created"] += 1
|
||||
consecutive_existing = 0
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""标题关键词提取:从视频/系列标题拆分出词作为 tag。
|
||||
|
||||
提取规则(原 scripts/build_tags_from_titles.py,抽成公共模块供采集入库复用):
|
||||
1. 番号(如 CAWD-601)及其系列前缀(CAWD)
|
||||
2. 书名号/方括号内容(≤12 字,去掉「无码破解」「中文字幕」等标记词)
|
||||
3. jieba TF-IDF 关键词 topK=6(过滤停用词、纯数字、单字)
|
||||
"""
|
||||
import re
|
||||
|
||||
import jieba.analyse
|
||||
|
||||
STOP = set(
|
||||
"第 集 部 篇 全集 完结 中文字幕 无码 破解 无码破解 字幕 高清 最新 精选 视频 在线 观看 "
|
||||
"一个 一起 没有 为了 开始 结果 只能 直接 就是 没想到 真爽".split()
|
||||
)
|
||||
|
||||
CODE_RE = re.compile(r"[A-Za-z]{2,6}-\d{2,4}")
|
||||
EPISODE_RE = re.compile(r"第[0-9一二三四五六七八九十百千零]+[集部期话卷章]")
|
||||
PUNCT_RE = re.compile(r"[《》「」『』\[\]【】()()_\-::,,.。!!??…、/\\|~~\s]+")
|
||||
BRACKET_RE = re.compile(r"[《「『\[【((]([^《》「」『』\[\]【】()()]{2,30})[》」』\]】))]")
|
||||
|
||||
|
||||
def extract_keywords(title: str) -> list[str]:
|
||||
if not title:
|
||||
return []
|
||||
kws: list[str] = []
|
||||
|
||||
# 1. 番号 + 系列前缀
|
||||
for code in CODE_RE.findall(title.upper()):
|
||||
kws.append(code)
|
||||
prefix = code.split("-")[0]
|
||||
if len(prefix) >= 2:
|
||||
kws.append(prefix)
|
||||
|
||||
# 2. 书名号/括号内容(剧名/主题)
|
||||
for inner in BRACKET_RE.findall(title):
|
||||
inner = inner.strip()
|
||||
if 2 <= len(inner) <= 12 and inner not in STOP and not CODE_RE.search(inner):
|
||||
kws.append(inner)
|
||||
|
||||
# 3. jieba TF-IDF
|
||||
t = EPISODE_RE.sub(" ", title)
|
||||
t = PUNCT_RE.sub(" ", t)
|
||||
for w in jieba.analyse.extract_tags(t, topK=6):
|
||||
w = w.strip()
|
||||
if not (2 <= len(w) <= 12):
|
||||
continue
|
||||
if w in STOP or re.fullmatch(r"\d+", w):
|
||||
continue
|
||||
kws.append(w)
|
||||
|
||||
# 去重保序(按 casefold 键,避免 MySQL 不区分大小写的唯一键冲突)
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for k in kws:
|
||||
if k.isascii():
|
||||
k = k.upper()
|
||||
key = k.casefold()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append(k)
|
||||
return out
|
||||
@@ -20,7 +20,7 @@ from app.models.billing import WatchRecord
|
||||
from app.models.incentive import VideoEarning
|
||||
from app.models.interaction import Favorite, Like
|
||||
from app.models.user import User
|
||||
from app.models.video import Category, Tag, Video, VideoTag
|
||||
from app.models.video import Category, Series, Tag, Video, VideoTag
|
||||
from app.schemas.video import PublishVideoReq, UploadTokenReq
|
||||
from app.services import membership_service, sensitive_service, storage_service, wallet_service
|
||||
|
||||
@@ -79,6 +79,18 @@ async def serialize_videos(
|
||||
|
||||
tags_map = await get_tags_map(db, video_ids)
|
||||
|
||||
# 内容来源(麻豆 / 91 ...)
|
||||
from app.models.video import Source
|
||||
|
||||
source_ids = {v.source_id for v in videos if v.source_id}
|
||||
source_map = {}
|
||||
if source_ids:
|
||||
source_map = {
|
||||
r.id: r.name for r in (
|
||||
await db.execute(select(Source.id, Source.name).where(Source.id.in_(source_ids)))
|
||||
).all()
|
||||
}
|
||||
|
||||
liked: set[int] = set()
|
||||
favorited: set[int] = set()
|
||||
if viewer_user_id:
|
||||
@@ -101,6 +113,8 @@ async def serialize_videos(
|
||||
"video_url": v.video_url,
|
||||
"duration": v.duration,
|
||||
"category_id": v.category_id,
|
||||
"source_id": v.source_id or 0,
|
||||
"source_name": source_map.get(v.source_id, ""),
|
||||
"author": _author_brief(authors.get(v.user_id)),
|
||||
"view_count": v.view_count,
|
||||
"like_count": v.like_count,
|
||||
@@ -114,6 +128,16 @@ async def serialize_videos(
|
||||
]
|
||||
|
||||
|
||||
async def list_sources(db: AsyncSession) -> dict:
|
||||
"""启用中的内容来源列表(App 分类页来源筛选)"""
|
||||
from app.models.video import Source
|
||||
|
||||
rows = (await db.execute(
|
||||
select(Source).where(Source.status == 1).order_by(Source.sort, Source.id)
|
||||
)).scalars().all()
|
||||
return {"list": [{"source_id": s.id, "name": s.name, "code": s.code} for s in rows]}
|
||||
|
||||
|
||||
async def get_tags_map(db: AsyncSession, video_ids: list[int]) -> dict[int, list[str]]:
|
||||
if not video_ids:
|
||||
return {}
|
||||
@@ -157,21 +181,110 @@ async def get_feed(
|
||||
cursor: str | None,
|
||||
limit: int,
|
||||
category_id: int | None,
|
||||
source_id: int | None = None,
|
||||
group: str | None = None,
|
||||
) -> dict:
|
||||
# group=series:系列型分类(如 AI短剧)按「剧」聚合,每部剧只返回首集一条
|
||||
if group == "series":
|
||||
return await _feed_series(db, identity, cursor, limit, category_id, source_id)
|
||||
if feed_type == "latest":
|
||||
return await _feed_latest(db, identity, cursor, limit, category_id)
|
||||
return await _feed_latest(db, identity, cursor, limit, category_id, source_id)
|
||||
if feed_type in ("liked", "viewed"):
|
||||
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id)
|
||||
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id)
|
||||
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id, source_id)
|
||||
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id, source_id)
|
||||
|
||||
|
||||
async def _feed_series(
|
||||
db: AsyncSession,
|
||||
identity: Identity,
|
||||
cursor: str | None,
|
||||
limit: int,
|
||||
category_id: int | None,
|
||||
source_id: int | None = None,
|
||||
) -> dict:
|
||||
"""系列聚合 Feed:一部剧一条(视频 id 为首集),按剧集播放量倒序。
|
||||
|
||||
卡片封面/标题取 Series(回退首集封面),点击后客户端拿到的 video_id 即首集,
|
||||
播放页会按 /videos/{id}/series 拉全集列表,从而支持选集。
|
||||
"""
|
||||
if not category_id:
|
||||
return {"next_cursor": None, "list": []}
|
||||
|
||||
# 仅返回至少有一集已发布的系列
|
||||
live_series_ids = select(Video.series_id).where(
|
||||
Video.status == STATUS_PUBLISHED, Video.series_id.isnot(None)
|
||||
).distinct()
|
||||
stmt = select(Series).where(
|
||||
Series.status == 1,
|
||||
Series.category_id == category_id,
|
||||
Series.id.in_(live_series_ids),
|
||||
)
|
||||
if source_id:
|
||||
stmt = stmt.where(Series.source_id == source_id)
|
||||
|
||||
offset = int(_decode_cursor(cursor).get("o", 0))
|
||||
rows = list(
|
||||
(await db.execute(
|
||||
stmt.order_by(Series.view_count.desc(), Series.id.desc())
|
||||
.offset(offset)
|
||||
.limit(limit + 1)
|
||||
)).scalars()
|
||||
)
|
||||
page, rows = (len(rows) > limit), rows[:limit]
|
||||
if not rows:
|
||||
return {"next_cursor": None, "list": []}
|
||||
|
||||
# 取这些剧的全部已发布集数,按 episode_no + id 找首集,同时统计实际集数
|
||||
series_ids = [s.id for s in rows]
|
||||
episodes = list(
|
||||
(await db.execute(
|
||||
select(Video).where(
|
||||
Video.series_id.in_(series_ids), Video.status == STATUS_PUBLISHED
|
||||
)
|
||||
)).scalars()
|
||||
)
|
||||
first_ep: dict[int, Video] = {}
|
||||
ep_count: dict[int, int] = {}
|
||||
for v in episodes:
|
||||
ep_count[v.series_id] = ep_count.get(v.series_id, 0) + 1
|
||||
cur = first_ep.get(v.series_id)
|
||||
if cur is None or _ep_key(v) < _ep_key(cur):
|
||||
first_ep[v.series_id] = v
|
||||
|
||||
first_videos = [first_ep[s.id] for s in rows if s.id in first_ep]
|
||||
serialized = await serialize_videos(db, first_videos, identity.user_id)
|
||||
by_id = {it["video_id"]: it for it in serialized}
|
||||
|
||||
items = []
|
||||
for s in rows:
|
||||
v = first_ep.get(s.id)
|
||||
if v is None:
|
||||
continue
|
||||
it = by_id[v.id]
|
||||
# 系列封面/标题优先,空值回退首集自身字段
|
||||
it["title"] = s.title or it["title"]
|
||||
it["cover_url"] = s.cover_url or it["cover_url"]
|
||||
it["view_count"] = s.view_count
|
||||
it["series_id"] = s.id
|
||||
it["total_episodes"] = ep_count.get(s.id, s.total_episodes)
|
||||
items.append(it)
|
||||
|
||||
next_cursor = _encode_cursor({"o": offset + limit}) if page else None
|
||||
return {"next_cursor": next_cursor, "list": items}
|
||||
|
||||
|
||||
def _ep_key(v: Video) -> tuple[int, int]:
|
||||
"""首集排序键:episode_no 缺失视为最大,再按 id 兜底"""
|
||||
return (v.episode_no if v.episode_no is not None else 1_000_000, v.id)
|
||||
|
||||
|
||||
async def _feed_ranked(
|
||||
db: AsyncSession, identity: Identity, feed_type: str, cursor: str | None,
|
||||
limit: int, category_id: int | None,
|
||||
limit: int, category_id: int | None, source_id: int | None = None,
|
||||
) -> dict:
|
||||
"""liked / viewed:按点赞量 / 播放量严格倒序(值 + id 打破并列游标)"""
|
||||
col = Video.like_count if feed_type == "liked" else Video.view_count
|
||||
stmt = await _base_pool_stmt(category_id)
|
||||
stmt = _base_pool_stmt(category_id, source_id)
|
||||
c = _decode_cursor(cursor)
|
||||
if c:
|
||||
val, vid = int(c["n"]), int(c["i"])
|
||||
@@ -191,18 +304,21 @@ async def _feed_ranked(
|
||||
}
|
||||
|
||||
|
||||
async def _base_pool_stmt(category_id: int | None):
|
||||
def _base_pool_stmt(category_id: int | None, source_id: int | None = None):
|
||||
stmt = select(Video).where(Video.status == STATUS_PUBLISHED, Video.published_at.isnot(None))
|
||||
if category_id:
|
||||
stmt = stmt.where(Video.category_id == category_id)
|
||||
if source_id:
|
||||
stmt = stmt.where(Video.source_id == source_id)
|
||||
return stmt
|
||||
|
||||
|
||||
async def _feed_latest(
|
||||
db: AsyncSession, identity: Identity, cursor: str | None, limit: int, category_id: int | None
|
||||
db: AsyncSession, identity: Identity, cursor: str | None, limit: int, category_id: int | None,
|
||||
source_id: int | None = None,
|
||||
) -> dict:
|
||||
"""latest:published_at 游标倒序(ts + id 打破并列)"""
|
||||
stmt = await _base_pool_stmt(category_id)
|
||||
stmt = _base_pool_stmt(category_id, source_id)
|
||||
c = _decode_cursor(cursor)
|
||||
if c:
|
||||
ts, vid = datetime.fromtimestamp(c["p"]), int(c["i"])
|
||||
@@ -260,6 +376,7 @@ async def _feed_scored(
|
||||
cursor: str | None,
|
||||
limit: int,
|
||||
category_id: int | None,
|
||||
source_id: int | None = None,
|
||||
) -> dict:
|
||||
"""recommend / hot:候选池打分 → 当日种子确定性加权抽样 → 当日去重(不足兜底)→ 偏移游标"""
|
||||
now = datetime.now()
|
||||
@@ -267,7 +384,7 @@ async def _feed_scored(
|
||||
subject = identity.subject or "anon"
|
||||
|
||||
# 1. 候选池:近 30 天已发布,封顶 FEED_POOL_SIZE
|
||||
stmt = (await _base_pool_stmt(category_id)).where(
|
||||
stmt = _base_pool_stmt(category_id, source_id).where(
|
||||
Video.published_at > now - timedelta(days=30)
|
||||
).order_by(Video.published_at.desc()).limit(FEED_POOL_SIZE)
|
||||
pool = list((await db.execute(stmt)).scalars())
|
||||
@@ -505,7 +622,20 @@ async def list_categories(db: AsyncSession) -> list[dict]:
|
||||
rows = (await db.execute(
|
||||
select(Category).where(Category.status == 1).order_by(Category.sort, Category.id)
|
||||
)).scalars().all()
|
||||
return [{"category_id": c.id, "name": c.name} for c in rows]
|
||||
# 系列型分类标记(该分类下存在正常状态的系列),App 据此按「剧」聚合展示
|
||||
cnt_rows = (await db.execute(
|
||||
select(Series.category_id, func.count()).where(Series.status == 1)
|
||||
.group_by(Series.category_id)
|
||||
)).all()
|
||||
series_cnt = {cid: n for cid, n in cnt_rows}
|
||||
return [
|
||||
{
|
||||
"category_id": c.id,
|
||||
"name": c.name,
|
||||
"has_series": series_cnt.get(c.id, 0) > 0,
|
||||
}
|
||||
for c in rows
|
||||
]
|
||||
|
||||
|
||||
# ---------- 上传发布(5.5 / 5.6 / 5.7) ----------
|
||||
|
||||
Reference in New Issue
Block a user