xm/video
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
lz_db
2026-10-25 11:46:44 +08:00
parent 893dfed0ed
commit 494f667719
63 changed files with 5213 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/alembic/versions/a7c2e5f18b04_collection_source_kind_config.py
+29
View File
@@ -0,0 +1,29 @@
"""collection_sources 增加 kind/config（接口类采集源）
Revision ID: a7c2e5f18b04
Revises: f1b7d4a9c310
Create Date: 2026-10-24 15:00:00.000000
"""
from alembic import op
import sqlalchemy as sa
revision = 'a7c2e5f18b04'
down_revision = 'f1b7d4a9c310'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column('collection_sources', sa.Column(
'kind', sa.String(length=20), nullable=False, server_default='url',
comment='采集类型 url/aidrama91'))
# TEXT 在 MySQL 上不能用普通默认值：先加可空列，回填空串后再收紧
op.add_column('collection_sources', sa.Column('config', sa.Text(), nullable=True))
op.execute("UPDATE collection_sources SET config = '' WHERE config IS NULL")
op.alter_column('collection_sources', 'config', nullable=False,
existing_type=sa.Text(), existing_nullable=True)
def downgrade() -> None:
op.drop_column('collection_sources', 'config')
op.drop_column('collection_sources', 'kind')
server/alembic/versions/cd1b499eba7a_add_user_action_logs.py
+36
View File
@@ -0,0 +1,36 @@
"""add_user_action_logs
Revision ID: cd1b499eba7a
Revises: a7c2e5f18b04
Create Date: 2026-10-25 11:03:01.792362
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
revision = 'cd1b499eba7a'
down_revision = 'a7c2e5f18b04'
branch_labels = None
depends_on = None
def upgrade() -> None:
op.create_table('user_action_logs',
sa.Column('id', mysql.BIGINT(unsigned=True), autoincrement=True, nullable=False),
sa.Column('user_id', mysql.BIGINT(unsigned=True), nullable=False),
sa.Column('action', sa.String(length=50), nullable=False, comment='route_push/player_stopall/api_error/...'),
sa.Column('page', sa.String(length=100), nullable=False, comment='路由路径或页面标识'),
sa.Column('detail', sa.String(length=1000), nullable=False, comment='行为上下文（JSON 字符串）'),
sa.Column('device_info', sa.String(length=200), nullable=False, comment='机型/系统/版本'),
sa.Column('app_version', sa.String(length=20), nullable=False, comment='1.3.26+42'),
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_index('idx_action_created', 'user_action_logs', ['action', 'created_at'], unique=False)
op.create_index('idx_user_created', 'user_action_logs', ['user_id', 'created_at'], unique=False)
def downgrade() -> None:
op.drop_index('idx_user_created', table_name='user_action_logs')
op.drop_index('idx_action_created', table_name='user_action_logs')
op.drop_table('user_action_logs')
server/alembic/versions/f1b7d4a9c310_add_sources_and_source_id.py
+49
View File
@@ -0,0 +1,49 @@
"""add sources table + source_id on videos/series/collection_sources
Revision ID: f1b7d4a9c310
Revises: e7a1f3c58d02
Create Date: 2026-10-24 10:00:00.000000
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
revision = 'f1b7d4a9c310'
down_revision = 'e7a1f3c58d02'
branch_labels = None
depends_on = None
def upgrade() -> None:
# 内容来源表（片商）
op.create_table(
'sources',
sa.Column('id', mysql.INTEGER(unsigned=True), autoincrement=True, nullable=False),
sa.Column('name', sa.String(length=50), nullable=False, comment='来源名称，如 麻豆'),
sa.Column('code', sa.String(length=50), nullable=False, server_default='', comment='英文/拼音标识'),
sa.Column('sort', sa.Integer(), nullable=False, server_default='0', comment='越小越靠前'),
sa.Column('status', mysql.TINYINT(), nullable=False, server_default='1', comment='1启用 0停用'),
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp()),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name'),
)
# 存量数据全部归为「麻豆」
op.execute("INSERT INTO sources (name, code, sort, status) VALUES ('麻豆', 'madou', 1, 1)")
for table in ('videos', 'series', 'collection_sources'):
op.add_column(table, sa.Column('source_id', mysql.INTEGER(unsigned=True),
nullable=False, server_default='0', comment='内容来源 id'))
op.execute(f"UPDATE {table} SET source_id = (SELECT id FROM sources WHERE code = 'madou')")
op.create_index('idx_video_source', 'videos', ['source_id', 'status'])
op.create_index('idx_series_source', 'series', ['source_id', 'status'])
def downgrade() -> None:
op.drop_index('idx_series_source', table_name='series')
op.drop_index('idx_video_source', table_name='videos')
op.drop_column('collection_sources', 'source_id')
op.drop_column('series', 'source_id')
op.drop_column('videos', 'source_id')
op.drop_table('sources')
server/app/api/v1/__pycache__/videos.cpython-311.pyc
BIN
View File
Binary file not shown.
server/app/api/v1/admin.py
+377 -24
View File
@@ -66,11 +66,11 @@ async def list_pending_videos(page: int = Query(1, ge=1), page_size: int = Query
@router.get("/videos")
async def list_videos(keyword: str | None = Query(None), status: int | None = Query(None),
author_id: int | None = Query(None),
author_id: int | None = Query(None), source_id: int | None = Query(None),
page: int = Query(1, ge=1), page_size: int = Query(20, ge=1, le=100),
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
return ok(await admin_service.list_videos(db, keyword, status, author_id, page, page_size))
return ok(await admin_service.list_videos(db, keyword, status, author_id, page, page_size, source_id))
@router.post("/videos/{video_id}/review")
@@ -90,17 +90,90 @@ async def offline_video(video_id: int, req: schema.ReasonReq, request: Request,
return ok()
class _VideoBatchOffline(_BM):
video_ids: list[int] = _Field(min_length=1, max_length=500)
reason: str = _Field(default="管理员批量下架", max_length=200)
# 批量路由须在 /videos/{video_id} 之前注册，否则 "batch-offline" 会被当成 video_id
@router.post("/videos/batch-offline")
async def batch_offline_videos(req: _VideoBatchOffline, request: Request,
db: AsyncSession = Depends(get_db),
redis: aioredis.Redis = Depends(get_redis),
admin: AdminIdentity = Depends(require_super)):
"""批量下架：复用单条下架逻辑（通知作者、写日志、清详情缓存），跳过非已发布"""
success = 0
skipped: list[int] = []
for vid in req.video_ids:
try:
await admin_service.offline_video(
db, redis, admin, vid, req.reason, _ip(request))
success += 1
except BizException:
await db.rollback()
skipped.append(vid)
return ok({"success": success, "skipped": skipped})
class _VideoUpdateReq(_BM):
title: str | None = _F(None, min_length=1, max_length=100)
description: str | None = None
cover_url: str | None = None
video_url: str | None = None
category_id: int | None = None
source_id: int | None = _F(None, ge=0)
series_id: int | None = None
episode_no: int | None = _F(None, ge=1)
status: int | None = _F(None, ge=0, le=5)
class _VideoBatchUpdate(BaseModel):
video_ids: list[int] = _Field(min_length=1, max_length=500)
source_id: int | None = _Field(None, ge=0)
status: int | None = _Field(None, ge=0, le=5)
category_id: int | None = _Field(None, ge=0)
# 批量路由须在 /videos/{video_id} 之前注册，否则 "batch" 会被当成 video_id
@router.put("/videos/batch")
async def batch_update_videos(req: _VideoBatchUpdate, request: Request,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
"""批量编辑视频属性（来源 / 状态 / 分类）"""
from app.models.admin import AdminLog
from app.models.video import Video
from sqlalchemy import select
if req.source_id is None and req.status is None and req.category_id is None:
raise BizException(errors.PARAM_ERROR, "至少要传一个要修改的字段")
rows = (await db.execute(select(Video).where(Video.id.in_(req.video_ids)))).scalars().all()
affected = 0
for v in rows:
changed = False
if req.source_id is not None and v.source_id != req.source_id:
v.source_id = req.source_id
changed = True
if req.status is not None and v.status != req.status:
v.status = req.status
changed = True
if req.category_id is not None and v.category_id != req.category_id:
v.category_id = req.category_id
changed = True
if changed:
affected += 1
db.add(AdminLog(
admin_id=admin.admin_id, action="video_batch_update",
target=f"videos:{len(req.video_ids)}",
detail=f"source_id={req.source_id} status={req.status} "
f"category_id={req.category_id} affected={affected}",
ip=_ip(request),
))
await db.commit()
return ok({"affected": affected})
@router.put("/videos/{video_id}")
async def update_video(video_id: int, req: _VideoUpdateReq,
db: AsyncSession = Depends(get_db),
@@ -332,6 +405,64 @@ async def list_users(keyword: str | None = Query(None), status: int | None = Que
return ok(await admin_service.list_users(db, keyword, status, page, page_size))
@router.get("/user-action-logs")
async def list_user_action_logs(user_id: int | None = Query(None),
action: str | None = Query(None),
keyword: str | None = Query(None),
start: str | None = Query(None),
end: str | None = Query(None),
page: int = Query(1, ge=1),
page_size: int = Query(50, ge=1, le=200),
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
"""用户操作日志查询（仅 super 可见）。用于排查特定用户 bug 时间线"""
from app.models.user_action_log import UserActionLog
from sqlalchemy import select, func, desc
conds = []
if user_id:
conds.append(UserActionLog.user_id == user_id)
if action:
conds.append(UserActionLog.action == action)
if keyword:
conds.append(UserActionLog.detail.like(f"%{keyword}%"))
if start:
conds.append(UserActionLog.created_at >= start)
if end:
conds.append(UserActionLog.created_at <= end + " 23:59:59")
total = (await db.execute(
select(func.count()).select_from(UserActionLog).where(*conds)
)).scalar() or 0
rows = (await db.execute(
select(UserActionLog)
.where(*conds)
.order_by(desc(UserActionLog.id))
.offset((page - 1) * page_size)
.limit(page_size)
)).scalars().all()
return ok({
"total": total,
"page": page,
"page_size": page_size,
"items": [
{
"id": r.id,
"user_id": r.user_id,
"action": r.action,
"page": r.page,
"detail": r.detail,
"device_info": r.device_info,
"app_version": r.app_version,
"created_at": r.created_at.strftime("%Y-%m-%d %H:%M:%S") if r.created_at else "",
}
for r in rows
],
})
@router.get("/users/{user_id}")
async def get_user_detail(user_id: int, db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
@@ -557,18 +688,23 @@ async def delete_sensitive_word(word_id: int, request: Request, db: AsyncSession
# ---------- 系列管理 ----------
@router.get("/series")
async def list_series(keyword: str = "", page: int = 1, page_size: int = 20,
async def list_series(keyword: str = "", source_id: int | None = None,
page: int = 1, page_size: int = 20,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_operator)):
from app.models.video import Series
from sqlalchemy import select, func
stmt = select(Series)
conds = []
if keyword:
stmt = stmt.where(Series.title.contains(keyword))
total = (await db.execute(select(func.count()).select_from(Series).where(
Series.title.contains(keyword) if keyword else True
))).scalar_one()
conds.append(Series.title.contains(keyword))
if source_id is not None:
conds.append(Series.source_id == source_id)
stmt = select(Series).where(*conds)
total = (await db.execute(
select(func.count()).select_from(Series).where(*conds)
)).scalar_one()
rows = (await db.execute(
stmt.order_by(Series.id.desc()).offset((page - 1) * page_size).limit(page_size)
)).scalars().all()
@@ -576,14 +712,16 @@ async def list_series(keyword: str = "", page: int = 1, page_size: int = 20,
# 关联采集记录（标记采集来源，供"同步更新"按钮使用）
from app.models.video import CollectionRecord
series_ids = [s.id for s in rows]
source_map: dict[int, int] = {}
collect_map: dict[int, int] = {}
if series_ids:
recs = (await db.execute(
select(CollectionRecord.series_id, CollectionRecord.source_id).where(
CollectionRecord.series_id.in_(series_ids)
)
)).all()
source_map = {r.series_id: r.source_id for r in recs}
collect_map = {r.series_id: r.source_id for r in recs}
names = await admin_service.source_names_map(db, {s.source_id for s in rows})
return ok({
"total": total,
@@ -591,13 +729,138 @@ async def list_series(keyword: str = "", page: int = 1, page_size: int = 20,
"id": s.id, "title": s.title, "cover_url": s.cover_url,
"description": s.description or "",
"category_id": s.category_id or 0,
"source_id": s.source_id or 0,
"source_name": names.get(s.source_id, ""),
"collect_source_id": collect_map.get(s.id),
"total_episodes": s.total_episodes, "status": s.status,
"source_id": source_map.get(s.id),
"created_at": s.created_at.strftime("%Y-%m-%d %H:%M:%S"),
} for s in rows],
})
# ---------- 内容来源管理（麻豆 / 91 / ...）----------
class _ContentSourceCreate(BaseModel):
name: str = _Field(min_length=1, max_length=50)
code: str = ""
sort: int = 0
status: int = _Field(1, ge=0, le=1)
class _ContentSourceUpdate(BaseModel):
name: str | None = _Field(None, min_length=1, max_length=50)
code: str | None = None
sort: int | None = None
status: int | None = _Field(None, ge=0, le=1)
async def _content_source_counts(db: AsyncSession, ids: list[int]) -> tuple[dict[int, int], dict[int, int]]:
"""批量统计各来源下的视频数/系列数"""
from app.models.video import Series, Video
from sqlalchemy import func, select
if not ids:
return {}, {}
v_rows = (await db.execute(
select(Video.source_id, func.count(Video.id))
.where(Video.source_id.in_(ids)).group_by(Video.source_id)
)).all()
s_rows = (await db.execute(
select(Series.source_id, func.count(Series.id))
.where(Series.source_id.in_(ids)).group_by(Series.source_id)
)).all()
return {r[0]: r[1] for r in v_rows}, {r[0]: r[1] for r in s_rows}
@router.get("/sources")
async def list_content_sources(db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_operator)):
from app.models.video import Source
from sqlalchemy import select
rows = (await db.execute(select(Source).order_by(Source.sort, Source.id))).scalars().all()
v_cnt, s_cnt = await _content_source_counts(db, [s.id for s in rows])
return ok({"list": [{
"id": s.id, "name": s.name, "code": s.code, "sort": s.sort, "status": s.status,
"video_count": v_cnt.get(s.id, 0), "series_count": s_cnt.get(s.id, 0),
} for s in rows]})
@router.post("/sources")
async def create_content_source(req: _ContentSourceCreate, request: Request,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
from app.models.admin import AdminLog
from app.models.video import Source
from sqlalchemy import select
name = req.name.strip()
exists = (await db.execute(select(Source.id).where(Source.name == name))).scalar_one_or_none()
if exists:
raise BizException(errors.PARAM_ERROR, "来源名称已存在")
s = Source(name=name, code=req.code.strip(), sort=req.sort, status=req.status)
db.add(s)
await db.flush()
db.add(AdminLog(admin_id=admin.admin_id, action="source_create",
target=f"source:{s.id}", detail=name, ip=_ip(request)))
await db.commit()
await db.refresh(s)
return ok({"id": s.id, "name": s.name, "code": s.code, "sort": s.sort, "status": s.status,
"video_count": 0, "series_count": 0})
@router.put("/sources/{source_id}")
async def update_content_source(source_id: int, req: _ContentSourceUpdate, request: Request,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
from app.models.admin import AdminLog
from app.models.video import Source
from sqlalchemy import select
s = await db.get(Source, source_id)
if not s:
raise BizException(errors.NOT_FOUND, "来源不存在")
d = req.model_dump(exclude_unset=True)
if "name" in d:
d["name"] = d["name"].strip()
dup = (await db.execute(
select(Source.id).where(Source.name == d["name"], Source.id != source_id)
)).scalar_one_or_none()
if dup:
raise BizException(errors.PARAM_ERROR, "来源名称已存在")
for k, v in d.items():
setattr(s, k, v)
db.add(AdminLog(admin_id=admin.admin_id, action="source_update",
target=f"source:{source_id}", detail=str(d), ip=_ip(request)))
await db.commit()
await db.refresh(s)
return ok({"id": s.id, "name": s.name, "code": s.code, "sort": s.sort, "status": s.status})
@router.delete("/sources/{source_id}")
async def delete_content_source(source_id: int, request: Request,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
from app.models.admin import AdminLog
from app.models.video import CollectionSource, Source
from sqlalchemy import update
s = await db.get(Source, source_id)
if not s:
raise BizException(errors.NOT_FOUND, "来源不存在")
v_cnt, s_cnt = await _content_source_counts(db, [source_id])
if v_cnt.get(source_id, 0) or s_cnt.get(source_id, 0):
raise BizException(errors.PARAM_ERROR, "该来源下仍有视频或系列，不能删除（可改为停用）")
# 采集源解除绑定
await db.execute(update(CollectionSource).where(CollectionSource.source_id == source_id)
.values(source_id=0))
await db.delete(s)
db.add(AdminLog(admin_id=admin.admin_id, action="source_delete",
target=f"source:{source_id}", detail=s.name, ip=_ip(request)))
await db.commit()
return ok()
# ---------- 系列管理 ----------
@@ -606,6 +869,7 @@ class _SeriesCreate(BaseModel):
cover_url: str = ""
description: str = ""
category_id: int = 0
source_id: int = 0
class _SeriesUpdate(BaseModel):
@@ -613,6 +877,7 @@ class _SeriesUpdate(BaseModel):
cover_url: str | None = None
description: str | None = None
category_id: int | None = None
source_id: int | None = None
status: int | None = _Field(None, ge=0, le=1)
@@ -654,18 +919,19 @@ class _SeriesBatchUpdate(BaseModel):
series_ids: list[int] = _Field(min_length=1, max_length=200)
status: int | None = _Field(None, ge=0, le=1)
category_id: int | None = _Field(None, ge=0)
source_id: int | None = _Field(None, ge=0)
@router.put("/series/batch")
async def batch_update_series(req: _SeriesBatchUpdate, request: Request,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_super)):
"""批量编辑系列属性（当前支持 status / category_id）"""
"""批量编辑系列属性（当前支持 status / category_id / source_id）"""
from app.models.admin import AdminLog
from app.models.video import Series
from sqlalchemy import select
if req.status is None and req.category_id is None:
if req.status is None and req.category_id is None and req.source_id is None:
raise BizException(errors.PARAM_ERROR, "至少要传一个要修改的字段")
stmt = select(Series).where(Series.id.in_(req.series_ids))
@@ -679,13 +945,17 @@ async def batch_update_series(req: _SeriesBatchUpdate, request: Request,
if req.category_id is not None and s.category_id != req.category_id:
s.category_id = req.category_id
changed = True
if req.source_id is not None and s.source_id != req.source_id:
s.source_id = req.source_id
changed = True
if changed:
affected += 1
db.add(AdminLog(
admin_id=admin.admin_id, action="series_batch_update",
target=f"series:{','.join(str(i) for i in req.series_ids)}",
detail=f"status={req.status} category_id={req.category_id} affected={affected}",
detail=f"status={req.status} category_id={req.category_id} "
f"source_id={req.source_id} affected={affected}",
ip=_ip(request),
))
await db.commit()
@@ -698,7 +968,7 @@ async def create_series(req: _SeriesCreate, db: AsyncSession = Depends(get_db),
from app.models.video import Series
s = Series(
title=req.title, cover_url=req.cover_url, description=req.description,
category_id=req.category_id, status=1,
category_id=req.category_id, source_id=req.source_id, status=1,
)
db.add(s)
await db.commit()
@@ -834,19 +1104,40 @@ async def collect_sync_all(req: _CollectSyncAllReq, request: Request,
class _SourceCreate(BaseModel):
name: str = _Field(min_length=1, max_length=50)
kind: str = _Field("url", pattern="^(url|aidrama91|aidrama91_vlog)$")
list_url: str = _Field(min_length=10, max_length=500)
source_id: int = 0
account: str = "" # aidrama91：源站登录账号
password: str = "" # aidrama91：源站登录密码
category_ids: list[int] = _Field(default_factory=lambda: [4]) # aidrama91：源站分类
page_size: int = _Field(20, ge=1, le=20)
class _SourceUpdate(BaseModel):
name: str | None = _Field(None, min_length=1, max_length=50)
list_url: str | None = _Field(None, min_length=10, max_length=500)
source_id: int | None = None
status: int | None = _Field(None, ge=0, le=1)
account: str | None = None
password: str | None = None
category_ids: list[int] | None = None
page_size: int | None = _Field(None, ge=1, le=20)
def _source_dict(s) -> dict:
import json as _json
try:
cfg = _json.loads(s.config or "{}")
except Exception:
cfg = {}
return {
"id": s.id, "name": s.name, "list_url": s.list_url,
"category_id": s.category_id, "status": s.status,
"id": s.id, "name": s.name, "kind": s.kind or "url", "list_url": s.list_url,
"category_id": s.category_id, "source_id": s.source_id or 0, "status": s.status,
"account": cfg.get("account", ""),
"category_ids": cfg.get("category_ids") or [],
"page_size": cfg.get("page_size") or 20,
"has_token": bool(cfg.get("token")),
"last_sync_at": s.last_sync_at.strftime("%Y-%m-%d %H:%M:%S") if s.last_sync_at else "",
"created_at": s.created_at.strftime("%Y-%m-%d %H:%M:%S"),
}
@@ -881,18 +1172,34 @@ async def list_sources(db: AsyncSession = Depends(get_db),
async def create_source(req: _SourceCreate, request: Request, db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_operator)):
"""添加采集源：自动按名称建/找分类"""
import json as _json
from app.models.admin import AdminLog
from app.models.video import CollectionSource
from app.services import aidrama91_service
if "{{page}}" not in req.list_url:
if req.kind == "url" and "{{page}}" not in req.list_url:
raise BizException(errors.PARAM_ERROR, "URL 需包含 {{page}} 页码占位符")
if req.kind in ("aidrama91", "aidrama91_vlog") and not (req.account and req.password):
raise BizException(errors.PARAM_ERROR, "91 源需填写源站账号和密码")
cid = await _resolve_category(db, req.name.strip())
s = CollectionSource(name=req.name.strip(), list_url=req.list_url.strip(), category_id=cid)
cfg = {}
if req.kind in ("aidrama91", "aidrama91_vlog"):
cfg = {"account": req.account, "password": req.password,
"category_ids": req.category_ids or [4], "page_size": req.page_size}
s = CollectionSource(name=req.name.strip(), kind=req.kind, list_url=req.list_url.strip(),
config=_json.dumps(cfg, ensure_ascii=False),
category_id=cid, source_id=req.source_id)
if req.kind in ("aidrama91", "aidrama91_vlog"):
s.source_id = await aidrama91_service.refresh_source_id(db, s)
if req.kind == "aidrama91_vlog":
s.category_id = await aidrama91_service.ensure_local_category(
db, aidrama91_service.VLOG_CATEGORY_NAME)
db.add(s)
await db.flush()
db.add(AdminLog(admin_id=admin.admin_id, action="collection_source_create",
target=f"source:{s.id}", detail=f"{s.name} {s.list_url}", ip=_ip(request)))
target=f"source:{s.id}", detail=f"{s.name} {s.kind} {s.list_url}", ip=_ip(request)))
await db.commit()
await db.refresh(s)
return ok(_source_dict(s))
@@ -901,17 +1208,28 @@ async def create_source(req: _SourceCreate, request: Request, db: AsyncSession =
@router.put("/collection/sources/{source_id}")
async def update_source(source_id: int, req: _SourceUpdate, db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_operator)):
import json as _json
from app.models.video import CollectionSource
s = await db.get(CollectionSource, source_id)
if not s:
raise BizException(errors.NOT_FOUND, "采集源不存在")
d = req.model_dump(exclude_unset=True)
if "list_url" in d and "{{page}}" not in d["list_url"]:
if "list_url" in d and (s.kind or "url") == "url" and "{{page}}" not in d["list_url"]:
raise BizException(errors.PARAM_ERROR, "URL 需包含 {{page}} 页码占位符")
if "name" in d:
d["category_id"] = await _resolve_category(db, d["name"].strip())
d["name"] = d["name"].strip()
# 接口类源的账号/密码/分类等写入 config
cfg_keys = ("account", "password", "category_ids", "page_size")
cfg = _json.loads(s.config or "{}")
for k in cfg_keys:
if k in d:
cfg[k] = d.pop(k)
s.config = _json.dumps(cfg, ensure_ascii=False)
for k, v in d.items():
setattr(s, k, v)
await db.commit()
@@ -991,23 +1309,42 @@ async def batch_create_sources(req: _SourceBatchReq, request: Request, db: Async
_sync_running_sources: set[int] = set()
_sync_cancelled_sources: set[int] = set()
async def _run_source_sync(source_id: int, max_pages: int) -> None:
from app.database import SessionFactory
from app.models.video import CollectionSource
from app.services import collection_service
from app.services import aidrama91_service, collection_service
try:
async with SessionFactory() as s:
source = await s.get(CollectionSource, source_id)
if source:
stat = await collection_service.sync_source(s, source, max_pages=max_pages)
logger.info("采集源 %s 同步完成: %s", source_id, stat)
# 传入 should_cancel 回调，循环里每页/每条检查一次
def _cancelled() -> bool:
return source_id in _sync_cancelled_sources
if source.kind == aidrama91_service.KIND:
await aidrama91_service.refresh_source_id(s, source)
stat = await aidrama91_service.sync_source(
s, source, max_pages=max_pages, should_cancel=_cancelled)
elif source.kind == aidrama91_service.VLOG_KIND:
await aidrama91_service.refresh_source_id(s, source)
stat = await aidrama91_service.sync_vlog_source(
s, source, max_pages=max_pages, should_cancel=_cancelled)
else:
stat = await collection_service.sync_source(
s, source, max_pages=max_pages, should_cancel=_cancelled)
if source_id in _sync_cancelled_sources:
logger.info("采集源 %s 同步被取消: %s", source_id, stat)
else:
logger.info("采集源 %s 同步完成: %s", source_id, stat)
except Exception:
logger.exception("采集源 %s 同步失败", source_id)
finally:
_sync_running_sources.discard(source_id)
_sync_cancelled_sources.discard(source_id)
class _SourceSyncReq(BaseModel):
@@ -1063,6 +1400,22 @@ async def sync_all_sources(request: Request, db: AsyncSession = Depends(get_db),
return ok({"started": started, "skipped_running": [i for i in ids if i not in started]})
@router.post("/collection/sources/{source_id}/sync/cancel")
async def cancel_source_sync(source_id: int, request: Request,
db: AsyncSession = Depends(get_db),
admin: AdminIdentity = Depends(require_operator)):
"""请求取消某个采集源的后台同步：仅打个标记，正在执行的任务在下个分页边界自然退出"""
from app.models.admin import AdminLog
if source_id not in _sync_running_sources:
raise BizException(errors.NOT_FOUND, "该采集源当前没有正在运行的同步任务")
_sync_cancelled_sources.add(source_id)
db.add(AdminLog(admin_id=admin.admin_id, action="collection_source_sync_cancel",
target=f"source:{source_id}", detail="", ip=_ip(request)))
await db.commit()
return ok({"cancel_requested": True, "source_id": source_id})
# ---------- 分类管理 ----------
from pydantic import BaseModel as _BM
server/app/api/v1/logs.py
+74
View File
@@ -0,0 +1,74 @@
"""用户操作日志上报：仅特定账号（如 256kb@gmail.com）开启，用于排查 bug"""
import logging
from fastapi import APIRouter, Depends, Request
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.database import get_db
from app.deps import Identity, require_user
from app.core import errors
from app.core.response import BizException, ok
from app.models.user import User
from app.models.user_action_log import UserActionLog
logger = logging.getLogger(__name__)
router = APIRouter()
# 仅这些邮箱的用户允许上报操作日志（白名单，避免给所有用户上传）
ALLOWED_EMAILS = {"256kb@gmail.com"}
# 单条 detail 最大长度（防御性截断）
MAX_DETAIL_LEN = 800
# 单次批量最大条数
MAX_BATCH = 100
class ActionLogItem(BaseModel):
action: str = Field(..., max_length=50)
page: str = Field("", max_length=100)
detail: str = Field("", max_length=MAX_DETAIL_LEN)
device_info: str = Field("", max_length=200)
app_version: str = Field("", max_length=20)
# 客户端本地时间戳（毫秒），用于在服务端时钟漂移时仍能排序
client_ts: int = 0
class ActionLogBatch(BaseModel):
items: list[ActionLogItem] = Field(..., max_length=MAX_BATCH)
@router.post("/logs/action")
async def report_action_log(
body: ActionLogBatch,
request: Request,
db: AsyncSession = Depends(get_db),
identity: Identity = Depends(require_user),
):
"""接收 App 上报的用户操作日志。仅白名单邮箱可上报，其他账号返回 403 不上报"""
user = (await db.execute(
select(User).where(User.id == identity.user_id)
)).scalar_one_or_none()
if not user or user.email not in ALLOWED_EMAILS:
# 静默拒绝：不报错，不写入，避免在普通用户端暴露此通道存在
raise BizException(errors.FORBIDDEN, "forbidden")
rows = [
UserActionLog(
user_id=user.id,
action=item.action[:50],
page=item.page[:100],
detail=item.detail[:MAX_DETAIL_LEN],
device_info=item.device_info[:200],
app_version=item.app_version[:20],
)
for item in body.items
]
db.add_all(rows)
await db.commit()
logger.info(
"user_action_log user=%s count=%s ip=%s",
user.id, len(rows), request.client.host if request.client else "-",
)
return ok({"received": len(rows)})
server/app/api/v1/media.py
+106 -2
View File
@@ -1,14 +1,25 @@
"""媒体代理：m3u8 播放列表转发（源站 playlist 内 TS 链接带时效签名，每次回源拉取新鲜版本）"""
"""媒体代理
- /m3u8/proxy ：源站路径转发（playlist 内 TS 链接带时效签名，每次回源拉新鲜版本）
- /m3u8/aidrama ：91 短剧单集播放列表（上游 m3u8 地址 4 小时过期，播放时现取）
- /img/aidrama ：91 封面图解密代理（CDN 返回 AES 加密图片字节，代理解密后输出）
"""
import logging
import re
import httpx
from fastapi import APIRouter, Query, Response
from fastapi import APIRouter, Depends, Query, Response
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.errors import NOT_FOUND, PARAM_ERROR
from app.core.response import BizException
from app.database import get_db
from app.services.collection_service import HEADERS, UPSTREAM
logger = logging.getLogger(__name__)
router = APIRouter(prefix="/m3u8", tags=["媒体"])
img_router = APIRouter(prefix="/img", tags=["媒体"])
# 只允许相对资源路径，防 SSRF
_PATH_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9/._-]*$")
@@ -32,3 +43,96 @@ async def m3u8_proxy(path: str = Query(min_length=1, max_length=300)):
media_type="application/vnd.apple.mpegurl",
headers={"Cache-Control": "no-cache", "Access-Control-Allow-Origin": "*"},
)
@router.get("/aidrama")
async def aidrama_playlist(
d: int = Query(gt=0, description="源站短剧 id"),
e: int = Query(gt=0, description="源站集 id"),
n: int | None = Query(None, ge=1, description="集号（用于定位分页）"),
db: AsyncSession = Depends(get_db),
):
"""91 短剧播放列表：上游每次播放重新签发 TS 地址，避免存下来的签名过期"""
from app.services import aidrama91_service
try:
source = await aidrama91_service.get_collection_source(db)
text = await aidrama91_service.fetch_playlist(source, d, e, n)
except Exception:
logger.exception("91 播放列表获取失败 drama=%s ep=%s", d, e)
raise BizException(NOT_FOUND, "播放列表获取失败")
return Response(
content=text,
media_type="application/vnd.apple.mpegurl",
headers={"Cache-Control": "no-cache", "Access-Control-Allow-Origin": "*"},
)
@router.get("/vlog")
async def vlog_playlist(
vid: int = Query(gt=0, description="源站短视频 id"),
db: AsyncSession = Depends(get_db),
):
"""91 短视频播放列表：playUrl 带时效签名，播放时现取"""
from app.services import aidrama91_service
try:
source = await aidrama91_service.get_vlog_source(db)
text = await aidrama91_service.fetch_vlog_playlist(source, vid)
except Exception:
logger.exception("91 vlog 播放列表获取失败 vid=%s", vid)
raise BizException(NOT_FOUND, "播放列表获取失败")
return Response(
content=text,
media_type="application/vnd.apple.mpegurl",
headers={"Cache-Control": "no-cache", "Access-Control-Allow-Origin": "*"},
)
@img_router.get("/aidrama", include_in_schema=False)
async def aidrama_image(u: str = Query(min_length=10, max_length=600)):
"""91 封面解密代理：CDN 图为 AES-128-CBC 加密字节，解密后输出真实图片"""
from urllib.parse import urlparse
from app.services import aidrama91_service
parsed = urlparse(u)
if parsed.scheme not in ("http", "https") or not parsed.hostname:
raise BizException(PARAM_ERROR, "非法图片地址")
host = parsed.hostname.lower()
if not (host == aidrama91_service.IMG_HOST_SUFFIX.lstrip(".")
or host.endswith(aidrama91_service.IMG_HOST_SUFFIX)):
raise BizException(PARAM_ERROR, "图片域名不在白名单")
async with httpx.AsyncClient(follow_redirects=True) as client:
resp = await client.get(u, headers={
"User-Agent": "Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36",
}, timeout=20)
if resp.status_code != 200 or not resp.content:
raise BizException(NOT_FOUND, "图片获取失败")
try:
data = aidrama91_service.decrypt_image(resp.content)
except Exception:
logger.exception("91 封面解密失败 url=%s", u[:120])
raise BizException(NOT_FOUND, "图片解密失败")
# 识别真实格式（密文无 padding，尾部可能有 1 字节冗余，浏览器可正常渲染）
if data[:3] == b"\xff\xd8\xff":
mime = "image/jpeg"
elif data[:8] == b"\x89PNG\r\n\x1a\n":
mime = "image/png"
elif data[:6] in (b"GIF87a", b"GIF89a"):
mime = "image/gif"
else:
raise BizException(NOT_FOUND, "图片解密异常")
return Response(
content=data,
media_type=mime,
headers={"Cache-Control": "public, max-age=86400",
"Access-Control-Allow-Origin": "*"},
)
server/app/api/v1/system.py
+295
View File
@@ -0,0 +1,295 @@
"""系统维护：数据库备份 / 恢复（仅 super 角色）
实现要点：
- 备份：mysqldump 全库导出 → gzip 压缩 → 存到 runtime/backups/
- 恢复：上传 .sql.gz → gunzip → mysql 导入。**高危操作**，调用前需前端二次确认
- 所有文件操作限定在 backups/ 目录内，文件名做白名单校验防路径穿越
- 下载/恢复端点均验证 admin 身份，避免泄露数据库内容
"""
import asyncio
import gzip
import logging
import re
import shutil
from datetime import datetime
from pathlib import Path
from urllib.parse import urlparse
from fastapi import APIRouter, Depends, UploadFile, File
from fastapi.responses import FileResponse
from app.config import get_settings
from app.core import errors
from app.core.response import ok, BizException
from app.deps import AdminIdentity, require_roles
router = APIRouter(prefix="/admin/system", tags=["管理后台-系统"])
logger = logging.getLogger(__name__)
require_super = require_roles()
# 备份目录（项目 runtime/backups/，与 uploads/videos 平级）
BACKUP_DIR = Path("/root/project/video/runtime/backups")
BACKUP_DIR.mkdir(parents=True, exist_ok=True)
# 备份文件名白名单：video_backup_YYYYMMDD_HHMMSS.sql.gz
_FILENAME_RE = re.compile(r"^video_backup_\d{8}_\d{6}\.sql\.gz$")
def _parse_db_url() -> dict:
"""从 settings.database_url 解析 mysqldump 需要的连接参数"""
url = get_settings().database_url
# mysql+aiomysql://user:pass@host:port/dbname?charset=utf8mb4
p = urlparse(url)
return {
"host": p.hostname or "127.0.0.1",
"port": p.port or 3306,
"user": p.username or "root",
"password": p.password or "",
"database": p.path.lstrip("/").split("?")[0],
}
def _validate_filename(filename: str) -> Path:
"""校验文件名合法性，返回完整路径。任何非白名单字符直接拒绝"""
if not _FILENAME_RE.match(filename):
raise BizException(errors.INVALID_PARAMS, "非法的备份文件名")
path = BACKUP_DIR / filename
# resolve 后必须仍在 backups 目录内（防符号链接逃逸）
if not str(path.resolve()).startswith(str(BACKUP_DIR.resolve())):
raise BizException(errors.INVALID_PARAMS, "非法的备份文件路径")
return path
@router.get("/backups")
async def list_backups(admin: AdminIdentity = Depends(require_super)):
"""列出所有备份文件（按修改时间倒序）"""
items = []
for f in BACKUP_DIR.glob("video_backup_*.sql.gz"):
if not _FILENAME_RE.match(f.name):
continue
stat = f.stat()
items.append({
"filename": f.name,
"size": stat.st_size,
"created_at": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S"),
})
items.sort(key=lambda x: x["filename"], reverse=True)
return ok({"items": items, "total": len(items)})
@router.post("/backup")
async def create_backup(admin: AdminIdentity = Depends(require_super)):
"""触发一次备份：mysqldump → gzip。同步等待完成（一般几秒到几十秒）"""
db = _parse_db_url()
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
filename = f"video_backup_{ts}.sql.gz"
target = BACKUP_DIR / filename
# mysqldump 通过 stdin 传密码避免命令行泄露（MYSQL_PWD 环境变量也可，但子进程隔离更稳）
cmd = [
"mysqldump",
f"--host={db['host']}",
f"--port={db['port']}",
f"--user={db['user']}",
"--single-transaction", # InnoDB 一致性快照，不锁表
"--quick", # 大表逐行读取，避免内存撑爆
"--default-character-set=utf8mb4",
"--routines", "--triggers",
db["database"],
]
try:
proc_dump = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env={"MYSQL_PWD": db["password"], "PATH": "/usr/bin:/bin"},
)
# 用 gzip 流式压缩写入
with open(target, "wb") as fout:
with gzip.GzipFile(fileobj=fout, mode="wb", compresslevel=6) as gz:
while True:
chunk = await proc_dump.stdout.read(1 << 20) # 1MB
if not chunk:
break
gz.write(chunk)
stderr = await proc_dump.stderr.read()
await proc_dump.wait()
if proc_dump.returncode != 0:
target.unlink(missing_ok=True)
err_msg = stderr.decode("utf-8", errors="ignore")[:500]
logger.error(f"mysqldump failed: {err_msg}")
raise BizException(errors.SERVER_ERROR, f"备份失败：{err_msg}")
size = target.stat().st_size
if size < 100: # 异常小文件视为失败
target.unlink(missing_ok=True)
raise BizException(errors.SERVER_ERROR, "备份文件异常（过小）")
logger.info(f"DB backup created: {filename} ({size} bytes) by admin#{admin.admin_id}")
return ok({"filename": filename, "size": size})
except BizException:
raise
except Exception as e:
target.unlink(missing_ok=True)
logger.exception("backup failed")
raise BizException(errors.SERVER_ERROR, f"备份异常：{e}")
@router.get("/backups/{filename}/download")
async def download_backup(filename: str, admin: AdminIdentity = Depends(require_super)):
"""下载备份文件"""
path = _validate_filename(filename)
if not path.exists():
raise BizException(errors.NOT_FOUND, "备份文件不存在")
return FileResponse(
path,
media_type="application/gzip",
filename=filename,
)
@router.delete("/backups/{filename}")
async def delete_backup(filename: str, admin: AdminIdentity = Depends(require_super)):
"""删除指定备份"""
path = _validate_filename(filename)
if not path.exists():
raise BizException(errors.NOT_FOUND, "备份文件不存在")
path.unlink()
logger.info(f"DB backup deleted: {filename} by admin#{admin.admin_id}")
return ok()
@router.post("/restore")
async def restore_backup(
file: UploadFile = File(...),
admin: AdminIdentity = Depends(require_super),
):
"""上传 .sql.gz 并导入数据库（覆盖现有数据，高危操作）。
流程：
1. 接收上传文件，暂存到 backups/_restore_tmp.sql.gz
2. gunzip 解压 + mysql 导入（事务由 dump 内的 SET 语句保证）
3. 无论成败清理临时文件
"""
if not file.filename or not file.filename.endswith(".sql.gz"):
raise BizException(errors.INVALID_PARAMS, "仅支持 .sql.gz 格式")
db = _parse_db_url()
tmp_path = BACKUP_DIR / "_restore_tmp.sql.gz"
try:
# 限制 500MB
size = 0
with open(tmp_path, "wb") as f:
while True:
chunk = await file.read(1 << 20)
if not chunk:
break
size += len(chunk)
if size > 500 * 1024 * 1024:
raise BizException(errors.INVALID_PARAMS, "备份文件过大（>500MB）")
f.write(chunk)
# 解压 + 导入（用 zcat 管道到 mysql，避免中间落地大 SQL 文件）
# gunzip -c file.sql.gz | mysql -h.. -u.. dbname
zcat_cmd = ["gunzip", "-c", str(tmp_path)]
mysql_cmd = [
"mysql",
f"--host={db['host']}",
f"--port={db['port']}",
f"--user={db['user']}",
"--default-character-set=utf8mb4",
db["database"],
]
proc_zcat = await asyncio.create_subprocess_exec(
*zcat_cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
proc_mysql = await asyncio.create_subprocess_exec(
*mysql_cmd,
stdin=proc_zcat.stdout,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env={"MYSQL_PWD": db["password"], "PATH": "/usr/bin:/bin"},
)
# zcat.stdout 已被 mysql 消费，避免 zcat 因管道堵塞而无法退出
if proc_zcat.stdout is not None:
proc_zcat.stdout = None # 让 asyncio 不要等待它
_, zcat_err = await proc_zcat.wait(), b""
mysql_out, mysql_err = await proc_mysql.communicate()
if proc_mysql.returncode != 0:
err_msg = mysql_err.decode("utf-8", errors="ignore")[:500]
logger.error(f"mysql restore failed: {err_msg}")
raise BizException(errors.SERVER_ERROR, f"恢复失败：{err_msg}")
logger.warning(
f"DB restored from upload ({file.filename}, {size} bytes) by admin#{admin.admin_id}"
)
return ok({"restored": True, "size": size})
except BizException:
raise
except Exception as e:
logger.exception("restore failed")
raise BizException(errors.SERVER_ERROR, f"恢复异常：{e}")
finally:
tmp_path.unlink(missing_ok=True)
@router.post("/backups/{filename}/restore")
async def restore_from_existing(
filename: str,
admin: AdminIdentity = Depends(require_super),
):
"""从服务器已有备份恢复（等价于先下载再上传恢复，但省去传输）"""
path = _validate_filename(filename)
if not path.exists():
raise BizException(errors.NOT_FOUND, "备份文件不存在")
db = _parse_db_url()
try:
zcat_cmd = ["gunzip", "-c", str(path)]
mysql_cmd = [
"mysql",
f"--host={db['host']}",
f"--port={db['port']}",
f"--user={db['user']}",
"--default-character-set=utf8mb4",
db["database"],
]
proc_zcat = await asyncio.create_subprocess_exec(
*zcat_cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
proc_mysql = await asyncio.create_subprocess_exec(
*mysql_cmd,
stdin=proc_zcat.stdout,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
env={"MYSQL_PWD": db["password"], "PATH": "/usr/bin:/bin"},
)
if proc_zcat.stdout is not None:
proc_zcat.stdout = None
await proc_zcat.wait()
_, mysql_err = await proc_mysql.communicate()
if proc_mysql.returncode != 0:
err_msg = mysql_err.decode("utf-8", errors="ignore")[:500]
logger.error(f"mysql restore failed: {err_msg}")
raise BizException(errors.SERVER_ERROR, f"恢复失败：{err_msg}")
logger.warning(f"DB restored from {filename} by admin#{admin.admin_id}")
return ok({"restored": True, "filename": filename})
except BizException:
raise
except Exception as e:
logger.exception("restore from existing failed")
raise BizException(errors.SERVER_ERROR, f"恢复异常：{e}")
server/app/api/v1/videos.py
+54 -2
View File
@@ -24,11 +24,13 @@ UPLOAD_RATE_LIMIT = 5
# 注意：/videos/feed、/videos/categories、/videos/search、/videos/upload/* 须先于 /videos/{video_id} 注册
@router.get("/feed")
async def get_feed(cursor: str | None = Query(None), limit: int = Query(10, ge=1, le=20),
category_id: int | None = Query(None),
category_id: int | None = Query(None), source_id: int | None = Query(None),
type: str = Query("recommend", pattern="^(recommend|latest|hot|liked|viewed)$"),
group: str | None = Query(None, pattern="^(series)$"),
db: AsyncSession = Depends(get_db), redis: aioredis.Redis = Depends(get_redis),
identity: Identity = Depends(get_identity)):
return ok(await video_service.get_feed(db, redis, identity, type, cursor, limit, category_id))
return ok(await video_service.get_feed(
db, redis, identity, type, cursor, limit, category_id, source_id, group))
@router.get("/categories")
@@ -36,6 +38,12 @@ async def get_categories(db: AsyncSession = Depends(get_db)):
return ok(await video_service.list_categories(db))
@router.get("/sources")
async def get_sources(db: AsyncSession = Depends(get_db)):
"""内容来源列表（麻豆/91...），供 App 分类页筛选"""
return ok(await video_service.list_sources(db))
@router.get("/search")
async def search_videos(request: Request, keyword: str = Query(min_length=1, max_length=30),
sort: str = Query("hot", pattern="^(hot|latest)$"),
@@ -103,6 +111,38 @@ async def delete_video(video_id: int, db: AsyncSession = Depends(get_db),
return ok()
# 站点管理员（按注册邮箱判定）：可在 App 长按弹窗里下架任意视频
SITE_ADMIN_EMAIL = "256kb@gmail.com"
@router.post("/{video_id}/offline")
async def offline_video_app(
video_id: int,
reason: str = Form(default="管理员长按下架"),
db: AsyncSession = Depends(get_db),
redis: aioredis.Redis = Depends(get_redis),
identity: Identity = Depends(require_user),
):
"""App 端长按下架（仅限指定管理员邮箱）；与后台 /admin/videos/{id}/offline 复用同一业务"""
from sqlalchemy import select
from app.core import errors
from app.core.response import BizException
from app.deps import AdminIdentity
from app.models.user import User
from app.services import admin_service
user = (await db.execute(
select(User).where(User.id == identity.user_id)
)).scalars().first()
if user is None or (user.email or "").lower() != SITE_ADMIN_EMAIL:
raise BizException(errors.FORBIDDEN, "无权限")
admin = AdminIdentity(admin_id=user.id, role="super")
await admin_service.offline_video(db, redis, admin, video_id, reason, ip="app")
return ok()
@router.post("/{video_id}/watch")
async def report_watch(video_id: int, req: WatchReq, db: AsyncSession = Depends(get_db),
redis: aioredis.Redis = Depends(get_redis),
@@ -141,6 +181,7 @@ async def get_series(series_id: int, db: AsyncSession = Depends(get_db)):
"title": series.title,
"cover_url": series.cover_url,
"description": series.description,
"source_id": series.source_id or 0,
"total_episodes": len(eps),
},
"episodes": [
@@ -181,6 +222,15 @@ async def get_video_series(video_id: int, db: AsyncSession = Depends(get_db)):
prev_ep = all_eps[current_idx - 1] if current_idx > 0 else None
next_ep = all_eps[current_idx + 1] if 0 <= current_idx < len(all_eps) - 1 else None
# 系列所属来源名称
from app.models.video import Source
source_name = ""
if series.source_id:
source_name = (await db.execute(
select(Source.name).where(Source.id == series.source_id)
)).scalar_one_or_none() or ""
return ok({
"has_series": True,
"current_episode_no": video.episode_no,
@@ -188,6 +238,8 @@ async def get_video_series(video_id: int, db: AsyncSession = Depends(get_db)):
"id": series.id,
"title": series.title,
"cover_url": series.cover_url,
"source_id": series.source_id or 0,
"source_name": source_name,
"total_episodes": len(all_eps),
},
"episodes": [
server/app/main.py
+4 -1
View File
@@ -69,9 +69,10 @@ from app.config import BASE_DIR
app.mount("/static", StaticFiles(directory=BASE_DIR / "uploads"), name="static")
# 业务路由（按模块逐步注册）
from app.api.v1 import admin, auth, common, incentives, interactions, invite, media, membership, payments, users, videos
from app.api.v1 import admin, auth, common, incentives, interactions, invite, logs, media, membership, payments, users, videos, system
app.include_router(media.router, prefix="/api/v1")
app.include_router(media.img_router, prefix="/api/v1")
app.include_router(auth.router, prefix="/api/v1")
app.include_router(users.router, prefix="/api/v1")
app.include_router(videos.router, prefix="/api/v1")
@@ -86,6 +87,8 @@ app.include_router(incentives.router_referral, prefix="/api/v1")
app.include_router(incentives.router_assets, prefix="/api/v1")
app.include_router(common.router, prefix="/api/v1")
app.include_router(admin.router, prefix="/api/v1")
app.include_router(system.router, prefix="/api/v1")
app.include_router(logs.router, prefix="/api/v1")
app.include_router(invite.router, prefix="/api/v1")
# 推广落地页（根路径 /r/{code}，对应 referral/me 返回的 invite_url）
app.include_router(invite.router_page)
server/app/models/__init__.py
+2 -1
View File
@@ -12,11 +12,12 @@ from app.models.incentive import (
)
from app.models.interaction import Comment, Favorite, Like
from app.models.user import MembershipOrder, User, UserMembership, UserWallet
from app.models.user_action_log import UserActionLog
from app.models.video import Category, Tag, TranscodeTask, Video, VideoTag
__all__ = [
"Base",
"User", "UserWallet", "UserMembership", "MembershipOrder",
"User", "UserWallet", "UserMembership", "MembershipOrder", "UserActionLog",
"Video", "Category", "Tag", "VideoTag", "TranscodeTask",
"Comment", "Like", "Favorite",
"WatchRecord", "PointTransaction", "RechargeOrder", "RechargePackage",
server/app/models/__pycache__/__init__.cpython-311.pyc
BIN
View File
Binary file not shown.
server/app/models/__pycache__/video.cpython-311.pyc
BIN
View File
Binary file not shown.
server/app/models/user_action_log.py
+26
View File
@@ -0,0 +1,26 @@
"""用户操作日志：用于排查 bug 时还原用户行为时间线（仅特定账号开启上报）"""
from datetime import datetime
from sqlalchemy import DateTime, Index, String, func
from sqlalchemy.dialects.mysql import BIGINT
from sqlalchemy.orm import Mapped, mapped_column
from app.models.base import Base
class UserActionLog(Base):
__tablename__ = "user_action_logs"
id: Mapped[int] = mapped_column(BIGINT(unsigned=True), primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(BIGINT(unsigned=True))
action: Mapped[str] = mapped_column(String(50), comment="route_push/player_stopall/api_error/...")
page: Mapped[str] = mapped_column(String(100), default="", comment="路由路径或页面标识")
detail: Mapped[str] = mapped_column(String(1000), default="", comment="行为上下文（JSON 字符串）")
device_info: Mapped[str] = mapped_column(String(200), default="", comment="机型/系统/版本")
app_version: Mapped[str] = mapped_column(String(20), default="", comment="1.3.26+42")
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
__table_args__ = (
Index("idx_user_created", "user_id", "created_at"),
Index("idx_action_created", "action", "created_at"),
)
server/app/models/video.py
+26 -2
View File
@@ -1,7 +1,7 @@
"""视频域：videos / categories / tags / video_tags / transcode_tasks"""
from datetime import datetime
from sqlalchemy import BigInteger, DateTime, Index, SmallInteger, String, func
from sqlalchemy import BigInteger, DateTime, Index, SmallInteger, String, Text, func
from sqlalchemy.dialects.mysql import BIGINT, INTEGER
from sqlalchemy.orm import Mapped, mapped_column
@@ -14,6 +14,7 @@ class Video(Base):
id: Mapped[int] = mapped_column(BIGINT(unsigned=True), primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(BIGINT(unsigned=True), comment="作者")
category_id: Mapped[int] = mapped_column(default=0)
source_id: Mapped[int] = mapped_column(INTEGER(unsigned=True), default=0, comment="内容来源 id")
series_id: Mapped[int | None] = mapped_column(BIGINT(unsigned=True))
episode_no: Mapped[int | None] = mapped_column()
title: Mapped[str] = mapped_column(String(100), default="")
@@ -43,6 +44,7 @@ class Video(Base):
Index("idx_user_status", "user_id", "status"),
Index("idx_feed", "status", "published_at"),
Index("idx_category", "category_id", "status"),
Index("idx_video_source", "source_id", "status"),
)
@@ -104,6 +106,7 @@ class Series(Base):
description: Mapped[str] = mapped_column(String(500), default="")
author_id: Mapped[int | None] = mapped_column(BIGINT(unsigned=True))
category_id: Mapped[int] = mapped_column(default=0)
source_id: Mapped[int] = mapped_column(INTEGER(unsigned=True), default=0, comment="内容来源 id")
total_episodes: Mapped[int] = mapped_column(default=0)
view_count: Mapped[int] = mapped_column(BigInteger, default=0)
status: Mapped[int] = mapped_column(SmallInteger, default=1, comment="1正常 0隐藏")
@@ -113,9 +116,23 @@ class Series(Base):
__table_args__ = (
Index("idx_series_cat_status", "category_id", "status"),
Index("idx_series_status_created", "status", "created_at"),
Index("idx_series_source", "source_id", "status"),
)
class Source(Base):
"""内容来源（片商）：麻豆、91 等。视频/系列/采集源都按它区分出处"""
__tablename__ = "sources"
id: Mapped[int] = mapped_column(INTEGER(unsigned=True), primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(String(50), unique=True, comment="来源名称，如 麻豆")
code: Mapped[str] = mapped_column(String(50), default="", comment="英文/拼音标识，如 madou")
sort: Mapped[int] = mapped_column(default=0, comment="越小越靠前")
status: Mapped[int] = mapped_column(SmallInteger, default=1, comment="1启用 0停用")
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
class CollectionRecord(Base):
"""采集记录：外部内容源 → 本站系列的映射（幂等导入）"""
@@ -136,14 +153,21 @@ class CollectionRecord(Base):
class CollectionSource(Base):
"""采集源配置：分类名 + 列表URL模板（后台自助添加）"""
"""采集源配置：分类名 + 列表URL模板（后台自助添加）
kind: url（通用 URL 模板，单视频） / aidrama91（91 短剧接口，按系列+集数入库）
config: JSON 字符串，存放接口类采集源所需配置（账号/密码/分类id/每页条数）
"""
__tablename__ = "collection_sources"
id: Mapped[int] = mapped_column(BIGINT(unsigned=True), primary_key=True, autoincrement=True)
name: Mapped[str] = mapped_column(String(50), comment="分类名")
kind: Mapped[str] = mapped_column(String(20), default="url", comment="采集类型 url/aidrama91")
list_url: Mapped[str] = mapped_column(String(500), comment="列表URL模板，{{page}}为页码占位")
config: Mapped[str] = mapped_column(Text, default="", comment="JSON 配置")
category_id: Mapped[int] = mapped_column(INTEGER(unsigned=True), comment="本站分类 id")
source_id: Mapped[int] = mapped_column(INTEGER(unsigned=True), default=0, comment="内容来源 id（采集入库时打标）")
status: Mapped[int] = mapped_column(SmallInteger, default=1, comment="1启用 0停用")
last_sync_at: Mapped[datetime | None] = mapped_column(DateTime)
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
server/app/services/__pycache__/admin_service.cpython-311.pyc
BIN
View File
Binary file not shown.
server/app/services/__pycache__/video_service.cpython-311.pyc
BIN
View File
Binary file not shown.
server/app/services/admin_service.py
+20 -3
View File
@@ -155,7 +155,7 @@ async def dashboard(db: AsyncSession) -> dict:
# ---------- 视频审核 / 管理 ----------
def _video_item(video: Video, author: User | None) -> dict:
def _video_item(video: Video, author: User | None, source_name: str = "") -> dict:
return {
"video_id": video.id,
"title": video.title,
@@ -164,6 +164,8 @@ def _video_item(video: Video, author: User | None) -> dict:
"video_url": video.video_url,
"duration": video.duration,
"category_id": video.category_id or 0,
"source_id": video.source_id or 0,
"source_name": source_name,
"series_id": video.series_id,
"episode_no": video.episode_no,
"status": video.status,
@@ -207,7 +209,7 @@ async def list_pending_videos(db: AsyncSession, page: int, page_size: int) -> di
async def list_videos(
db: AsyncSession, keyword: str | None, status: int | None, author_id: int | None,
page: int, page_size: int,
page: int, page_size: int, source_id: int | None = None,
) -> dict:
"""视频列表（11.25）"""
stmt = select(Video)
@@ -218,12 +220,27 @@ async def list_videos(
stmt = stmt.where(Video.status == status)
if author_id is not None:
stmt = stmt.where(Video.user_id == author_id)
if source_id is not None:
stmt = stmt.where(Video.source_id == source_id)
total = (await db.execute(select(func.count()).select_from(stmt.subquery()))).scalar() or 0
rows = (await db.execute(
stmt.order_by(Video.id.desc()).offset((page - 1) * page_size).limit(page_size)
)).scalars().all()
authors = await _authors_map(db, rows)
return {"total": total, "list": [_video_item(v, authors.get(v.user_id)) for v in rows]}
names = await source_names_map(db, {v.source_id for v in rows})
return {"total": total,
"list": [_video_item(v, authors.get(v.user_id), names.get(v.source_id, "")) for v in rows]}
async def source_names_map(db: AsyncSession, ids: set[int]) -> dict[int, str]:
"""来源 id → 名称"""
from app.models.video import Source
ids = {i for i in ids if i}
if not ids:
return {}
rows = (await db.execute(select(Source.id, Source.name).where(Source.id.in_(ids)))).all()
return {r.id: r.name for r in rows}
async def review_video(
server/app/services/aidrama91_service.py
+850
View File
@@ -0,0 +1,850 @@
"""91 短剧（AI短剧）采集：加密接口客户端 + 系列/集数入库
上游协议：pwa.php 单入口，AES-256-CFB(segment 128) 加密请求体 + MD5(SHA256) 签名。
时间戳签名有效期约 4 小时，因此**不存原始 m3u8 地址**，改为存本站代理地址，
播放时再向上游换取新鲜签名（见 /api/v1/m3u8/aidrama）。
"""
import asyncio
import hashlib
import json
import logging
import random
import string
import time
from datetime import datetime
import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.video import (
Category,
CollectionRecord,
CollectionSource,
Series,
Tag,
Video,
VideoTag,
)
logger = logging.getLogger(__name__)
KIND = "aidrama91"
VLOG_KIND = "aidrama91_vlog" # 91 非系列短视频
SOURCE_KEY = "91aidrama" # collection_records.source_key
VLOG_SOURCE_KEY = "91vlog" # 短视频采集记录 key
SOURCE_CODE = "91" # sources.code
SELF_API_BASE = "https://vapi.256kb.top"
AUTHOR_USER_ID = 1
VLOG_CATEGORY_NAME = "91短视频" # vlog 无真实分类，归到本地分类
# 协议常量（前端 JS 逆向）
AES_KEY = b"cc88ddc9357ff461e08f047aedee692b"
AES_IV = b"e89225cfbbimgkcu"
SIGN_KEY = "cc88ddc9357ff461e08f047aedee692b"
APP_VERSION = "7.5.0"
CLIENT_ORIGIN = "https://pwa4.jl8v0p5t.cc"
# 封面图解密：CDN 返回的是 AES-128-CBC 加密的图片字节（无 padding），
# 源站前端 DecryptImage 解密后展示 —— 本站通过 /api/v1/img/aidrama 代理解密
IMG_AES_KEY = b"f5d965df75336270"
IMG_AES_IV = b"97b60394abc2fbe1"
IMG_HOST_SUFFIX = ".drleao.cn" # 封面 CDN 域名白名单（防 SSRF）
HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36"
),
"Origin": CLIENT_ORIGIN,
"Referer": f"{CLIENT_ORIGIN}/",
"Accept": "application/json, text/plain, */*",
"Content-Type": "application/x-www-form-urlencoded",
}
OAUTH_CHARSET = string.ascii_letters + string.digits
MAX_PAGE_LIMIT = 20 # 上游单页上限（传 100 实际只返回 20）
# ---------- 加密 / 签名 ----------
def _cfb(key: bytes, iv: bytes):
try:
from cryptography.hazmat.decrepit.ciphers.modes import CFB
except ImportError: # cryptography < 43
from cryptography.hazmat.primitives.ciphers.modes import CFB
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms
return Cipher(algorithms.AES(key), CFB(iv))
def _encrypt(plaintext: str) -> str:
op = _cfb(AES_KEY, AES_IV).encryptor()
return (op.update(plaintext.encode()) + op.finalize()).hex().upper()
def _decrypt(hex_data: str) -> str:
op = _cfb(AES_KEY, AES_IV).decryptor()
return (op.update(bytes.fromhex(hex_data)) + op.finalize()).decode("utf-8", "replace")
def _new_oauth_id() -> str:
return "".join(random.choices(OAUTH_CHARSET, k=16)) + str(int(time.time() * 1000))
async def _post(client: httpx.AsyncClient, api_url: str, payload: dict) -> dict:
"""加密并发送，返回解密后的业务数据（已剥掉外层包装）"""
ts_ms = int(time.time() * 1000)
payload = dict(payload, timestamp=ts_ms)
data_hex = _encrypt(json.dumps(payload, separators=(",", ":")))
ts_sec = ts_ms // 1000
raw = f"client=pwa&data={data_hex}&timestamp={ts_sec}{SIGN_KEY}"
sign = hashlib.md5(hashlib.sha256(raw.encode()).hexdigest().encode()).hexdigest()
resp = await client.post(
api_url,
content=f"client=pwa&timestamp={ts_sec}&data={data_hex}&sign={sign}",
headers=HEADERS,
timeout=30,
)
if resp.status_code != 200:
raise RuntimeError(f"上游 HTTP {resp.status_code}（oauth_id 可能未注册或已被限流）")
body = resp.json()
if body.get("errcode") not in (0, None):
raise RuntimeError(f"上游错误: {str(body)[:200]}")
payload_text = body.get("data")
if not payload_text:
return body
return json.loads(_decrypt(payload_text))
def _base(oauth_id: str, token: str) -> dict:
return {
"oauth_type": "pwa", "oauth_id": oauth_id, "trace_id": "",
"aff_x_code": "", "version": APP_VERSION, "token": token,
}
# ---------- 会话 ----------
async def create_session(
client: httpx.AsyncClient, api_url: str, account: str, password: str
) -> tuple[str, str]:
"""注册 oauth_id 并登录，返回 (oauth_id, token)"""
oauth_id = _new_oauth_id()
# 上游对未注册的 oauth_id 直接 503，必须先 system/index
await _post(client, api_url, dict(_base(oauth_id, ""), mod="system", code="index", data={}))
result = await _post(client, api_url, dict(
_base(oauth_id, ""), mod="user", code="regOrLogin",
account=account, password=password, type="login",
))
inner = result.get("data", result) if isinstance(result, dict) else {}
token = inner.get("token") or ""
if not token:
raise RuntimeError(f"91 登录失败: {inner.get('msg') or str(result)[:150]}")
return oauth_id, token
# 进程内 token 缓存（2 个 uvicorn worker → 最多 2 次登录）；失效时自动重登
_TOKENS: dict[int, tuple[str, str, float]] = {}
TOKEN_TTL = 3600
async def open_session(
client: httpx.AsyncClient, source: CollectionSource, force: bool = False
) -> tuple[str, str]:
"""获取可用的 (oauth_id, token)：进程缓存 → DB 配置 → 重新登录"""
key = int(source.id or 0)
cfg = load_config(source)
if not force:
hit = _TOKENS.get(key)
if hit and hit[2] > time.time():
return hit[0], hit[1]
if cfg.get("oauth_id") and cfg.get("token"):
_TOKENS[key] = (cfg["oauth_id"], cfg["token"], time.time() + TOKEN_TTL)
return cfg["oauth_id"], cfg["token"]
oauth_id, token = await create_session(
client, source.list_url, cfg["account"], cfg["password"])
_TOKENS[key] = (oauth_id, token, time.time() + TOKEN_TTL)
return oauth_id, token
def load_config(source: CollectionSource) -> dict:
try:
cfg = json.loads(source.config or "{}")
except json.JSONDecodeError:
cfg = {}
cfg.setdefault("account", "")
cfg.setdefault("password", "")
cfg.setdefault("category_ids", [4])
cfg.setdefault("page_size", MAX_PAGE_LIMIT)
return cfg
# ---------- 业务接口 ----------
async def fetch_list(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
page: int, limit: int, category_ids: list[int],
) -> dict:
"""短剧列表；返回 {list, total, has_more}"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="aidrama", code="home",
page=page, limit=min(limit, MAX_PAGE_LIMIT), pay_type="all", sort="all",
category_ids=category_ids or [4],
))
inner = data.get("data") or {}
return {
"list": inner.get("list") or [],
"total": int(inner.get("total") or 0),
"has_more": bool(inner.get("has_more")),
}
async def fetch_detail(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
) -> dict:
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="aidrama", code="detail", drama_id=drama_id,
))
return (data.get("data") or {}).get("row") or {}
async def fetch_episodes(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
drama_id: int, page: int = 1, limit: int = MAX_PAGE_LIMIT,
) -> dict:
"""集数列表；返回 {list, has_more}"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="aidrama", code="episodes",
drama_id=drama_id, page=page, limit=min(limit, MAX_PAGE_LIMIT),
))
inner = data.get("data") or {}
return {"list": inner.get("list") or [], "has_more": bool(inner.get("has_more"))}
async def fetch_all_episodes(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, drama_id: int
) -> list[dict]:
"""翻页取全部集数（每页 20 上限）"""
out: list[dict] = []
page = 1
while True:
part = await fetch_episodes(client, api_url, oauth_id, token, drama_id, page=page)
out.extend(part["list"])
if not part["has_more"] or not part["list"]:
break
page += 1
return out
# ---------- vlog 短视频接口 ----------
VLOG_MAX_LIMIT = 50 # 实测 vlog 单页上限 50（传 100 只返回 50）
async def fetch_vlog_list(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str,
page: int, limit: int = VLOG_MAX_LIMIT, vlog_type: str = "new",
) -> dict:
"""vlog 列表（非系列短视频）；返回 {list, has_more}"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="choice", code="vlog",
type=vlog_type, page=page, limit=min(limit, VLOG_MAX_LIMIT),
))
lst = data.get("data") or []
if not isinstance(lst, list):
lst = []
return {"list": lst, "has_more": len(lst) >= min(limit, VLOG_MAX_LIMIT)}
async def fetch_vlog_detail(
client: httpx.AsyncClient, api_url: str, oauth_id: str, token: str, vid: int
) -> dict:
"""vlog 详情（含 playUrl / tags）"""
data = await _post(client, api_url, dict(
_base(oauth_id, token), mod="index", code="detail", id=vid))
return data.get("data") or {}
# ---------- 字段解析 ----------
def _pick_url(raw: str | dict, prefer: str = "720") -> str:
"""封面字段是 JSON 字符串或 dict：{"ori","360","720"}，取指定档位"""
if not raw:
return ""
try:
obj = json.loads(raw) if isinstance(raw, str) else raw
except json.JSONDecodeError:
return raw if isinstance(raw, str) else ""
if not isinstance(obj, dict):
return ""
return obj.get(prefer) or obj.get("ori") or ""
def proxied_cover(url: str) -> str:
"""封面地址改写为本站解密代理（CDN 图是加密的，直连无法显示）"""
if not url or "/api/v1/img/aidrama" in url:
return url
if not url.startswith(("http://", "https://")):
return url
from urllib.parse import quote
return f"{SELF_API_BASE}/api/v1/img/aidrama?u={quote(url, safe='')}"
def decrypt_image(data: bytes) -> bytes:
"""AES-128-CBC(NoPadding) 解密 CDN 加密图片，返回原始 JPEG/PNG 字节"""
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
op = Cipher(algorithms.AES(IMG_AES_KEY), modes.CBC(IMG_AES_IV)).decryptor()
return op.update(data) + op.finalize()
def _play_url(drama_id: int, episode_id: int, episode_no: int | None) -> str:
"""本站代理地址（含 m3u8 字样，App 端会走本地缓存代理）"""
url = f"{SELF_API_BASE}/api/v1/m3u8/aidrama?d={drama_id}&e={episode_id}"
if episode_no:
url += f"&n={episode_no}"
return url
def _safe_dt(ts) -> datetime | None:
try:
n = int(ts)
except (TypeError, ValueError):
return None
return datetime.fromtimestamp(n) if n > 0 else None
def _parse_count(v) -> int:
"""解析播放/点赞等计数，兼容 '143.7万' / '1.2万' / '34' / 34"""
if v is None:
return 0
if isinstance(v, (int, float)):
return int(v)
s = str(v).strip()
if not s:
return 0
if s.endswith("万"):
try:
return int(float(s[:-1]) * 10000)
except ValueError:
return 0
try:
return int(s)
except ValueError:
return 0
def _vlog_play_url(vid: int) -> str:
"""vlog 单视频播放代理地址（含 m3u8 字样，App 走本地缓存）"""
return f"{SELF_API_BASE}/api/v1/m3u8/vlog?vid={vid}"
# ---------- 入库 ----------
async def _upsert_tags(db: AsyncSession, names: list[str]) -> list[int]:
"""按名称取/建标签，返回 tag id 列表"""
ids: list[int] = []
for name in {n.strip() for n in names if n and n.strip()}:
name = name[:20]
tid = (await db.execute(select(Tag.id).where(Tag.name == name))).scalar_one_or_none()
if not tid:
t = Tag(name=name, video_count=0)
db.add(t)
await db.flush()
tid = t.id
ids.append(tid)
return ids
async def import_drama(
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
drama: dict, oauth_id: str, token: str,
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
) -> tuple[int, int]:
"""导入单部短剧（系列 + 集数视频），返回 (新增集数, 跳过集数)"""
drama_id = int(drama.get("id") or 0)
if not drama_id:
return 0, 0
api_url = api_url or source.list_url
cat = category_id if category_id is not None else source.category_id
sid = source_id if source_id is not None else source.source_id
cfg = load_config(source)
detail = drama
try:
full = await fetch_detail(client, api_url, oauth_id, token, drama_id)
if full:
detail = {**drama, **full}
except Exception:
logger.warning("短剧 %s 详情获取失败，使用列表字段", drama_id)
record = (await db.execute(
select(CollectionRecord).where(
CollectionRecord.source_key == SOURCE_KEY,
CollectionRecord.source_id == drama_id,
)
)).scalar_one_or_none()
cover = proxied_cover(_pick_url(detail.get("cover") or ""))
tags = list(detail.get("tag_list") or []) + list(detail.get("category_list") or [])
# 像既有采集一样：按标题拆分关键词作为 tag
from app.services.title_tags import extract_keywords
tags += extract_keywords(detail.get("title") or "")
total_declared = int(detail.get("total_episodes") or 0)
series = await db.get(Series, record.series_id) if record else None
if series is None:
series = Series(
title=(detail.get("title") or f"短剧{drama_id}")[:100],
cover_url=cover[:500],
description=(detail.get("description") or "")[:500],
author_id=AUTHOR_USER_ID,
category_id=cat,
source_id=sid,
total_episodes=total_declared,
status=1,
)
db.add(series)
await db.flush()
if record:
record.series_id = series.id
else:
db.add(CollectionRecord(
source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id))
await db.flush()
else:
# 已有系列：同步标题/封面/简介（源站可能改过）
series.title = (detail.get("title") or series.title)[:100]
series.cover_url = (cover or series.cover_url)[:500]
series.total_episodes = max(series.total_episodes, total_declared)
existing = set((await db.execute(
select(Video.episode_no).where(
Video.series_id == series.id, Video.episode_no.is_not(None))
)).scalars().all())
episodes = await fetch_all_episodes(client, api_url, oauth_id, token, drama_id)
tag_ids: list[int] | None = None
created = skipped = 0
now = datetime.now()
for ep in episodes:
ep_id = int(ep.get("id") or 0)
ep_no = int(ep.get("episode_number") or 0)
if not ep_id or not ep_no or ep_no in existing:
skipped += 1
continue
ep_cover = proxied_cover(_pick_url(ep.get("cover") or "")) or cover
video = Video(
user_id=AUTHOR_USER_ID,
category_id=cat,
source_id=sid,
series_id=series.id,
episode_no=ep_no,
title=(ep.get("title") or f"第{ep_no}集")[:100],
description=(detail.get("description") or "")[:500],
cover_url=ep_cover[:500],
video_url=_play_url(drama_id, ep_id, ep_no),
duration=int(ep.get("duration") or 0),
status=2,
like_count=int(ep.get("like") or 0),
favorite_count=int(ep.get("favorite_num") or 0),
comment_count=int(ep.get("comment_count") or 0),
published_at=_safe_dt(ep.get("created_at")) or now,
)
db.add(video)
await db.flush()
# 标签挂到每集视频上（系列标签来自 tag_list / category_list）
if tags:
if tag_ids is None:
tag_ids = await _upsert_tags(db, tags)
for tid in tag_ids:
db.add(VideoTag(video_id=video.id, tag_id=tid))
existing.add(ep_no)
created += 1
series.total_episodes = max(len(existing), total_declared)
return created, skipped
async def sync_source(
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
should_cancel=None,
) -> dict:
"""按列表翻页增量采集（已导入的短剧跳过，仍会补同步新集数）
should_cancel: 可选回调，返回 True 时在每页/每条边界退出循环（用于手动取消）
"""
cfg = load_config(source)
stat: dict = {
"kind": KIND, "dramas_new": 0, "dramas_exist": 0,
"episodes_created": 0, "episodes_skipped": 0, "pages": 0, "total": 0,
}
page_size = min(int(cfg.get("page_size") or MAX_PAGE_LIMIT), MAX_PAGE_LIMIT)
category_ids = cfg.get("category_ids") or [4]
def _cancelled() -> bool:
return bool(should_cancel and should_cancel())
for attempt in (1, 2):
try:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=attempt == 2)
api_url = source.list_url
cat = source.category_id
sid = source.source_id
page = 1
while True:
if _cancelled():
logger.info("91 短剧同步被取消（第 %s 页前）", page)
return stat
data = await fetch_list(
client, api_url, oauth_id, token, page, page_size, category_ids)
stat["pages"] = page
stat["total"] = data["total"]
items = data["list"]
logger.info("91 短剧第 %s 页：%s 部（共 %s）", page, len(items), data["total"])
for drama in items:
if _cancelled():
logger.info("91 短剧同步被取消（处理 drama 中）")
return stat
drama_id = int(drama.get("id") or 0)
if not drama_id:
continue
try:
created, skipped = await import_drama(
db, client, source, drama, oauth_id, token,
api_url=api_url, category_id=cat, source_id=sid)
await db.commit()
if created > 0:
stat["dramas_new"] += 1
else:
stat["dramas_exist"] += 1
stat["episodes_created"] += created
stat["episodes_skipped"] += skipped
except Exception:
await db.rollback()
logger.exception("导入 91 短剧 %s 失败，跳过", drama_id)
await asyncio.sleep(delay)
if not data["has_more"] or not items:
break
if max_pages and page >= max_pages:
break
page += 1
await asyncio.sleep(delay)
cfg["oauth_id"], cfg["token"] = oauth_id, token
source.config = json.dumps(cfg, ensure_ascii=False)
break
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
source.last_sync_at = datetime.now()
await db.commit()
raise
logger.warning("91 同步失败，重新登录后重试一次")
await asyncio.sleep(1)
source.last_sync_at = datetime.now()
await db.commit()
return stat
async def refresh_source_id(db: AsyncSession, source: CollectionSource) -> int:
"""确保采集源绑定的 sources 记录存在，返回来源 id"""
if source.source_id:
return source.source_id
from app.models.video import Source
sid = (await db.execute(
select(Source.id).where(Source.code == SOURCE_CODE)
)).scalar_one_or_none()
if not sid:
max_sort = 0
s = Source(name="91", code=SOURCE_CODE, sort=max_sort + 1, status=1)
db.add(s)
await db.flush()
sid = s.id
source.source_id = sid
return sid
async def ensure_local_category(db: AsyncSession, name: str) -> int:
"""按采集源名称找/建本站分类"""
cid = (await db.execute(
select(Category.id).where(Category.name == name)
)).scalar_one_or_none()
if cid:
return cid
c = Category(name=name, sort=99, status=1)
db.add(c)
await db.flush()
return c.id
async def fetch_playlist(source: CollectionSource, drama_id: int, episode_id: int,
episode_no: int | None = None) -> str:
"""换取某集的最新 m3u8 文本（签名有时效，每次播放现取；token 失效自动重登）"""
for attempt in (1, 2):
try:
return await _fetch_playlist_once(
source, drama_id, episode_id, episode_no, force=attempt == 2)
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
raise
logger.warning("91 播放列表获取失败，重新登录后重试一次")
await asyncio.sleep(0.5)
raise RuntimeError("播放列表获取失败")
async def _fetch_playlist_once(
source: CollectionSource, drama_id: int, episode_id: int,
episode_no: int | None, force: bool,
) -> str:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=force)
target: dict | None = None
# 有集号就直接算所在页；否则从第 1 页起扫（最多 5 页兜底）
pages: list[int] = []
if episode_no and episode_no > 0:
pages.append((episode_no - 1) // MAX_PAGE_LIMIT + 1)
pages.extend(p for p in (1, 2, 3, 4, 5) if p not in pages)
for page in pages:
part = await fetch_episodes(
client, source.list_url, oauth_id, token, drama_id, page=page)
for ep in part["list"]:
if int(ep.get("id") or 0) == episode_id:
target = ep
break
if target or not part["has_more"]:
break
if not target:
raise RuntimeError("源站未找到该集")
m3u8 = target.get("full_m3u8") or ""
if not m3u8:
raise RuntimeError("该集暂无可播放地址（可能需付费或未解锁）")
resp = await client.get(
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
resp.raise_for_status()
if not resp.text.lstrip().startswith("#EXTM3U"):
raise RuntimeError("源站播放列表异常")
return resp.text
async def get_collection_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
"""取 91 采集源（播放代理用）"""
stmt = select(CollectionSource).where(CollectionSource.kind == KIND, CollectionSource.status == 1)
if source_id:
stmt = stmt.where(CollectionSource.id == source_id)
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
if not row:
raise RuntimeError("未配置 91 采集源")
return row
async def get_vlog_source(db: AsyncSession, source_id: int | None = None) -> CollectionSource:
"""取 91 短视频采集源（播放代理用）"""
stmt = select(CollectionSource).where(
CollectionSource.kind == VLOG_KIND, CollectionSource.status == 1)
if source_id:
stmt = stmt.where(CollectionSource.id == source_id)
row = (await db.execute(stmt.order_by(CollectionSource.id))).scalars().first()
if not row:
raise RuntimeError("未配置 91 短视频采集源")
return row
# ---------- vlog 入库 ----------
async def import_vlog_video(
db: AsyncSession, client: httpx.AsyncClient, source: CollectionSource,
item: dict, oauth_id: str, token: str,
api_url: str | None = None, category_id: int | None = None, source_id: int | None = None,
) -> bool:
"""导入单条 vlog 短视频，返回是否新建"""
vid = int(item.get("id") or 0)
if not vid:
return False
api_url = api_url or source.list_url
cat = category_id if category_id is not None else source.category_id
sid = source_id if source_id is not None else source.source_id
record = (await db.execute(select(CollectionRecord).where(
CollectionRecord.source_key == VLOG_SOURCE_KEY,
CollectionRecord.source_id == vid,
))).scalar_one_or_none()
if record:
v = await db.get(Video, record.video_id) if record.video_id else None
if v:
return False
# 详情取 playUrl + tags
detail = item
try:
full = await fetch_vlog_detail(client, api_url, oauth_id, token, vid)
if full:
detail = {**item, **full}
except Exception:
logger.warning("vlog %s 详情获取失败，使用列表字段", vid)
cover = proxied_cover(_pick_url(detail.get("thumbImg") or detail.get("cover_thumb") or ""))
title = (detail.get("title") or f"短视频{vid}")[:100]
tags = list(detail.get("tags") or [])
from app.services.title_tags import extract_keywords
tags += extract_keywords(title)
now = datetime.now()
video = Video(
user_id=AUTHOR_USER_ID,
category_id=cat,
source_id=sid,
title=title,
description=(str(detail.get("descriptions") or "")[:500]),
cover_url=cover[:500],
video_url=_vlog_play_url(vid),
duration=int(detail.get("duration") or 0),
status=2,
like_count=_parse_count(detail.get("like")),
favorite_count=0,
comment_count=_parse_count(detail.get("comment")),
view_count=_parse_count(detail.get("play_count")),
published_at=now,
)
db.add(video)
await db.flush()
if tags:
tag_ids = await _upsert_tags(db, tags)
for tid in tag_ids:
db.add(VideoTag(video_id=video.id, tag_id=tid))
if record:
record.video_id = video.id
else:
db.add(CollectionRecord(source_key=VLOG_SOURCE_KEY, source_id=vid, video_id=video.id))
return True
async def sync_vlog_source(
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
should_cancel=None,
) -> dict:
"""vlog 短视频翻页增量采集。should_cancel 返回 True 时每页/每条边界退出"""
cfg = load_config(source)
stat: dict = {
"kind": VLOG_KIND, "videos_new": 0, "videos_exist": 0,
"pages": 0,
}
page_size = min(int(cfg.get("page_size") or VLOG_MAX_LIMIT), VLOG_MAX_LIMIT)
def _cancelled() -> bool:
return bool(should_cancel and should_cancel())
for attempt in (1, 2):
try:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=attempt == 2)
# 缓存源基本字段，避免 rollback 后 ORM 对象过期导致属性访问失败
api_url = source.list_url
category_id = source.category_id
source_id = source.source_id
page = 1
while True:
if _cancelled():
logger.info("91 vlog 同步被取消（第 %s 页前）", page)
return stat
data = await fetch_vlog_list(
client, api_url, oauth_id, token, page, page_size)
stat["pages"] = page
items = data["list"]
logger.info("91 vlog 第 %s 页：%s 条", page, len(items))
for item in items:
if _cancelled():
logger.info("91 vlog 同步被取消（处理 item 中）")
return stat
vid = int(item.get("id") or 0)
if not vid:
continue
try:
created = await import_vlog_video(
db, client, source, item, oauth_id, token,
api_url=api_url, category_id=category_id, source_id=source_id)
await db.commit()
if created:
stat["videos_new"] += 1
else:
stat["videos_exist"] += 1
except Exception:
await db.rollback()
logger.exception("导入 91 vlog %s 失败，跳过", vid)
await asyncio.sleep(delay)
if not data["has_more"] or not items:
break
if max_pages and page >= max_pages:
break
page += 1
await asyncio.sleep(delay)
cfg["oauth_id"], cfg["token"] = oauth_id, token
source.config = json.dumps(cfg, ensure_ascii=False)
break
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
source.last_sync_at = datetime.now()
await db.commit()
raise
logger.warning("91 vlog 同步失败，重新登录后重试一次")
await asyncio.sleep(1)
source.last_sync_at = datetime.now()
await db.commit()
return stat
async def fetch_vlog_playlist(source: CollectionSource, vid: int) -> str:
"""换取 vlog 播放列表（playUrl 带时效签名，播放时现取）"""
for attempt in (1, 2):
try:
return await _fetch_vlog_playlist_once(source, vid, force=attempt == 2)
except Exception:
_TOKENS.pop(int(source.id or 0), None)
if attempt == 2:
raise
logger.warning("91 vlog 播放列表获取失败，重新登录后重试一次")
await asyncio.sleep(0.5)
raise RuntimeError("播放列表获取失败")
async def _fetch_vlog_playlist_once(source: CollectionSource, vid: int, force: bool) -> str:
async with httpx.AsyncClient(follow_redirects=True) as client:
oauth_id, token = await open_session(client, source, force=force)
detail = await fetch_vlog_detail(client, source.list_url, oauth_id, token, vid)
m3u8 = detail.get("playUrl") or ""
if not m3u8:
raise RuntimeError("该视频暂无可播放地址（可能需付费或未解锁）")
resp = await client.get(
m3u8, headers={"User-Agent": HEADERS["User-Agent"]}, timeout=30)
resp.raise_for_status()
if not resp.text.lstrip().startswith("#EXTM3U"):
raise RuntimeError("源站播放列表异常")
return resp.text
server/app/services/collection_service.py
+35 -2
View File
@@ -14,6 +14,7 @@ logger = logging.getLogger(__name__)
# 内容源配置（域名轮换时只需改这里）
SOURCE_KEY = "mdcmai4"
SOURCE_CODE = "madou" # sources 表中的 code，用于给导入内容打「来源」标
UPSTREAM = "https://mdcmai4.xyz"
PRODUCT_ID = 1
CATEGORY_ID = 12 # 本站分类：AI短剧
@@ -37,6 +38,21 @@ def _proxy_url(path: str) -> str:
return f"{SELF_API_BASE}/api/v1/m3u8/proxy?path={quote(path, safe='')}"
async def default_source_id(db: AsyncSession) -> int:
"""麻豆等内容源的来源 id（sources.code 匹配；缺失时取第一个启用来源）"""
from app.models.video import Source
sid = (
await db.execute(select(Source.id).where(Source.code == SOURCE_CODE))
).scalar_one_or_none()
if sid:
return sid
sid = (
await db.execute(select(Source.id).where(Source.status == 1).order_by(Source.sort, Source.id))
).scalars().first()
return sid or 0
async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: int) -> tuple[int, int]:
"""导入单部短剧，返回 (新增集数, 跳过集数)"""
detail = await _fetch_json(client, f"{UPSTREAM}/api/v1/short-dramas/{drama_id}", {"productId": PRODUCT_ID})
@@ -63,6 +79,7 @@ async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: i
description=(detail.get("description") or "")[:500],
author_id=AUTHOR_USER_ID,
category_id=CATEGORY_ID,
source_id=await default_source_id(db),
total_episodes=0,
view_count=int(detail.get("heatCount") or 0),
status=1,
@@ -94,6 +111,7 @@ async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: i
Video(
user_id=AUTHOR_USER_ID,
category_id=CATEGORY_ID,
source_id=series.source_id,
series_id=series.id,
episode_no=ep_no,
title=(ep.get("titleOverride") or ep.get("title") or f"第{ep_no}集")[:100],
@@ -209,6 +227,7 @@ async def _import_video_item(db: AsyncSession, source: CollectionSource, item: d
v = Video(
user_id=AUTHOR_USER_ID,
category_id=source.category_id,
source_id=source.source_id or await default_source_id(db),
title=(item.get("title") or f"视频{vid}")[:100],
description=(item.get("description") or "")[:500],
cover_url=_abs_cover(item)[:500],
@@ -232,13 +251,23 @@ async def sync_source(
max_pages: int = 0,
stop_after_existing: int = 30,
delay: float = 0.25,
should_cancel=None,
) -> dict:
"""按源的 URL 模板翻页采集；连续 stop_after_existing 条已存在则提前停止（增量）"""
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False}
"""按源的 URL 模板翻页采集；连续 stop_after_existing 条已存在则提前停止（增量）。
should_cancel 返回 True 时每页/每条边界退出（用于手动取消）"""
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False, "cancelled": False}
consecutive_existing = 0
def _cancelled() -> bool:
return bool(should_cancel and should_cancel())
async with httpx.AsyncClient(follow_redirects=True) as client:
page = 1
while True:
if _cancelled():
stat["cancelled"] = True
logger.info("通用采集源 %s 同步被取消（第 %s 页前）", source.id, page)
return stat
url = source.list_url.replace("{{page}}", str(page))
data = await _fetch_json(client, url)
items = data.get("items", [])
@@ -246,6 +275,10 @@ async def sync_source(
break
stat["pages"] = page
for item in items:
if _cancelled():
stat["cancelled"] = True
logger.info("通用采集源 %s 同步被取消（处理 item 中）", source.id)
return stat
if await _import_video_item(db, source, item):
stat["created"] += 1
consecutive_existing = 0
server/app/services/title_tags.py
+62
View File
@@ -0,0 +1,62 @@
"""标题关键词提取：从视频/系列标题拆分出词作为 tag。
提取规则（原 scripts/build_tags_from_titles.py，抽成公共模块供采集入库复用）：
1. 番号（如 CAWD-601）及其系列前缀（CAWD）
2. 书名号/方括号内容（≤12 字，去掉「无码破解」「中文字幕」等标记词）
3. jieba TF-IDF 关键词 topK=6（过滤停用词、纯数字、单字）
"""
import re
import jieba.analyse
STOP = set(
"第 集 部 篇 全集 完结 中文字幕 无码 破解 无码破解 字幕 高清 最新 精选 视频 在线 观看 "
"一个 一起 没有 为了 开始 结果 只能 直接 就是 没想到 真爽".split()
)
CODE_RE = re.compile(r"[A-Za-z]{2,6}-\d{2,4}")
EPISODE_RE = re.compile(r"第[0-9一二三四五六七八九十百千零]+[集部期话卷章]")
PUNCT_RE = re.compile(r"[《》「」『』\[\]【】()（）_\-:：,，.。!！?？…、/\\|~～\s]+")
BRACKET_RE = re.compile(r"[《「『\[【(（]([^《》「」『』\[\]【】()（）]{2,30})[》」』\]】)）]")
def extract_keywords(title: str) -> list[str]:
if not title:
return []
kws: list[str] = []
# 1. 番号 + 系列前缀
for code in CODE_RE.findall(title.upper()):
kws.append(code)
prefix = code.split("-")[0]
if len(prefix) >= 2:
kws.append(prefix)
# 2. 书名号/括号内容（剧名/主题）
for inner in BRACKET_RE.findall(title):
inner = inner.strip()
if 2 <= len(inner) <= 12 and inner not in STOP and not CODE_RE.search(inner):
kws.append(inner)
# 3. jieba TF-IDF
t = EPISODE_RE.sub(" ", title)
t = PUNCT_RE.sub(" ", t)
for w in jieba.analyse.extract_tags(t, topK=6):
w = w.strip()
if not (2 <= len(w) <= 12):
continue
if w in STOP or re.fullmatch(r"\d+", w):
continue
kws.append(w)
# 去重保序（按 casefold 键，避免 MySQL 不区分大小写的唯一键冲突）
seen: set[str] = set()
out: list[str] = []
for k in kws:
if k.isascii():
k = k.upper()
key = k.casefold()
if key not in seen:
seen.add(key)
out.append(k)
return out
server/app/services/video_service.py
+141 -11
View File
@@ -20,7 +20,7 @@ from app.models.billing import WatchRecord
from app.models.incentive import VideoEarning
from app.models.interaction import Favorite, Like
from app.models.user import User
from app.models.video import Category, Tag, Video, VideoTag
from app.models.video import Category, Series, Tag, Video, VideoTag
from app.schemas.video import PublishVideoReq, UploadTokenReq
from app.services import membership_service, sensitive_service, storage_service, wallet_service
@@ -79,6 +79,18 @@ async def serialize_videos(
tags_map = await get_tags_map(db, video_ids)
# 内容来源（麻豆 / 91 ...）
from app.models.video import Source
source_ids = {v.source_id for v in videos if v.source_id}
source_map = {}
if source_ids:
source_map = {
r.id: r.name for r in (
await db.execute(select(Source.id, Source.name).where(Source.id.in_(source_ids)))
).all()
}
liked: set[int] = set()
favorited: set[int] = set()
if viewer_user_id:
@@ -101,6 +113,8 @@ async def serialize_videos(
"video_url": v.video_url,
"duration": v.duration,
"category_id": v.category_id,
"source_id": v.source_id or 0,
"source_name": source_map.get(v.source_id, ""),
"author": _author_brief(authors.get(v.user_id)),
"view_count": v.view_count,
"like_count": v.like_count,
@@ -114,6 +128,16 @@ async def serialize_videos(
]
async def list_sources(db: AsyncSession) -> dict:
"""启用中的内容来源列表（App 分类页来源筛选）"""
from app.models.video import Source
rows = (await db.execute(
select(Source).where(Source.status == 1).order_by(Source.sort, Source.id)
)).scalars().all()
return {"list": [{"source_id": s.id, "name": s.name, "code": s.code} for s in rows]}
async def get_tags_map(db: AsyncSession, video_ids: list[int]) -> dict[int, list[str]]:
if not video_ids:
return {}
@@ -157,21 +181,110 @@ async def get_feed(
cursor: str | None,
limit: int,
category_id: int | None,
source_id: int | None = None,
group: str | None = None,
) -> dict:
# group=series：系列型分类（如 AI短剧）按「剧」聚合，每部剧只返回首集一条
if group == "series":
return await _feed_series(db, identity, cursor, limit, category_id, source_id)
if feed_type == "latest":
return await _feed_latest(db, identity, cursor, limit, category_id)
return await _feed_latest(db, identity, cursor, limit, category_id, source_id)
if feed_type in ("liked", "viewed"):
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id)
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id)
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id, source_id)
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id, source_id)
async def _feed_series(
db: AsyncSession,
identity: Identity,
cursor: str | None,
limit: int,
category_id: int | None,
source_id: int | None = None,
) -> dict:
"""系列聚合 Feed：一部剧一条（视频 id 为首集），按剧集播放量倒序。
卡片封面/标题取 Series（回退首集封面），点击后客户端拿到的 video_id 即首集，
播放页会按 /videos/{id}/series 拉全集列表，从而支持选集。
"""
if not category_id:
return {"next_cursor": None, "list": []}
# 仅返回至少有一集已发布的系列
live_series_ids = select(Video.series_id).where(
Video.status == STATUS_PUBLISHED, Video.series_id.isnot(None)
).distinct()
stmt = select(Series).where(
Series.status == 1,
Series.category_id == category_id,
Series.id.in_(live_series_ids),
)
if source_id:
stmt = stmt.where(Series.source_id == source_id)
offset = int(_decode_cursor(cursor).get("o", 0))
rows = list(
(await db.execute(
stmt.order_by(Series.view_count.desc(), Series.id.desc())
.offset(offset)
.limit(limit + 1)
)).scalars()
)
page, rows = (len(rows) > limit), rows[:limit]
if not rows:
return {"next_cursor": None, "list": []}
# 取这些剧的全部已发布集数，按 episode_no + id 找首集，同时统计实际集数
series_ids = [s.id for s in rows]
episodes = list(
(await db.execute(
select(Video).where(
Video.series_id.in_(series_ids), Video.status == STATUS_PUBLISHED
)
)).scalars()
)
first_ep: dict[int, Video] = {}
ep_count: dict[int, int] = {}
for v in episodes:
ep_count[v.series_id] = ep_count.get(v.series_id, 0) + 1
cur = first_ep.get(v.series_id)
if cur is None or _ep_key(v) < _ep_key(cur):
first_ep[v.series_id] = v
first_videos = [first_ep[s.id] for s in rows if s.id in first_ep]
serialized = await serialize_videos(db, first_videos, identity.user_id)
by_id = {it["video_id"]: it for it in serialized}
items = []
for s in rows:
v = first_ep.get(s.id)
if v is None:
continue
it = by_id[v.id]
# 系列封面/标题优先，空值回退首集自身字段
it["title"] = s.title or it["title"]
it["cover_url"] = s.cover_url or it["cover_url"]
it["view_count"] = s.view_count
it["series_id"] = s.id
it["total_episodes"] = ep_count.get(s.id, s.total_episodes)
items.append(it)
next_cursor = _encode_cursor({"o": offset + limit}) if page else None
return {"next_cursor": next_cursor, "list": items}
def _ep_key(v: Video) -> tuple[int, int]:
"""首集排序键：episode_no 缺失视为最大，再按 id 兜底"""
return (v.episode_no if v.episode_no is not None else 1_000_000, v.id)
async def _feed_ranked(
db: AsyncSession, identity: Identity, feed_type: str, cursor: str | None,
limit: int, category_id: int | None,
limit: int, category_id: int | None, source_id: int | None = None,
) -> dict:
"""liked / viewed：按点赞量 / 播放量严格倒序（值 + id 打破并列游标）"""
col = Video.like_count if feed_type == "liked" else Video.view_count
stmt = await _base_pool_stmt(category_id)
stmt = _base_pool_stmt(category_id, source_id)
c = _decode_cursor(cursor)
if c:
val, vid = int(c["n"]), int(c["i"])
@@ -191,18 +304,21 @@ async def _feed_ranked(
}
async def _base_pool_stmt(category_id: int | None):
def _base_pool_stmt(category_id: int | None, source_id: int | None = None):
stmt = select(Video).where(Video.status == STATUS_PUBLISHED, Video.published_at.isnot(None))
if category_id:
stmt = stmt.where(Video.category_id == category_id)
if source_id:
stmt = stmt.where(Video.source_id == source_id)
return stmt
async def _feed_latest(
db: AsyncSession, identity: Identity, cursor: str | None, limit: int, category_id: int | None
db: AsyncSession, identity: Identity, cursor: str | None, limit: int, category_id: int | None,
source_id: int | None = None,
) -> dict:
"""latest：published_at 游标倒序（ts + id 打破并列）"""
stmt = await _base_pool_stmt(category_id)
stmt = _base_pool_stmt(category_id, source_id)
c = _decode_cursor(cursor)
if c:
ts, vid = datetime.fromtimestamp(c["p"]), int(c["i"])
@@ -260,6 +376,7 @@ async def _feed_scored(
cursor: str | None,
limit: int,
category_id: int | None,
source_id: int | None = None,
) -> dict:
"""recommend / hot：候选池打分 → 当日种子确定性加权抽样 → 当日去重（不足兜底）→ 偏移游标"""
now = datetime.now()
@@ -267,7 +384,7 @@ async def _feed_scored(
subject = identity.subject or "anon"
# 1. 候选池：近 30 天已发布，封顶 FEED_POOL_SIZE
stmt = (await _base_pool_stmt(category_id)).where(
stmt = _base_pool_stmt(category_id, source_id).where(
Video.published_at > now - timedelta(days=30)
).order_by(Video.published_at.desc()).limit(FEED_POOL_SIZE)
pool = list((await db.execute(stmt)).scalars())
@@ -505,7 +622,20 @@ async def list_categories(db: AsyncSession) -> list[dict]:
rows = (await db.execute(
select(Category).where(Category.status == 1).order_by(Category.sort, Category.id)
)).scalars().all()
return [{"category_id": c.id, "name": c.name} for c in rows]
# 系列型分类标记（该分类下存在正常状态的系列），App 据此按「剧」聚合展示
cnt_rows = (await db.execute(
select(Series.category_id, func.count()).where(Series.status == 1)
.group_by(Series.category_id)
)).all()
series_cnt = {cid: n for cid, n in cnt_rows}
return [
{
"category_id": c.id,
"name": c.name,
"has_series": series_cnt.get(c.id, 0) > 0,
}
for c in rows
]
# ---------- 上传发布（5.5 / 5.6 / 5.7） ----------
server/scripts/build_tags_from_titles.py
+2 -56
View File
@@ -4,79 +4,25 @@
cd /root/project/video/server
.venv/bin/python scripts/build_tags_from_titles.py [--dry-run]
提取规则：
1. 番号（如 CAWD-601）及其系列前缀（CAWD）
2. 书名号/方括号内容（≤12 字，去掉「无码破解」「中文字幕」等标记词）
3. jieba TF-IDF 关键词 topK=6（过滤停用词、纯数字、单字）
提取规则见 app/services/title_tags.py（幂等可重复执行）。
"""
import argparse
import asyncio
import logging
import re
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import jieba.analyse # noqa: E402
from sqlalchemy import select # noqa: E402
from app.database import SessionFactory, engine # noqa: E402
from app.models.video import Tag, Video, VideoTag # noqa: E402
from app.services.title_tags import extract_keywords # noqa: E402
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
log = logging.getLogger(__name__)
STOP = set(
"第 集 部 篇 全集 完结 中文字幕 无码 破解 无码破解 字幕 高清 最新 精选 视频 在线 观看 "
"一个 一起 没有 为了 开始 结果 只能 直接 就是 没想到 真爽".split()
)
CODE_RE = re.compile(r"[A-Za-z]{2,6}-\d{2,4}")
EPISODE_RE = re.compile(r"第[0-9一二三四五六七八九十百千零]+[集部期话卷章]")
PUNCT_RE = re.compile(r"[《》「」『』\[\]【】()（）_\-:：,，.。!！?？…、/\\|~～\s]+")
BRACKET_RE = re.compile(r"[《「『\[【(（]([^《》「」『』\[\]【】()（）]{2,30})[》」』\]】)）]")
def extract_keywords(title: str) -> list[str]:
kws: list[str] = []
# 1. 番号 + 系列前缀
for code in CODE_RE.findall(title.upper()):
kws.append(code)
prefix = code.split("-")[0]
if len(prefix) >= 2:
kws.append(prefix)
# 2. 书名号/括号内容（剧名/主题）
for inner in BRACKET_RE.findall(title):
inner = inner.strip()
if 2 <= len(inner) <= 12 and inner not in STOP and not CODE_RE.search(inner):
kws.append(inner)
# 3. jieba TF-IDF
t = EPISODE_RE.sub(" ", title)
t = PUNCT_RE.sub(" ", t)
for w in jieba.analyse.extract_tags(t, topK=6):
w = w.strip()
if not (2 <= len(w) <= 12):
continue
if w in STOP or re.fullmatch(r"\d+", w):
continue
kws.append(w)
# 去重保序（按 casefold 键，避免 MySQL 不区分大小写的唯一键冲突）
seen: set[str] = set()
out: list[str] = []
for k in kws:
if k.isascii():
k = k.upper()
key = k.casefold()
if key not in seen:
seen.add(key)
out.append(k)
return out
async def main(dry_run: bool) -> None:
async with SessionFactory() as db: