1
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
"""add collection_records
|
||||
|
||||
Revision ID: b3c7d9e21f4a
|
||||
Revises: d659ecc1b6e7
|
||||
Create Date: 2026-10-17 12:00:00.000000
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
revision = 'b3c7d9e21f4a'
|
||||
down_revision = 'd659ecc1b6e7'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table('collection_records',
|
||||
sa.Column('id', mysql.BIGINT(unsigned=True), autoincrement=True, nullable=False),
|
||||
sa.Column('source_key', sa.String(length=50), nullable=False, comment='内容源标识'),
|
||||
sa.Column('source_id', mysql.BIGINT(unsigned=True), nullable=False, comment='源站内容 id'),
|
||||
sa.Column('series_id', mysql.BIGINT(unsigned=True), nullable=False, comment='本站系列 id'),
|
||||
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id')
|
||||
)
|
||||
op.create_index('uk_source', 'collection_records', ['source_key', 'source_id'], unique=True)
|
||||
op.create_index('idx_record_series', 'collection_records', ['series_id'], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index('idx_record_series', table_name='collection_records')
|
||||
op.drop_index('uk_source', table_name='collection_records')
|
||||
op.drop_table('collection_records')
|
||||
@@ -0,0 +1,37 @@
|
||||
"""add collection_sources + collection_records.video_id
|
||||
|
||||
Revision ID: c8f2a15e93b7
|
||||
Revises: b3c7d9e21f4a
|
||||
Create Date: 2026-10-17 13:00:00.000000
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import mysql
|
||||
|
||||
revision = 'c8f2a15e93b7'
|
||||
down_revision = 'b3c7d9e21f4a'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table('collection_sources',
|
||||
sa.Column('id', mysql.BIGINT(unsigned=True), autoincrement=True, nullable=False),
|
||||
sa.Column('name', sa.String(length=50), nullable=False, comment='分类名'),
|
||||
sa.Column('list_url', sa.String(length=500), nullable=False, comment='列表URL模板, {{page}}为页码占位'),
|
||||
sa.Column('category_id', mysql.INTEGER(unsigned=True), nullable=False, comment='本站分类id'),
|
||||
sa.Column('status', mysql.SMALLINT(), nullable=False, server_default='1', comment='1启用 0停用'),
|
||||
sa.Column('last_sync_at', sa.DateTime(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id'),
|
||||
sa.UniqueConstraint('list_url', name='uk_list_url')
|
||||
)
|
||||
op.add_column('collection_records', sa.Column('video_id', mysql.BIGINT(unsigned=True), nullable=True, comment='本站视频id(单视频类采集)'))
|
||||
op.alter_column('collection_records', 'series_id', existing_type=mysql.BIGINT(unsigned=True), nullable=True)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.alter_column('collection_records', 'series_id', existing_type=mysql.BIGINT(unsigned=True), nullable=False)
|
||||
op.drop_column('collection_records', 'video_id')
|
||||
op.drop_table('collection_sources')
|
||||
Binary file not shown.
@@ -5,6 +5,8 @@
|
||||
- operator:套餐、活动、看板 → 仪表盘 / 套餐 CRUD / 赠送
|
||||
- 其余接口仅 super
|
||||
"""
|
||||
import logging
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from pydantic import BaseModel, BaseModel as _BM, Field as _F, Field as _Field
|
||||
@@ -19,6 +21,8 @@ from app.services import admin_service
|
||||
|
||||
router = APIRouter(prefix="/admin", tags=["管理后台"])
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 角色依赖:super 自动放行
|
||||
require_reviewer = require_roles("reviewer")
|
||||
require_operator = require_roles("operator")
|
||||
@@ -341,6 +345,15 @@ async def set_user_status(user_id: int, req: schema.UserStatusReq, request: Requ
|
||||
return ok(await admin_service.set_user_status(db, redis, admin, user_id, req, _ip(request)))
|
||||
|
||||
|
||||
@router.put("/users/{user_id}/password")
|
||||
async def reset_user_password(user_id: int, req: schema.UserPasswordResetReq, request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
redis: aioredis.Redis = Depends(get_redis),
|
||||
admin: AdminIdentity = Depends(require_super)):
|
||||
await admin_service.reset_user_password(db, redis, admin, user_id, req, _ip(request))
|
||||
return ok()
|
||||
|
||||
|
||||
@router.post("/points/adjust")
|
||||
async def adjust_points(req: schema.PointsAdjustReq, request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
@@ -558,6 +571,19 @@ async def list_series(keyword: str = "", page: int = 1, page_size: int = 20,
|
||||
rows = (await db.execute(
|
||||
stmt.order_by(Series.id.desc()).offset((page - 1) * page_size).limit(page_size)
|
||||
)).scalars().all()
|
||||
|
||||
# 关联采集记录(标记采集来源,供"同步更新"按钮使用)
|
||||
from app.models.video import CollectionRecord
|
||||
series_ids = [s.id for s in rows]
|
||||
source_map: dict[int, int] = {}
|
||||
if series_ids:
|
||||
recs = (await db.execute(
|
||||
select(CollectionRecord.series_id, CollectionRecord.source_id).where(
|
||||
CollectionRecord.series_id.in_(series_ids)
|
||||
)
|
||||
)).all()
|
||||
source_map = {r.series_id: r.source_id for r in recs}
|
||||
|
||||
return ok({
|
||||
"total": total,
|
||||
"list": [{
|
||||
@@ -565,6 +591,7 @@ async def list_series(keyword: str = "", page: int = 1, page_size: int = 20,
|
||||
"description": s.description or "",
|
||||
"category_id": s.category_id or 0,
|
||||
"total_episodes": s.total_episodes, "status": s.status,
|
||||
"source_id": source_map.get(s.id),
|
||||
"created_at": s.created_at.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
} for s in rows],
|
||||
})
|
||||
@@ -664,6 +691,306 @@ async def bind_video_series(video_id: int, req: _VideoSeriesBind,
|
||||
return ok()
|
||||
|
||||
|
||||
# ---------- 短剧采集 ----------
|
||||
|
||||
_collect_sync_running = False
|
||||
|
||||
|
||||
@router.post("/collection/sync/{series_id}")
|
||||
async def collect_sync_one(series_id: int, request: Request, db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
"""手动同步单部采集短剧(补充源站新出的集数)"""
|
||||
from app.models.admin import AdminLog
|
||||
from app.models.video import CollectionRecord
|
||||
from app.services import collection_service
|
||||
from sqlalchemy import select
|
||||
|
||||
source_id = (await db.execute(
|
||||
select(CollectionRecord.source_id).where(
|
||||
CollectionRecord.source_key == collection_service.SOURCE_KEY,
|
||||
CollectionRecord.series_id == series_id,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if not source_id:
|
||||
raise BizException(errors.NOT_FOUND, "该系列不是采集内容")
|
||||
|
||||
stat = await collection_service.sync_drama_by_id(db, source_id)
|
||||
db.add(AdminLog(admin_id=admin.admin_id, action="collection_sync",
|
||||
target=f"series:{series_id}", detail=str(stat), ip=_ip(request)))
|
||||
await db.commit()
|
||||
return ok(stat)
|
||||
|
||||
|
||||
class _CollectSyncAllReq(BaseModel):
|
||||
pages: int = _Field(2, ge=1, le=20) # 增量同步的列表页数
|
||||
|
||||
|
||||
@router.post("/collection/sync-all")
|
||||
async def collect_sync_all(req: _CollectSyncAllReq, request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
"""触发增量同步(后台异步执行,立即返回)"""
|
||||
import asyncio
|
||||
|
||||
from app.database import SessionFactory
|
||||
from app.models.admin import AdminLog
|
||||
from app.services import collection_service
|
||||
|
||||
global _collect_sync_running
|
||||
if _collect_sync_running:
|
||||
raise BizException(errors.RATE_LIMITED, "已有同步任务进行中")
|
||||
|
||||
db.add(AdminLog(admin_id=admin.admin_id, action="collection_sync_all",
|
||||
target="", detail=f"pages={req.pages}", ip=_ip(request)))
|
||||
await db.commit()
|
||||
|
||||
async def _run() -> None:
|
||||
global _collect_sync_running
|
||||
_collect_sync_running = True
|
||||
try:
|
||||
async with SessionFactory() as s:
|
||||
await collection_service.sync_short_dramas(s, max_pages=req.pages)
|
||||
finally:
|
||||
_collect_sync_running = False
|
||||
|
||||
asyncio.create_task(_run())
|
||||
return ok({"started": True, "pages": req.pages})
|
||||
|
||||
|
||||
# ---------- 采集源管理(单视频类,自助添加)----------
|
||||
|
||||
|
||||
class _SourceCreate(BaseModel):
|
||||
name: str = _Field(min_length=1, max_length=50)
|
||||
list_url: str = _Field(min_length=10, max_length=500)
|
||||
|
||||
|
||||
class _SourceUpdate(BaseModel):
|
||||
name: str | None = _Field(None, min_length=1, max_length=50)
|
||||
list_url: str | None = _Field(None, min_length=10, max_length=500)
|
||||
status: int | None = _Field(None, ge=0, le=1)
|
||||
|
||||
|
||||
def _source_dict(s) -> dict:
|
||||
return {
|
||||
"id": s.id, "name": s.name, "list_url": s.list_url,
|
||||
"category_id": s.category_id, "status": s.status,
|
||||
"last_sync_at": s.last_sync_at.strftime("%Y-%m-%d %H:%M:%S") if s.last_sync_at else "",
|
||||
"created_at": s.created_at.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
||||
|
||||
async def _resolve_category(db: AsyncSession, name: str) -> int:
|
||||
"""按名称找分类,不存在则创建"""
|
||||
from app.models.video import Category
|
||||
from sqlalchemy import func, select
|
||||
|
||||
cid = (await db.execute(select(Category.id).where(Category.name == name))).scalar_one_or_none()
|
||||
if cid:
|
||||
return cid
|
||||
max_sort = (await db.execute(select(func.coalesce(func.max(Category.sort), 0)))).scalar_one()
|
||||
c = Category(name=name, sort=max_sort + 1, status=1)
|
||||
db.add(c)
|
||||
await db.flush()
|
||||
return c.id
|
||||
|
||||
|
||||
@router.get("/collection/sources")
|
||||
async def list_sources(db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
from app.models.video import CollectionSource
|
||||
from sqlalchemy import select
|
||||
|
||||
rows = (await db.execute(select(CollectionSource).order_by(CollectionSource.id.desc()))).scalars().all()
|
||||
return ok({"list": [_source_dict(s) for s in rows]})
|
||||
|
||||
|
||||
@router.post("/collection/sources")
|
||||
async def create_source(req: _SourceCreate, request: Request, db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
"""添加采集源:自动按名称建/找分类"""
|
||||
from app.models.admin import AdminLog
|
||||
from app.models.video import CollectionSource
|
||||
|
||||
if "{{page}}" not in req.list_url:
|
||||
raise BizException(errors.PARAM_ERROR, "URL 需包含 {{page}} 页码占位符")
|
||||
|
||||
cid = await _resolve_category(db, req.name.strip())
|
||||
s = CollectionSource(name=req.name.strip(), list_url=req.list_url.strip(), category_id=cid)
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
db.add(AdminLog(admin_id=admin.admin_id, action="collection_source_create",
|
||||
target=f"source:{s.id}", detail=f"{s.name} {s.list_url}", ip=_ip(request)))
|
||||
await db.commit()
|
||||
await db.refresh(s)
|
||||
return ok(_source_dict(s))
|
||||
|
||||
|
||||
@router.put("/collection/sources/{source_id}")
|
||||
async def update_source(source_id: int, req: _SourceUpdate, db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
from app.models.video import CollectionSource
|
||||
|
||||
s = await db.get(CollectionSource, source_id)
|
||||
if not s:
|
||||
raise BizException(errors.NOT_FOUND, "采集源不存在")
|
||||
d = req.model_dump(exclude_unset=True)
|
||||
if "list_url" in d and "{{page}}" not in d["list_url"]:
|
||||
raise BizException(errors.PARAM_ERROR, "URL 需包含 {{page}} 页码占位符")
|
||||
if "name" in d:
|
||||
d["category_id"] = await _resolve_category(db, d["name"].strip())
|
||||
d["name"] = d["name"].strip()
|
||||
for k, v in d.items():
|
||||
setattr(s, k, v)
|
||||
await db.commit()
|
||||
await db.refresh(s)
|
||||
return ok(_source_dict(s))
|
||||
|
||||
|
||||
@router.delete("/collection/sources/{source_id}")
|
||||
async def delete_source(source_id: int, db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_super)):
|
||||
from app.models.video import CollectionSource
|
||||
|
||||
s = await db.get(CollectionSource, source_id)
|
||||
if not s:
|
||||
raise BizException(errors.NOT_FOUND, "采集源不存在")
|
||||
await db.delete(s)
|
||||
await db.commit()
|
||||
return ok()
|
||||
|
||||
|
||||
class _SourceBatchReq(BaseModel):
|
||||
text: str = _Field(min_length=1, max_length=20000)
|
||||
|
||||
|
||||
@router.post("/collection/sources/batch")
|
||||
async def batch_create_sources(req: _SourceBatchReq, request: Request, db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
"""批量添加采集源:每行一条,格式 "分类名 URL"(URL 可带反引号),成功后自动触发采集"""
|
||||
import re
|
||||
|
||||
from app.models.admin import AdminLog
|
||||
from app.models.video import CollectionSource
|
||||
from sqlalchemy import select
|
||||
|
||||
line_re = re.compile(r"^\s*(?P<name>.+?)[\s]+[`'\"]*(?P<url>https?://[^\s`'\"]+)[`'\"]*\s*$")
|
||||
created, skipped, failed = [], [], []
|
||||
|
||||
for raw in req.text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line:
|
||||
continue
|
||||
m = line_re.match(line)
|
||||
if not m or "{{page}}" not in m.group("url"):
|
||||
failed.append({"line": line[:80], "reason": "格式错误或缺少 {{page}} 占位"})
|
||||
continue
|
||||
name, url = m.group("name").strip().strip("`'\" "), m.group("url")
|
||||
if not name or len(name) > 50:
|
||||
failed.append({"line": line[:80], "reason": "分类名为空或过长"})
|
||||
continue
|
||||
exists = (await db.execute(
|
||||
select(CollectionSource.id).where(CollectionSource.list_url == url)
|
||||
)).scalar_one_or_none()
|
||||
if exists:
|
||||
skipped.append({"name": name, "reason": "URL 已存在"})
|
||||
continue
|
||||
cid = await _resolve_category(db, name)
|
||||
s = CollectionSource(name=name, list_url=url, category_id=cid)
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
created.append({"id": s.id, "name": name})
|
||||
|
||||
db.add(AdminLog(admin_id=admin.admin_id, action="collection_source_batch",
|
||||
target="", detail=f"created={len(created)} skipped={len(skipped)} failed={len(failed)}",
|
||||
ip=_ip(request)))
|
||||
await db.commit()
|
||||
|
||||
# 新建的源自动触发后台采集
|
||||
if created:
|
||||
import asyncio
|
||||
for c in created:
|
||||
if c["id"] not in _sync_running_sources:
|
||||
_sync_running_sources.add(c["id"])
|
||||
asyncio.create_task(_run_source_sync(c["id"], 0))
|
||||
|
||||
return ok({"created": created, "skipped": skipped, "failed": failed,
|
||||
"sync_started": [c["id"] for c in created]})
|
||||
|
||||
|
||||
_sync_running_sources: set[int] = set()
|
||||
|
||||
|
||||
async def _run_source_sync(source_id: int, max_pages: int) -> None:
|
||||
from app.database import SessionFactory
|
||||
from app.models.video import CollectionSource
|
||||
from app.services import collection_service
|
||||
|
||||
try:
|
||||
async with SessionFactory() as s:
|
||||
source = await s.get(CollectionSource, source_id)
|
||||
if source:
|
||||
stat = await collection_service.sync_source(s, source, max_pages=max_pages)
|
||||
logger.info("采集源 %s 同步完成: %s", source_id, stat)
|
||||
except Exception:
|
||||
logger.exception("采集源 %s 同步失败", source_id)
|
||||
finally:
|
||||
_sync_running_sources.discard(source_id)
|
||||
|
||||
|
||||
class _SourceSyncReq(BaseModel):
|
||||
max_pages: int = _Field(0, ge=0, le=200) # 0=不限(增量会自动提前停止)
|
||||
|
||||
|
||||
@router.post("/collection/sources/{source_id}/sync")
|
||||
async def sync_one_source(source_id: int, req: _SourceSyncReq, request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
"""同步单个采集源(后台异步,立即返回)"""
|
||||
import asyncio
|
||||
|
||||
from app.models.admin import AdminLog
|
||||
from app.models.video import CollectionSource
|
||||
|
||||
s = await db.get(CollectionSource, source_id)
|
||||
if not s:
|
||||
raise BizException(errors.NOT_FOUND, "采集源不存在")
|
||||
if source_id in _sync_running_sources:
|
||||
raise BizException(errors.RATE_LIMITED, "该源正在同步中")
|
||||
|
||||
db.add(AdminLog(admin_id=admin.admin_id, action="collection_source_sync",
|
||||
target=f"source:{source_id}", detail=f"max_pages={req.max_pages}", ip=_ip(request)))
|
||||
await db.commit()
|
||||
|
||||
_sync_running_sources.add(source_id)
|
||||
asyncio.create_task(_run_source_sync(source_id, req.max_pages))
|
||||
return ok({"started": True, "source_id": source_id})
|
||||
|
||||
|
||||
@router.post("/collection/sources/sync-all")
|
||||
async def sync_all_sources(request: Request, db: AsyncSession = Depends(get_db),
|
||||
admin: AdminIdentity = Depends(require_operator)):
|
||||
"""同步全部启用的采集源(后台异步)"""
|
||||
import asyncio
|
||||
|
||||
from app.models.admin import AdminLog
|
||||
from app.models.video import CollectionSource
|
||||
from sqlalchemy import select
|
||||
|
||||
ids = (await db.execute(
|
||||
select(CollectionSource.id).where(CollectionSource.status == 1)
|
||||
)).scalars().all()
|
||||
started = [i for i in ids if i not in _sync_running_sources]
|
||||
for i in started:
|
||||
_sync_running_sources.add(i)
|
||||
asyncio.create_task(_run_source_sync(i, 0))
|
||||
|
||||
db.add(AdminLog(admin_id=admin.admin_id, action="collection_sources_sync_all",
|
||||
target="", detail=f"sources={started}", ip=_ip(request)))
|
||||
await db.commit()
|
||||
return ok({"started": started, "skipped_running": [i for i in ids if i not in started]})
|
||||
|
||||
|
||||
# ---------- 分类管理 ----------
|
||||
|
||||
from pydantic import BaseModel as _BM
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
"""媒体代理:m3u8 播放列表转发(源站 playlist 内 TS 链接带时效签名,每次回源拉取新鲜版本)"""
|
||||
import re
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Query, Response
|
||||
|
||||
from app.core.errors import NOT_FOUND, PARAM_ERROR
|
||||
from app.core.response import BizException
|
||||
from app.services.collection_service import HEADERS, UPSTREAM
|
||||
|
||||
router = APIRouter(prefix="/m3u8", tags=["媒体"])
|
||||
|
||||
# 只允许相对资源路径,防 SSRF
|
||||
_PATH_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9/._-]*$")
|
||||
|
||||
|
||||
@router.get("/proxy")
|
||||
async def m3u8_proxy(path: str = Query(min_length=1, max_length=300)):
|
||||
if ".." in path or not _PATH_RE.match(path):
|
||||
raise BizException(PARAM_ERROR, "非法路径")
|
||||
|
||||
url = f"{UPSTREAM}/api/v1/m3u8/proxy"
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
resp = await client.get(url, params={"path": path}, headers=HEADERS, timeout=20)
|
||||
|
||||
text = resp.text
|
||||
if resp.status_code != 200 or not text.lstrip().startswith("#EXTM3U"):
|
||||
raise BizException(NOT_FOUND, "播放列表获取失败")
|
||||
|
||||
return Response(
|
||||
content=text,
|
||||
media_type="application/vnd.apple.mpegurl",
|
||||
headers={"Cache-Control": "no-cache", "Access-Control-Allow-Origin": "*"},
|
||||
)
|
||||
@@ -25,7 +25,7 @@ UPLOAD_RATE_LIMIT = 5
|
||||
@router.get("/feed")
|
||||
async def get_feed(cursor: str | None = Query(None), limit: int = Query(10, ge=1, le=20),
|
||||
category_id: int | None = Query(None),
|
||||
type: str = Query("recommend", pattern="^(recommend|latest|hot)$"),
|
||||
type: str = Query("recommend", pattern="^(recommend|latest|hot|liked|viewed)$"),
|
||||
db: AsyncSession = Depends(get_db), redis: aioredis.Redis = Depends(get_redis),
|
||||
identity: Identity = Depends(get_identity)):
|
||||
return ok(await video_service.get_feed(db, redis, identity, type, cursor, limit, category_id))
|
||||
|
||||
+2
-1
@@ -69,8 +69,9 @@ from app.config import BASE_DIR
|
||||
app.mount("/static", StaticFiles(directory=BASE_DIR / "uploads"), name="static")
|
||||
|
||||
# 业务路由(按模块逐步注册)
|
||||
from app.api.v1 import admin, auth, common, incentives, interactions, invite, membership, payments, users, videos
|
||||
from app.api.v1 import admin, auth, common, incentives, interactions, invite, media, membership, payments, users, videos
|
||||
|
||||
app.include_router(media.router, prefix="/api/v1")
|
||||
app.include_router(auth.router, prefix="/api/v1")
|
||||
app.include_router(users.router, prefix="/api/v1")
|
||||
app.include_router(videos.router, prefix="/api/v1")
|
||||
|
||||
Binary file not shown.
@@ -114,3 +114,37 @@ class Series(Base):
|
||||
Index("idx_series_cat_status", "category_id", "status"),
|
||||
Index("idx_series_status_created", "status", "created_at"),
|
||||
)
|
||||
|
||||
|
||||
class CollectionRecord(Base):
|
||||
"""采集记录:外部内容源 → 本站系列的映射(幂等导入)"""
|
||||
|
||||
__tablename__ = "collection_records"
|
||||
|
||||
id: Mapped[int] = mapped_column(BIGINT(unsigned=True), primary_key=True, autoincrement=True)
|
||||
source_key: Mapped[str] = mapped_column(String(50), comment="内容源标识")
|
||||
source_id: Mapped[int] = mapped_column(BIGINT(unsigned=True), comment="源站内容 id")
|
||||
series_id: Mapped[int | None] = mapped_column(BIGINT(unsigned=True), comment="本站系列 id")
|
||||
video_id: Mapped[int | None] = mapped_column(BIGINT(unsigned=True), comment="本站视频 id(单视频类)")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
||||
|
||||
__table_args__ = (
|
||||
Index("uk_source", "source_key", "source_id", unique=True),
|
||||
Index("idx_record_series", "series_id"),
|
||||
Index("idx_record_video", "video_id"),
|
||||
)
|
||||
|
||||
|
||||
class CollectionSource(Base):
|
||||
"""采集源配置:分类名 + 列表URL模板(后台自助添加)"""
|
||||
|
||||
__tablename__ = "collection_sources"
|
||||
|
||||
id: Mapped[int] = mapped_column(BIGINT(unsigned=True), primary_key=True, autoincrement=True)
|
||||
name: Mapped[str] = mapped_column(String(50), comment="分类名")
|
||||
list_url: Mapped[str] = mapped_column(String(500), comment="列表URL模板,{{page}}为页码占位")
|
||||
category_id: Mapped[int] = mapped_column(INTEGER(unsigned=True), comment="本站分类 id")
|
||||
status: Mapped[int] = mapped_column(SmallInteger, default=1, comment="1启用 0停用")
|
||||
last_sync_at: Mapped[datetime | None] = mapped_column(DateTime)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), onupdate=func.now())
|
||||
|
||||
Binary file not shown.
@@ -37,6 +37,13 @@ class PointsAdjustReq(BaseModel):
|
||||
reason: str = Field(min_length=1, max_length=200)
|
||||
|
||||
|
||||
class UserPasswordResetReq(BaseModel):
|
||||
"""后台重置用户密码"""
|
||||
|
||||
new_password: str = Field(min_length=6, max_length=100)
|
||||
reason: str = Field(min_length=1, max_length=200)
|
||||
|
||||
|
||||
class ResettleReq(BaseModel):
|
||||
"""人工补单(11.12)"""
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -462,6 +462,26 @@ async def set_user_status(
|
||||
return {"user_id": user_id, "status": req.status, "status_text": USER_STATUS_TEXT[req.status]}
|
||||
|
||||
|
||||
async def reset_user_password(
|
||||
db: AsyncSession, redis: aioredis.Redis, admin: AdminIdentity,
|
||||
user_id: int, req, ip: str,
|
||||
) -> None:
|
||||
"""后台重置用户密码【敏感】:重置后全端 Token 立即失效"""
|
||||
user = await db.get(User, user_id)
|
||||
if user is None:
|
||||
raise BizException(errors.NOT_FOUND, "用户不存在")
|
||||
if user.status == 3:
|
||||
raise BizException(errors.PARAM_ERROR, "账号已注销,不可操作")
|
||||
|
||||
user.password_hash = hash_password(req.new_password)
|
||||
# 全端 Token 失效:递增令牌纪元 + 清 refresh 白名单
|
||||
await redis.incr(f"auth:epoch:{user_id}")
|
||||
await redis.delete(f"auth:refresh:{user_id}")
|
||||
await _log(db, admin.admin_id, "user_password_reset", target=str(user_id),
|
||||
detail=req.reason, ip=ip)
|
||||
await db.commit()
|
||||
|
||||
|
||||
# ---------- 积分 ----------
|
||||
|
||||
async def adjust_points(
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
"""短剧采集服务:从内容源拉取短剧列表/详情,导入为 series + videos(幂等)"""
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.video import CollectionRecord, CollectionSource, Series, Video
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 内容源配置(域名轮换时只需改这里)
|
||||
SOURCE_KEY = "mdcmai4"
|
||||
UPSTREAM = "https://mdcmai4.xyz"
|
||||
PRODUCT_ID = 1
|
||||
CATEGORY_ID = 12 # 本站分类:AI短剧
|
||||
AUTHOR_USER_ID = 1 # 导入内容归属用户
|
||||
SELF_API_BASE = "https://vapi.256kb.top" # 本站 API 对外地址
|
||||
|
||||
HEADERS = {"User-Agent": "Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36"}
|
||||
|
||||
|
||||
async def _fetch_json(client: httpx.AsyncClient, url: str, params: dict | None = None) -> dict:
|
||||
resp = await client.get(url, params=params, headers=HEADERS, timeout=20)
|
||||
resp.raise_for_status()
|
||||
body = resp.json()
|
||||
if body.get("code") != 200 or not isinstance(body.get("data"), dict):
|
||||
raise RuntimeError(f"上游响应异常: {url} -> {str(body)[:200]}")
|
||||
return body["data"]
|
||||
|
||||
|
||||
def _proxy_url(path: str) -> str:
|
||||
"""把源站相对 m3u8 路径转为本站代理地址(域名轮换只改 SELF_API_BASE/上游常量)"""
|
||||
return f"{SELF_API_BASE}/api/v1/m3u8/proxy?path={quote(path, safe='')}"
|
||||
|
||||
|
||||
async def _import_drama(client: httpx.AsyncClient, db: AsyncSession, drama_id: int) -> tuple[int, int]:
|
||||
"""导入单部短剧,返回 (新增集数, 跳过集数)"""
|
||||
detail = await _fetch_json(client, f"{UPSTREAM}/api/v1/short-dramas/{drama_id}", {"productId": PRODUCT_ID})
|
||||
|
||||
record = (
|
||||
await db.execute(
|
||||
select(CollectionRecord).where(
|
||||
CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == drama_id
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
published_at = None
|
||||
try:
|
||||
published_at = datetime.fromisoformat(detail.get("publishedAt", "")) or None
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
series = await db.get(Series, record.series_id) if record else None
|
||||
if series is None:
|
||||
series = Series(
|
||||
title=(detail.get("title") or f"短剧{drama_id}")[:100],
|
||||
cover_url=f"{UPSTREAM}{detail.get('coverUrl', '')}",
|
||||
description=(detail.get("description") or "")[:500],
|
||||
author_id=AUTHOR_USER_ID,
|
||||
category_id=CATEGORY_ID,
|
||||
total_episodes=0,
|
||||
view_count=int(detail.get("heatCount") or 0),
|
||||
status=1,
|
||||
)
|
||||
db.add(series)
|
||||
await db.flush()
|
||||
if record: # 系列曾被删除:重挂映射
|
||||
record.series_id = series.id
|
||||
else:
|
||||
db.add(CollectionRecord(source_key=SOURCE_KEY, source_id=drama_id, series_id=series.id))
|
||||
await db.flush()
|
||||
|
||||
# 已有集数去重
|
||||
existing = set(
|
||||
(
|
||||
await db.execute(
|
||||
select(Video.episode_no).where(Video.series_id == series.id, Video.episode_no.is_not(None))
|
||||
)
|
||||
).scalars().all()
|
||||
)
|
||||
|
||||
created = skipped = 0
|
||||
for ep in detail.get("episodes", []):
|
||||
ep_no = int(ep.get("episodeNo") or 0)
|
||||
if not ep_no or ep_no in existing:
|
||||
skipped += 1
|
||||
continue
|
||||
db.add(
|
||||
Video(
|
||||
user_id=AUTHOR_USER_ID,
|
||||
category_id=CATEGORY_ID,
|
||||
series_id=series.id,
|
||||
episode_no=ep_no,
|
||||
title=(ep.get("titleOverride") or ep.get("title") or f"第{ep_no}集")[:100],
|
||||
description=(detail.get("description") or "")[:500],
|
||||
cover_url=f"{UPSTREAM}{detail.get('coverUrl', '')}",
|
||||
video_url=_proxy_url(ep.get("videoUrl", "")),
|
||||
duration=int(ep.get("durationSec") or 0),
|
||||
status=2,
|
||||
published_at=published_at,
|
||||
)
|
||||
)
|
||||
existing.add(ep_no)
|
||||
created += 1
|
||||
|
||||
series.total_episodes = len(existing)
|
||||
return created, skipped
|
||||
|
||||
|
||||
async def sync_drama_by_id(db: AsyncSession, drama_id: int) -> dict:
|
||||
"""手动同步单部短剧(新增或补充缺失集数);返回统计"""
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
created, skipped = await _import_drama(client, db, drama_id)
|
||||
await db.commit()
|
||||
return {"drama_id": drama_id, "episodes_created": created, "episodes_skipped": skipped}
|
||||
|
||||
|
||||
async def sync_short_dramas(
|
||||
db: AsyncSession,
|
||||
max_pages: int = 0,
|
||||
page_size: int = 20,
|
||||
sort_by: str = "heat",
|
||||
delay: float = 0.25,
|
||||
) -> dict:
|
||||
"""按列表页增量同步短剧;返回统计"""
|
||||
stat = {"dramas_new": 0, "dramas_exist": 0, "episodes_created": 0, "episodes_skipped": 0, "pages": 0}
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
page = 1
|
||||
while True:
|
||||
data = await _fetch_json(
|
||||
client,
|
||||
f"{UPSTREAM}/api/v1/short-dramas",
|
||||
{"productId": PRODUCT_ID, "sortBy": sort_by, "page": page, "size": page_size},
|
||||
)
|
||||
items = data.get("items", [])
|
||||
stat["pages"] = page
|
||||
logger.info("列表第 %s/%s 页,%s 部", page, data.get("totalPages"), len(items))
|
||||
|
||||
for item in items:
|
||||
drama_id = int(item.get("id") or 0)
|
||||
if not drama_id:
|
||||
continue
|
||||
exists = (
|
||||
await db.execute(
|
||||
select(CollectionRecord.id).where(
|
||||
CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == drama_id
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists:
|
||||
stat["dramas_exist"] += 1
|
||||
continue
|
||||
try:
|
||||
created, skipped = await _import_drama(client, db, drama_id)
|
||||
await db.commit()
|
||||
stat["dramas_new"] += 1
|
||||
stat["episodes_created"] += created
|
||||
stat["episodes_skipped"] += skipped
|
||||
logger.info("导入短剧 %s《%s》:新增 %s 集", drama_id, item.get("title"), created)
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
logger.exception("导入短剧 %s 失败,跳过", drama_id)
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
total_pages = int(data.get("totalPages") or 1)
|
||||
if page >= total_pages or (max_pages and page >= max_pages):
|
||||
break
|
||||
page += 1
|
||||
await asyncio.sleep(delay)
|
||||
return stat
|
||||
|
||||
|
||||
# ---------- 单视频采集(通用 URL 模板,后台自助配置的采集源)----------
|
||||
|
||||
def _abs_cover(item: dict) -> str:
|
||||
"""封面转绝对地址:源站 /api/... 代理路径或 /uploads/... 静态路径"""
|
||||
u = item.get("coverUrl") or ""
|
||||
if u.startswith("http"):
|
||||
return u
|
||||
return f"{UPSTREAM}{u}" if u else ""
|
||||
|
||||
|
||||
async def _import_video_item(db: AsyncSession, source: CollectionSource, item: dict) -> bool:
|
||||
"""导入单条视频,已存在返回 False"""
|
||||
vid = int(item.get("id") or 0)
|
||||
if not vid or not (item.get("videoUrl") or "").endswith(".m3u8"):
|
||||
return False
|
||||
exists = (
|
||||
await db.execute(
|
||||
select(CollectionRecord.id).where(
|
||||
CollectionRecord.source_key == SOURCE_KEY, CollectionRecord.source_id == vid
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists:
|
||||
return False
|
||||
|
||||
published_at = None
|
||||
try:
|
||||
published_at = datetime.fromisoformat(item.get("publishedAt", "")) or None
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
v = Video(
|
||||
user_id=AUTHOR_USER_ID,
|
||||
category_id=source.category_id,
|
||||
title=(item.get("title") or f"视频{vid}")[:100],
|
||||
description=(item.get("description") or "")[:500],
|
||||
cover_url=_abs_cover(item)[:500],
|
||||
video_url=_proxy_url(item["videoUrl"]),
|
||||
duration=int(item.get("durationSec") or 0),
|
||||
status=2,
|
||||
view_count=int(item.get("viewCount") or 0),
|
||||
like_count=int(item.get("likeCount") or 0),
|
||||
favorite_count=int(item.get("favoriteCount") or 0),
|
||||
published_at=published_at,
|
||||
)
|
||||
db.add(v)
|
||||
await db.flush()
|
||||
db.add(CollectionRecord(source_key=SOURCE_KEY, source_id=vid, video_id=v.id))
|
||||
return True
|
||||
|
||||
|
||||
async def sync_source(
|
||||
db: AsyncSession,
|
||||
source: CollectionSource,
|
||||
max_pages: int = 0,
|
||||
stop_after_existing: int = 30,
|
||||
delay: float = 0.25,
|
||||
) -> dict:
|
||||
"""按源的 URL 模板翻页采集;连续 stop_after_existing 条已存在则提前停止(增量)"""
|
||||
stat = {"created": 0, "skipped": 0, "pages": 0, "stopped_early": False}
|
||||
consecutive_existing = 0
|
||||
async with httpx.AsyncClient(follow_redirects=True) as client:
|
||||
page = 1
|
||||
while True:
|
||||
url = source.list_url.replace("{{page}}", str(page))
|
||||
data = await _fetch_json(client, url)
|
||||
items = data.get("items", [])
|
||||
if not items:
|
||||
break
|
||||
stat["pages"] = page
|
||||
for item in items:
|
||||
if await _import_video_item(db, source, item):
|
||||
stat["created"] += 1
|
||||
consecutive_existing = 0
|
||||
else:
|
||||
stat["skipped"] += 1
|
||||
consecutive_existing += 1
|
||||
if consecutive_existing >= stop_after_existing:
|
||||
stat["stopped_early"] = True
|
||||
break
|
||||
await db.commit()
|
||||
if stat["stopped_early"]:
|
||||
break
|
||||
total_pages = int(data.get("totalPages") or page)
|
||||
if page >= total_pages or (max_pages and page >= max_pages):
|
||||
break
|
||||
page += 1
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
source.last_sync_at = datetime.now()
|
||||
await db.commit()
|
||||
return stat
|
||||
@@ -160,9 +160,37 @@ async def get_feed(
|
||||
) -> dict:
|
||||
if feed_type == "latest":
|
||||
return await _feed_latest(db, identity, cursor, limit, category_id)
|
||||
if feed_type in ("liked", "viewed"):
|
||||
return await _feed_ranked(db, identity, feed_type, cursor, limit, category_id)
|
||||
return await _feed_scored(db, redis, identity, feed_type, cursor, limit, category_id)
|
||||
|
||||
|
||||
async def _feed_ranked(
|
||||
db: AsyncSession, identity: Identity, feed_type: str, cursor: str | None,
|
||||
limit: int, category_id: int | None,
|
||||
) -> dict:
|
||||
"""liked / viewed:按点赞量 / 播放量严格倒序(值 + id 打破并列游标)"""
|
||||
col = Video.like_count if feed_type == "liked" else Video.view_count
|
||||
stmt = await _base_pool_stmt(category_id)
|
||||
c = _decode_cursor(cursor)
|
||||
if c:
|
||||
val, vid = int(c["n"]), int(c["i"])
|
||||
stmt = stmt.where((col < val) | ((col == val) & (Video.id < vid)))
|
||||
rows = list(
|
||||
(await db.execute(stmt.order_by(col.desc(), Video.id.desc()).limit(limit + 1))).scalars()
|
||||
)
|
||||
page, rows = (len(rows) > limit), rows[:limit]
|
||||
next_cursor = None
|
||||
if page and rows:
|
||||
last = rows[-1]
|
||||
last_val = last.like_count if feed_type == "liked" else last.view_count
|
||||
next_cursor = _encode_cursor({"n": last_val, "i": last.id})
|
||||
return {
|
||||
"next_cursor": next_cursor,
|
||||
"list": await serialize_videos(db, rows, identity.user_id),
|
||||
}
|
||||
|
||||
|
||||
async def _base_pool_stmt(category_id: int | None):
|
||||
stmt = select(Video).where(Video.status == STATUS_PUBLISHED, Video.published_at.isnot(None))
|
||||
if category_id:
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""一次性脚本:根据现有视频标题拆分关键词写入 tags / video_tags。
|
||||
|
||||
用法:
|
||||
cd /root/project/video/server
|
||||
.venv/bin/python scripts/build_tags_from_titles.py [--dry-run]
|
||||
|
||||
提取规则:
|
||||
1. 番号(如 CAWD-601)及其系列前缀(CAWD)
|
||||
2. 书名号/方括号内容(≤12 字,去掉「无码破解」「中文字幕」等标记词)
|
||||
3. jieba TF-IDF 关键词 topK=6(过滤停用词、纯数字、单字)
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import jieba.analyse # noqa: E402
|
||||
from sqlalchemy import select # noqa: E402
|
||||
|
||||
from app.database import SessionFactory, engine # noqa: E402
|
||||
from app.models.video import Tag, Video, VideoTag # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
STOP = set(
|
||||
"第 集 部 篇 全集 完结 中文字幕 无码 破解 无码破解 字幕 高清 最新 精选 视频 在线 观看 "
|
||||
"一个 一起 没有 为了 开始 结果 只能 直接 就是 没想到 真爽".split()
|
||||
)
|
||||
|
||||
CODE_RE = re.compile(r"[A-Za-z]{2,6}-\d{2,4}")
|
||||
EPISODE_RE = re.compile(r"第[0-9一二三四五六七八九十百千零]+[集部期话卷章]")
|
||||
PUNCT_RE = re.compile(r"[《》「」『』\[\]【】()()_\-::,,.。!!??…、/\\|~~\s]+")
|
||||
BRACKET_RE = re.compile(r"[《「『\[【((]([^《》「」『』\[\]【】()()]{2,30})[》」』\]】))]")
|
||||
|
||||
|
||||
def extract_keywords(title: str) -> list[str]:
|
||||
kws: list[str] = []
|
||||
|
||||
# 1. 番号 + 系列前缀
|
||||
for code in CODE_RE.findall(title.upper()):
|
||||
kws.append(code)
|
||||
prefix = code.split("-")[0]
|
||||
if len(prefix) >= 2:
|
||||
kws.append(prefix)
|
||||
|
||||
# 2. 书名号/括号内容(剧名/主题)
|
||||
for inner in BRACKET_RE.findall(title):
|
||||
inner = inner.strip()
|
||||
if 2 <= len(inner) <= 12 and inner not in STOP and not CODE_RE.search(inner):
|
||||
kws.append(inner)
|
||||
|
||||
# 3. jieba TF-IDF
|
||||
t = EPISODE_RE.sub(" ", title)
|
||||
t = PUNCT_RE.sub(" ", t)
|
||||
for w in jieba.analyse.extract_tags(t, topK=6):
|
||||
w = w.strip()
|
||||
if not (2 <= len(w) <= 12):
|
||||
continue
|
||||
if w in STOP or re.fullmatch(r"\d+", w):
|
||||
continue
|
||||
kws.append(w)
|
||||
|
||||
# 去重保序(按 casefold 键,避免 MySQL 不区分大小写的唯一键冲突)
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for k in kws:
|
||||
if k.isascii():
|
||||
k = k.upper()
|
||||
key = k.casefold()
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
out.append(k)
|
||||
return out
|
||||
|
||||
|
||||
async def main(dry_run: bool) -> None:
|
||||
async with SessionFactory() as db:
|
||||
videos = (await db.execute(select(Video.id, Video.title))).all()
|
||||
log.info("共 %d 个视频", len(videos))
|
||||
|
||||
tag_map: dict[str, int] = {} # name -> tag_id(含新建暂存)
|
||||
video_count: dict[str, int] = {} # name -> 关联视频数
|
||||
links: list[tuple[int, str]] = [] # (video_id, tag_name)
|
||||
|
||||
for vid, title in videos:
|
||||
if not title:
|
||||
continue
|
||||
kws = extract_keywords(title)
|
||||
for k in kws:
|
||||
links.append((vid, k))
|
||||
video_count[k] = video_count.get(k, 0) + 1
|
||||
|
||||
log.info("提取出 %d 个不同标签,%d 条视频-标签关联", len(video_count), len(links))
|
||||
top = sorted(video_count.items(), key=lambda x: -x[1])[:15]
|
||||
log.info("Top15 标签: %s", top)
|
||||
|
||||
if dry_run:
|
||||
await engine.dispose()
|
||||
return
|
||||
|
||||
# 预载已存在的 tags(运行中的后端也可能并发写 tags,全程幂等)
|
||||
existing = (await db.execute(select(Tag.name, Tag.id))).all()
|
||||
tag_map: dict[str, int] = {n.casefold(): i for n, i in existing}
|
||||
|
||||
# 写入 tags:INSERT IGNORE 防并发/重复,再取回 id
|
||||
from sqlalchemy import text
|
||||
new_names = [n[:20] for n in video_count if n[:20].casefold() not in tag_map]
|
||||
for i in range(0, len(new_names), 500):
|
||||
chunk = new_names[i:i + 500]
|
||||
await db.execute(
|
||||
text("INSERT IGNORE INTO tags (name, video_count) VALUES " +
|
||||
",".join("(:n%d, 0)" % j for j in range(len(chunk)))),
|
||||
{f"n{j}": v for j, v in enumerate(chunk)},
|
||||
)
|
||||
await db.flush()
|
||||
|
||||
existing = (await db.execute(select(Tag.name, Tag.id))).all()
|
||||
tag_map = {n.casefold(): i for n, i in existing}
|
||||
|
||||
# 写入 video_tags:INSERT IGNORE(uk_video_tag 自动去重)
|
||||
seen_link: set[tuple[int, int]] = set()
|
||||
rows: list[dict] = []
|
||||
for vid, name in links:
|
||||
tid = tag_map.get(name[:20].casefold())
|
||||
if tid is None or (vid, tid) in seen_link:
|
||||
continue
|
||||
seen_link.add((vid, tid))
|
||||
rows.append({"v": vid, "t": tid})
|
||||
for i in range(0, len(rows), 1000):
|
||||
chunk = rows[i:i + 1000]
|
||||
await db.execute(
|
||||
text("INSERT IGNORE INTO video_tags (video_id, tag_id) VALUES " +
|
||||
",".join("(:v%d, :t%d)" % (j, j) for j in range(len(chunk)))),
|
||||
{**{f"v{j}": r["v"] for j, r in enumerate(chunk)},
|
||||
**{f"t{j}": r["t"] for j, r in enumerate(chunk)}},
|
||||
)
|
||||
|
||||
# 统一回算冗余计数
|
||||
await db.execute(text(
|
||||
"UPDATE tags t SET t.video_count = "
|
||||
"(SELECT COUNT(*) FROM video_tags vt WHERE vt.tag_id = t.id)"
|
||||
))
|
||||
await db.commit()
|
||||
log.info("写入完成:tags=%d(新增 %d),video_tags=%d", len(tag_map), len(new_names), len(rows))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--dry-run", action="store_true", help="只打印统计不写库")
|
||||
args = parser.parse_args()
|
||||
asyncio.run(main(args.dry_run))
|
||||
@@ -0,0 +1,41 @@
|
||||
"""短剧采集 CLI
|
||||
用法:
|
||||
.venv/bin/python scripts/collect_short_dramas.py # 全量增量同步
|
||||
.venv/bin/python scripts/collect_short_dramas.py --pages 1 --size 3 # 试跑
|
||||
.venv/bin/python scripts/collect_short_dramas.py --drama-id 130 # 手动更新单部
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.database import SessionFactory # noqa: E402
|
||||
from app.services.collection_service import sync_drama_by_id, sync_short_dramas # noqa: E402
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="短剧采集")
|
||||
parser.add_argument("--drama-id", type=int, default=0, help="只同步指定源站短剧 id")
|
||||
parser.add_argument("--pages", type=int, default=0, help="最多同步的列表页数,0=全部")
|
||||
parser.add_argument("--size", type=int, default=20, help="每页条数")
|
||||
parser.add_argument("--sort", choices=["heat", "latest"], default="heat")
|
||||
parser.add_argument("--delay", type=float, default=0.25, help="上游请求间隔秒")
|
||||
args = parser.parse_args()
|
||||
|
||||
async with SessionFactory() as db:
|
||||
if args.drama_id:
|
||||
stat = await sync_drama_by_id(db, args.drama_id)
|
||||
else:
|
||||
stat = await sync_short_dramas(
|
||||
db, max_pages=args.pages, page_size=args.size, sort_by=args.sort, delay=args.delay
|
||||
)
|
||||
print("同步完成:", stat)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user