This commit is contained in:
lz_db
2026-10-25 16:14:26 +08:00
parent 7fae7387f5
commit 1d9cd1f9c7
10 changed files with 288 additions and 13 deletions
+44 -3
View File
@@ -1143,6 +1143,9 @@ def _source_dict(s) -> dict:
"has_token": bool(cfg.get("token")),
"last_sync_at": s.last_sync_at.strftime("%Y-%m-%d %H:%M:%S") if s.last_sync_at else "",
"created_at": s.created_at.strftime("%Y-%m-%d %H:%M:%S"),
# 同步运行状态以列表接口合并的 Redis 进度为准(多 worker 下进程内存不可靠)
"syncing": False,
"sync_progress": None,
}
@@ -1161,14 +1164,32 @@ async def _resolve_category(db: AsyncSession, name: str) -> int:
return c.id
# 同步进度 Redis key(多 worker 共享,进程内存不可靠)
_SYNC_PROGRESS_KEY = "video:admin:sync_progress:{}"
@router.get("/collection/sources")
async def list_sources(db: AsyncSession = Depends(get_db),
redis: aioredis.Redis = Depends(get_redis),
admin: AdminIdentity = Depends(require_operator)):
import json as _json
from app.models.video import CollectionSource
from sqlalchemy import select
rows = (await db.execute(select(CollectionSource).order_by(CollectionSource.id.desc()))).scalars().all()
return ok({"list": [_source_dict(s) for s in rows]})
items = [_source_dict(s) for s in rows]
# 从 Redis 合并实时同步进度(同步任务可能跑在另一个 worker 进程上)
if rows:
vals = await redis.mget([_SYNC_PROGRESS_KEY.format(s.id) for s in rows])
for item, raw in zip(items, vals):
if raw:
try:
item["syncing"] = True
item["sync_progress"] = _json.loads(raw)
except Exception:
pass
return ok({"list": items})
@router.post("/collection/sources")
@@ -1317,10 +1338,27 @@ _sync_cancelled_sources: set[int] = set()
async def _run_source_sync(source_id: int, max_pages: int) -> None:
import asyncio
import json as _json
from datetime import datetime
from app.database import SessionFactory
from app.models.video import CollectionSource
from app.redis_client import redis_client
from app.services import aidrama91_service, collection_service
# 进度写 Redis(多 worker 共享,列表接口可能落在另一个进程上查询)
key = _SYNC_PROGRESS_KEY.format(source_id)
progress: dict = {
"started_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "pages": 0,
}
await redis_client.set(key, _json.dumps(progress), ex=6 * 3600) # TTL 兜底:进程被杀残留 key 自动过期
def _progress(stat: dict) -> None:
progress.update(stat)
# fire-and-forget:回调在采集循环内同步触发,不阻塞采集
asyncio.create_task(redis_client.set(key, _json.dumps(progress), ex=6 * 3600))
try:
async with SessionFactory() as s:
source = await s.get(CollectionSource, source_id)
@@ -1332,11 +1370,13 @@ async def _run_source_sync(source_id: int, max_pages: int) -> None:
if source.kind == aidrama91_service.KIND:
await aidrama91_service.refresh_source_id(s, source)
stat = await aidrama91_service.sync_source(
s, source, max_pages=max_pages, should_cancel=_cancelled)
s, source, max_pages=max_pages, should_cancel=_cancelled,
on_progress=_progress)
elif source.kind == aidrama91_service.VLOG_KIND:
await aidrama91_service.refresh_source_id(s, source)
stat = await aidrama91_service.sync_vlog_source(
s, source, max_pages=max_pages, should_cancel=_cancelled)
s, source, max_pages=max_pages, should_cancel=_cancelled,
on_progress=_progress)
else:
stat = await collection_service.sync_source(
s, source, max_pages=max_pages, should_cancel=_cancelled)
@@ -1349,6 +1389,7 @@ async def _run_source_sync(source_id: int, max_pages: int) -> None:
finally:
_sync_running_sources.discard(source_id)
_sync_cancelled_sources.discard(source_id)
await redis_client.delete(key)
class _SourceSyncReq(BaseModel):