This commit is contained in:
lz_db
2026-10-25 16:14:26 +08:00
parent 7fae7387f5
commit 1d9cd1f9c7
10 changed files with 288 additions and 13 deletions
+9 -3
View File
@@ -477,11 +477,12 @@ async def import_drama(
async def sync_source(
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
should_cancel=None,
should_cancel=None, on_progress=None,
) -> dict:
"""按列表翻页增量采集(已导入的短剧跳过,仍会补同步新集数)
should_cancel: 可选回调,返回 True 时在每页/每条边界退出循环(用于手动取消)
on_progress: 可选回调,每页拉到列表后调用 on_progress(dict(stat))(用于后台展示进度)
"""
cfg = load_config(source)
# 每条/每页之间的间隔可由采集源 config 的 delay 自定义(秒),未配置时用默认值
@@ -515,6 +516,8 @@ async def sync_source(
stat["total"] = data["total"]
items = data["list"]
logger.info("91 短剧第 %s 页:%s 部(共 %s)", page, len(items), data["total"])
if on_progress:
on_progress(dict(stat))
for drama in items:
if _cancelled():
@@ -747,9 +750,10 @@ async def import_vlog_video(
async def sync_vlog_source(
db: AsyncSession, source: CollectionSource, max_pages: int = 0, delay: float = 0.3,
should_cancel=None,
should_cancel=None, on_progress=None,
) -> dict:
"""vlog 短视频翻页增量采集。should_cancel 返回 True 时每页/每条边界退出"""
"""vlog 短视频翻页增量采集。should_cancel 返回 True 时每页/每条边界退出;
on_progress 在每页拉到列表后回调 dict(stat)(用于后台展示进度)"""
cfg = load_config(source)
# 每条/每页之间的间隔可由采集源 config 的 delay 自定义(秒),未配置时用默认值
if cfg.get("delay") is not None:
@@ -781,6 +785,8 @@ async def sync_vlog_source(
stat["pages"] = page
items = data["list"]
logger.info("91 vlog 第 %s 页:%s 条", page, len(items))
if on_progress:
on_progress(dict(stat))
for item in items:
if _cancelled():