1

2026-10-17 19:04:05 +08:00
parent 28b11b4365
commit 1253c159f5
43 changed files with 2652 additions and 114 deletions
@@ -0,0 +1,157 @@
+"""一次性脚本：根据现有视频标题拆分关键词写入 tags / video_tags。
+
+用法：
+    cd /root/project/video/server
+    .venv/bin/python scripts/build_tags_from_titles.py [--dry-run]
+
+提取规则：
+    1. 番号（如 CAWD-601）及其系列前缀（CAWD）
+    2. 书名号/方括号内容（≤12 字，去掉「无码破解」「中文字幕」等标记词）
+    3. jieba TF-IDF 关键词 topK=6（过滤停用词、纯数字、单字）
+"""
+import argparse
+import asyncio
+import logging
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+import jieba.analyse  # noqa: E402
+from sqlalchemy import select  # noqa: E402
+
+from app.database import SessionFactory, engine  # noqa: E402
+from app.models.video import Tag, Video, VideoTag  # noqa: E402
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
+log = logging.getLogger(__name__)
+
+STOP = set(
+    "第 集 部 篇 全集 完结 中文字幕 无码 破解 无码破解 字幕 高清 最新 精选 视频 在线 观看 "
+    "一个 一起 没有 为了 开始 结果 只能 直接 就是 没想到 真爽".split()
+)
+
+CODE_RE = re.compile(r"[A-Za-z]{2,6}-\d{2,4}")
+EPISODE_RE = re.compile(r"第[0-9一二三四五六七八九十百千零]+[集部期话卷章]")
+PUNCT_RE = re.compile(r"[《》「」『』\[\]【】()（）_\-:：,，.。!！?？…、/\\|~～\s]+")
+BRACKET_RE = re.compile(r"[《「『\[【(（]([^《》「」『』\[\]【】()（）]{2,30})[》」』\]】)）]")
+
+
+def extract_keywords(title: str) -> list[str]:
+    kws: list[str] = []
+
+    # 1. 番号 + 系列前缀
+    for code in CODE_RE.findall(title.upper()):
+        kws.append(code)
+        prefix = code.split("-")[0]
+        if len(prefix) >= 2:
+            kws.append(prefix)
+
+    # 2. 书名号/括号内容（剧名/主题）
+    for inner in BRACKET_RE.findall(title):
+        inner = inner.strip()
+        if 2 <= len(inner) <= 12 and inner not in STOP and not CODE_RE.search(inner):
+            kws.append(inner)
+
+    # 3. jieba TF-IDF
+    t = EPISODE_RE.sub(" ", title)
+    t = PUNCT_RE.sub(" ", t)
+    for w in jieba.analyse.extract_tags(t, topK=6):
+        w = w.strip()
+        if not (2 <= len(w) <= 12):
+            continue
+        if w in STOP or re.fullmatch(r"\d+", w):
+            continue
+        kws.append(w)
+
+    # 去重保序（按 casefold 键，避免 MySQL 不区分大小写的唯一键冲突）
+    seen: set[str] = set()
+    out: list[str] = []
+    for k in kws:
+        if k.isascii():
+            k = k.upper()
+        key = k.casefold()
+        if key not in seen:
+            seen.add(key)
+            out.append(k)
+    return out
+
+
+async def main(dry_run: bool) -> None:
+    async with SessionFactory() as db:
+        videos = (await db.execute(select(Video.id, Video.title))).all()
+        log.info("共 %d 个视频", len(videos))
+
+        tag_map: dict[str, int] = {}  # name -> tag_id（含新建暂存）
+        video_count: dict[str, int] = {}  # name -> 关联视频数
+        links: list[tuple[int, str]] = []  # (video_id, tag_name)
+
+        for vid, title in videos:
+            if not title:
+                continue
+            kws = extract_keywords(title)
+            for k in kws:
+                links.append((vid, k))
+                video_count[k] = video_count.get(k, 0) + 1
+
+        log.info("提取出 %d 个不同标签，%d 条视频-标签关联", len(video_count), len(links))
+        top = sorted(video_count.items(), key=lambda x: -x[1])[:15]
+        log.info("Top15 标签: %s", top)
+
+        if dry_run:
+            await engine.dispose()
+            return
+
+        # 预载已存在的 tags（运行中的后端也可能并发写 tags，全程幂等）
+        existing = (await db.execute(select(Tag.name, Tag.id))).all()
+        tag_map: dict[str, int] = {n.casefold(): i for n, i in existing}
+
+        # 写入 tags：INSERT IGNORE 防并发/重复，再取回 id
+        from sqlalchemy import text
+        new_names = [n[:20] for n in video_count if n[:20].casefold() not in tag_map]
+        for i in range(0, len(new_names), 500):
+            chunk = new_names[i:i + 500]
+            await db.execute(
+                text("INSERT IGNORE INTO tags (name, video_count) VALUES " +
+                     ",".join("(:n%d, 0)" % j for j in range(len(chunk)))),
+                {f"n{j}": v for j, v in enumerate(chunk)},
+            )
+        await db.flush()
+
+        existing = (await db.execute(select(Tag.name, Tag.id))).all()
+        tag_map = {n.casefold(): i for n, i in existing}
+
+        # 写入 video_tags：INSERT IGNORE（uk_video_tag 自动去重）
+        seen_link: set[tuple[int, int]] = set()
+        rows: list[dict] = []
+        for vid, name in links:
+            tid = tag_map.get(name[:20].casefold())
+            if tid is None or (vid, tid) in seen_link:
+                continue
+            seen_link.add((vid, tid))
+            rows.append({"v": vid, "t": tid})
+        for i in range(0, len(rows), 1000):
+            chunk = rows[i:i + 1000]
+            await db.execute(
+                text("INSERT IGNORE INTO video_tags (video_id, tag_id) VALUES " +
+                     ",".join("(:v%d, :t%d)" % (j, j) for j in range(len(chunk)))),
+                {**{f"v{j}": r["v"] for j, r in enumerate(chunk)},
+                 **{f"t{j}": r["t"] for j, r in enumerate(chunk)}},
+            )
+
+        # 统一回算冗余计数
+        await db.execute(text(
+            "UPDATE tags t SET t.video_count = "
+            "(SELECT COUNT(*) FROM video_tags vt WHERE vt.tag_id = t.id)"
+        ))
+        await db.commit()
+        log.info("写入完成：tags=%d（新增 %d），video_tags=%d", len(tag_map), len(new_names), len(rows))
+        await engine.dispose()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dry-run", action="store_true", help="只打印统计不写库")
+    args = parser.parse_args()
+    asyncio.run(main(args.dry_run))
@@ -0,0 +1,41 @@
+"""短剧采集 CLI
+用法:
+  .venv/bin/python scripts/collect_short_dramas.py                # 全量增量同步
+  .venv/bin/python scripts/collect_short_dramas.py --pages 1 --size 3   # 试跑
+  .venv/bin/python scripts/collect_short_dramas.py --drama-id 130       # 手动更新单部
+"""
+import argparse
+import asyncio
+import logging
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from app.database import SessionFactory  # noqa: E402
+from app.services.collection_service import sync_drama_by_id, sync_short_dramas  # noqa: E402
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
+
+
+async def main() -> None:
+    parser = argparse.ArgumentParser(description="短剧采集")
+    parser.add_argument("--drama-id", type=int, default=0, help="只同步指定源站短剧 id")
+    parser.add_argument("--pages", type=int, default=0, help="最多同步的列表页数，0=全部")
+    parser.add_argument("--size", type=int, default=20, help="每页条数")
+    parser.add_argument("--sort", choices=["heat", "latest"], default="heat")
+    parser.add_argument("--delay", type=float, default=0.25, help="上游请求间隔秒")
+    args = parser.parse_args()
+
+    async with SessionFactory() as db:
+        if args.drama_id:
+            stat = await sync_drama_by_id(db, args.drama_id)
+        else:
+            stat = await sync_short_dramas(
+                db, max_pages=args.pages, page_size=args.size, sort_by=args.sort, delay=args.delay
+            )
+    print("同步完成:", stat)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())