299 lines
13 KiB
JavaScript
299 lines
13 KiB
JavaScript
|
// dsh-plugin-mcn 本地 SQLite 缓存（node:sqlite，Node 24 内置；拆分自 index.js）
|
|||
|
|
import { DatabaseSync } from "node:sqlite";
|
|||
|
|
import { join } from "node:path";
|
|||
|
|
import { readFileSync, readdirSync, existsSync } from "node:fs";
|
|||
|
|
import { DB_PATH } from "./config.js";
|
|||
|
|
import { normTitle, titleDistance, findVideoByFolder, resolveAccountDirs } from "./video-tools.js";
|
|||
|
|
|
|||
|
|
let db;
|
|||
|
|
|
|||
|
|
/** 从 AI 写作脚本正文提取标题（兼容现有多种首行格式）：
|
|||
|
|
* - 去掉 markdown 标题标记（# 前缀）
|
|||
|
|
* - 去掉「S9 故事脚本」等前缀
|
|||
|
|
* - 优先取《…》内内容，否则用整行
|
|||
|
|
* - 限制长度 80 字符，空则返回 null
|
|||
|
|
*/
|
|||
|
|
function extractScriptTitle(scriptText) {
|
|||
|
|
if (typeof scriptText !== "string" || !scriptText.trim()) return null;
|
|||
|
|
let line = scriptText.replace(/\r\n/g, "\n").split("\n").find((l) => l.trim()) || "";
|
|||
|
|
let t = line.trim();
|
|||
|
|
t = t.replace(/^#+\s*/, "").trim(); // 去 markdown 标题标记
|
|||
|
|
t = t.replace(/^S\d+\s*(故事)?(脚|剧)?本\s*/, "").trim(); // 去「S9 故事脚本」等前缀
|
|||
|
|
const m = t.match(/《([^》]+)》/);
|
|||
|
|
if (m && m[1].trim()) t = m[1].trim();
|
|||
|
|
t = t.replace(/^#+\s*/, "").trim();
|
|||
|
|
return t ? t.slice(0, 80) : null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function initDb() {
|
|||
|
|
if (db) return db;
|
|||
|
|
db = new DatabaseSync(DB_PATH);
|
|||
|
|
db.exec(`
|
|||
|
|
CREATE TABLE IF NOT EXISTS hot_accounts (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
account_name TEXT NOT NULL,
|
|||
|
|
content TEXT,
|
|||
|
|
track TEXT,
|
|||
|
|
tags TEXT,
|
|||
|
|
masterpiece TEXT,
|
|||
|
|
created_by TEXT,
|
|||
|
|
updated_by TEXT,
|
|||
|
|
created_time TEXT,
|
|||
|
|
updated_time TEXT,
|
|||
|
|
sort INTEGER DEFAULT 0,
|
|||
|
|
del_flag INTEGER DEFAULT 0,
|
|||
|
|
account_type TEXT DEFAULT 'signed',
|
|||
|
|
favorite INTEGER DEFAULT 0
|
|||
|
|
);
|
|||
|
|
`);
|
|||
|
|
// 存量数据库迁移：旧表可能缺 account_type 列（CREATE TABLE IF NOT EXISTS 不会为已存在表加列）
|
|||
|
|
const cols = db.prepare(`PRAGMA table_info(hot_accounts)`).all().map((c) => c.name);
|
|||
|
|
if (!cols.includes("account_type")) {
|
|||
|
|
db.exec(`ALTER TABLE hot_accounts ADD COLUMN account_type TEXT DEFAULT 'signed'`);
|
|||
|
|
}
|
|||
|
|
if (!cols.includes("favorite")) {
|
|||
|
|
db.exec(`ALTER TABLE hot_accounts ADD COLUMN favorite INTEGER DEFAULT 0`);
|
|||
|
|
}
|
|||
|
|
// 外部账号扩展列（幂等迁移）
|
|||
|
|
const extCols = {
|
|||
|
|
douyin_id: "TEXT",
|
|||
|
|
sec_uid: "TEXT",
|
|||
|
|
followers: "TEXT",
|
|||
|
|
total_likes: "TEXT",
|
|||
|
|
works_count: "INTEGER",
|
|||
|
|
ip_location: "TEXT",
|
|||
|
|
location: "TEXT",
|
|||
|
|
age: "INTEGER",
|
|||
|
|
bio: "TEXT",
|
|||
|
|
source_dir: "TEXT",
|
|||
|
|
};
|
|||
|
|
for (const [name, type] of Object.entries(extCols)) {
|
|||
|
|
if (!cols.includes(name)) {
|
|||
|
|
db.exec(`ALTER TABLE hot_accounts ADD COLUMN ${name} ${type}`);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
db.exec(`CREATE INDEX IF NOT EXISTS idx_hot_accounts_name ON hot_accounts(account_name);`);
|
|||
|
|
db.exec(`CREATE INDEX IF NOT EXISTS idx_hot_accounts_track ON hot_accounts(track);`);
|
|||
|
|
db.exec(`CREATE INDEX IF NOT EXISTS idx_hot_accounts_type ON hot_accounts(account_type);`);
|
|||
|
|
// 存量数据兼容：无 account_type 的按 signed 处理
|
|||
|
|
try { db.exec(`UPDATE hot_accounts SET account_type='signed' WHERE account_type IS NULL OR account_type=''`); } catch (e) {}
|
|||
|
|
// 外部账号关联表（见 Architecture/外部账号数据结构设计.md）
|
|||
|
|
db.exec(`
|
|||
|
|
CREATE TABLE IF NOT EXISTS account_videos (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
account_id INTEGER,
|
|||
|
|
aweme_id TEXT,
|
|||
|
|
video_title TEXT,
|
|||
|
|
video_url TEXT,
|
|||
|
|
like_count INTEGER DEFAULT 0,
|
|||
|
|
like_display TEXT,
|
|||
|
|
comment_count INTEGER DEFAULT 0,
|
|||
|
|
share_count INTEGER DEFAULT 0,
|
|||
|
|
collect_count INTEGER DEFAULT 0,
|
|||
|
|
play_count INTEGER DEFAULT 0,
|
|||
|
|
duration TEXT,
|
|||
|
|
publish_time TEXT,
|
|||
|
|
tags TEXT,
|
|||
|
|
collected_time TEXT,
|
|||
|
|
source_dir TEXT,
|
|||
|
|
topic TEXT,
|
|||
|
|
UNIQUE(account_id, aweme_id)
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS account_video_analysis (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
video_id INTEGER,
|
|||
|
|
aweme_id TEXT,
|
|||
|
|
analysis_time TEXT,
|
|||
|
|
content_json TEXT,
|
|||
|
|
summary TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS account_video_source (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
video_id INTEGER,
|
|||
|
|
analysis_time TEXT,
|
|||
|
|
content_json TEXT,
|
|||
|
|
summary TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS account_persona (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
account_id INTEGER,
|
|||
|
|
analysis_time TEXT,
|
|||
|
|
content_json TEXT,
|
|||
|
|
summary TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS account_analysis (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
account_id INTEGER,
|
|||
|
|
analysis_time TEXT,
|
|||
|
|
content_json TEXT,
|
|||
|
|
summary TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS creative_log (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
created_time TEXT,
|
|||
|
|
source TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS rewrite_log (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
video_id INTEGER,
|
|||
|
|
aweme_id TEXT,
|
|||
|
|
account_id INTEGER,
|
|||
|
|
script_text TEXT,
|
|||
|
|
topic TEXT,
|
|||
|
|
status TEXT DEFAULT 'pending',
|
|||
|
|
created_time TEXT,
|
|||
|
|
updated_time TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS storyboard_log (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
video_id INTEGER,
|
|||
|
|
rewrite_id INTEGER,
|
|||
|
|
script_text TEXT,
|
|||
|
|
status TEXT DEFAULT 'pending',
|
|||
|
|
created_time TEXT,
|
|||
|
|
updated_time TEXT
|
|||
|
|
);
|
|||
|
|
CREATE TABLE IF NOT EXISTS script_review (
|
|||
|
|
id INTEGER PRIMARY KEY,
|
|||
|
|
type TEXT NOT NULL,
|
|||
|
|
rewrite_id INTEGER,
|
|||
|
|
video_id INTEGER,
|
|||
|
|
dims_json TEXT,
|
|||
|
|
total TEXT,
|
|||
|
|
verdict TEXT,
|
|||
|
|
issues_json TEXT,
|
|||
|
|
content TEXT,
|
|||
|
|
summary TEXT,
|
|||
|
|
status TEXT DEFAULT 'pending',
|
|||
|
|
created_time TEXT,
|
|||
|
|
updated_time TEXT
|
|||
|
|
);
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_script_review_type ON script_review(type);
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_script_review_rewrite ON script_review(rewrite_id);
|
|||
|
|
CREATE INDEX IF NOT EXISTS idx_script_review_video ON script_review(video_id);
|
|||
|
|
`);
|
|||
|
|
// 存量迁移：老库 account_videos 缺 topic 列（选题，从源视频解析 json 提取）
|
|||
|
|
try {
|
|||
|
|
const tcols = db.prepare(`PRAGMA table_info(account_videos)`).all().map((c) => c.name);
|
|||
|
|
if (!tcols.includes("topic")) db.exec(`ALTER TABLE account_videos ADD COLUMN topic TEXT`);
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量迁移：rewrite_log 缺 title 列（AI 写作脚本标题，从脚本首行提取）
|
|||
|
|
try {
|
|||
|
|
const rwcols = db.prepare(`PRAGMA table_info(rewrite_log)`).all().map((c) => c.name);
|
|||
|
|
if (!rwcols.includes("title")) db.exec(`ALTER TABLE rewrite_log ADD COLUMN title TEXT`);
|
|||
|
|
// 存量回填：仅 title 为空且脚本非空时，从脚本首行提取（幂等）
|
|||
|
|
for (const row of db.prepare(`SELECT id, script_text FROM rewrite_log WHERE (title IS NULL OR title = '') AND script_text IS NOT NULL AND script_text <> ''`).all()) {
|
|||
|
|
const title = extractScriptTitle(row.script_text);
|
|||
|
|
if (title) db.prepare(`UPDATE rewrite_log SET title=? WHERE id=?`).run(title, row.id);
|
|||
|
|
}
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量迁移：account_video_analysis 缺 aweme_id 列（拆解分析关联视频ID，业务主键）
|
|||
|
|
try {
|
|||
|
|
const acols = db.prepare(`PRAGMA table_info(account_video_analysis)`).all().map((c) => c.name);
|
|||
|
|
if (!acols.includes("aweme_id")) db.exec(`ALTER TABLE account_video_analysis ADD COLUMN aweme_id TEXT`);
|
|||
|
|
db.exec(`UPDATE account_video_analysis SET aweme_id = (SELECT aweme_id FROM account_videos WHERE id = account_video_analysis.video_id) WHERE aweme_id IS NULL AND video_id IS NOT NULL`);
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量迁移：account_video_source 缺 aweme_id 列（原视频解析关联视频ID，业务主键）
|
|||
|
|
try {
|
|||
|
|
const scols = db.prepare(`PRAGMA table_info(account_video_source)`).all().map((c) => c.name);
|
|||
|
|
if (!scols.includes("aweme_id")) db.exec(`ALTER TABLE account_video_source ADD COLUMN aweme_id TEXT`);
|
|||
|
|
if (!scols.includes("analysis_json")) db.exec(`ALTER TABLE account_video_source ADD COLUMN analysis_json TEXT`);
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量迁移：hot_accounts 缺 is_ai 列（是否AI账号，默认「否」，赛道字段后展示）
|
|||
|
|
try {
|
|||
|
|
const hcols = db.prepare(`PRAGMA table_info(hot_accounts)`).all().map((c) => c.name);
|
|||
|
|
if (!hcols.includes("is_ai")) db.exec(`ALTER TABLE hot_accounts ADD COLUMN is_ai TEXT DEFAULT '否'`);
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量迁移：hot_accounts 缺 del_flag 列（软删标记，同步 V1.0 工作台：批量删除账号=软删+连坐清关联）
|
|||
|
|
try {
|
|||
|
|
const hdcols = db.prepare(`PRAGMA table_info(hot_accounts)`).all().map((c) => c.name);
|
|||
|
|
if (!hdcols.includes("del_flag")) db.exec(`ALTER TABLE hot_accounts ADD COLUMN del_flag INTEGER DEFAULT 0`);
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量迁移：rewrite_log 缺 gen_method 列（生成方式：ref=参考选题创作 / new=原创选题创作 / abc=ABC融合创作；
|
|||
|
|
// 同步 V1.0 工作台 v25 三 tab 口径；历史行按 video_id 是否为空回填）
|
|||
|
|
try {
|
|||
|
|
const gwcols = db.prepare(`PRAGMA table_info(rewrite_log)`).all().map((c) => c.name);
|
|||
|
|
if (!gwcols.includes("gen_method")) db.exec(`ALTER TABLE rewrite_log ADD COLUMN gen_method TEXT DEFAULT 'ref'`);
|
|||
|
|
db.exec(`UPDATE rewrite_log SET gen_method='new' WHERE (video_id IS NULL OR video_id = 0) AND (gen_method IS NULL OR gen_method = '')`);
|
|||
|
|
db.exec(`UPDATE rewrite_log SET gen_method='ref' WHERE (video_id IS NOT NULL AND video_id > 0) AND (gen_method IS NULL OR gen_method = '')`);
|
|||
|
|
} catch (e) { /* 迁移失败不阻塞 */ }
|
|||
|
|
// 存量清洗：赛道值去掉「赛道」后缀（如「生活vlog赛道」→「生活vlog」，全库统一；幂等）
|
|||
|
|
try {
|
|||
|
|
db.exec(`UPDATE hot_accounts SET track = REPLACE(track, '赛道', '') WHERE track LIKE '%赛道%'`);
|
|||
|
|
} catch (e) { /* 清洗失败不阻塞 */ }
|
|||
|
|
// 存量修复：source 按 content_json 标题匹配当前 account_videos，回填 aweme_id 并修正错位 video_id
|
|||
|
|
// （早期视频重建后自增 id 失效，旧记录指向已删除视频；aweme_id 为业务主键不受重建影响）
|
|||
|
|
try { backfillSourceAwemeIds(db); } catch (e) { console.warn(`[dsh-plugin-mcn] source 回填失败: ${e.message}`); }
|
|||
|
|
return db;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function backfillSourceAwemeIds(d) {
|
|||
|
|
// 补 analysis_json（每次启动执行，不依赖 aweme_id 回填）：文件系统里存在 analysis.json 的记录（按账号内标题匹配）
|
|||
|
|
try {
|
|||
|
|
for (const acc of d.prepare(`SELECT id, account_name, source_dir FROM hot_accounts`).all()) {
|
|||
|
|
for (const dir of resolveAccountDirs(acc)) {
|
|||
|
|
const vsDir = join(dir, "视频对标");
|
|||
|
|
if (!existsSync(vsDir)) continue;
|
|||
|
|
for (const e of readdirSync(vsDir, { withFileTypes: true })) {
|
|||
|
|
if (!e.isDirectory()) continue;
|
|||
|
|
const vDir = join(vsDir, e.name);
|
|||
|
|
const ap = join(vDir, "analysis.json");
|
|||
|
|
if (!existsSync(ap)) continue;
|
|||
|
|
const vd = findVideoByFolder(d, acc.id, vDir, e.name);
|
|||
|
|
if (!vd) continue;
|
|||
|
|
const row = d.prepare(`SELECT id FROM account_video_source WHERE aweme_id=? OR video_id=?`).get(vd.aweme_id, vd.id);
|
|||
|
|
if (row) {
|
|||
|
|
const aj = readFileSync(ap, "utf8");
|
|||
|
|
d.prepare(`UPDATE account_video_source SET analysis_json=? WHERE id=?`).run(aj, row.id);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
} catch (e) { console.warn(`[dsh-plugin-mcn] analysis_json 补录失败: ${e.message}`); }
|
|||
|
|
const rows = d.prepare(`SELECT id, video_id, aweme_id, content_json FROM account_video_source`).all();
|
|||
|
|
const missing = rows.filter((r) => !r.aweme_id);
|
|||
|
|
if (missing.length === 0) return 0;
|
|||
|
|
const all = d.prepare(`SELECT id, aweme_id, video_title FROM account_videos`).all();
|
|||
|
|
const find = (title) => {
|
|||
|
|
const t = String(title || "").trim();
|
|||
|
|
if (!t) return null;
|
|||
|
|
const exact = all.find((v) => String(v.video_title || "") === t);
|
|||
|
|
if (exact) return exact;
|
|||
|
|
const tn = normTitle(t);
|
|||
|
|
if (!tn) return null;
|
|||
|
|
let best = null;
|
|||
|
|
for (const v of all) {
|
|||
|
|
const cn = normTitle(v.video_title);
|
|||
|
|
if (!cn || cn.length < 6 || tn.length < 6) continue;
|
|||
|
|
if (tn.includes(cn) || cn.includes(tn)) {
|
|||
|
|
if (!best || tn.length > best._len) { best = { ...v, _len: tn.length }; }
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
if (best) return best;
|
|||
|
|
for (const v of all) {
|
|||
|
|
const cn = normTitle(v.video_title);
|
|||
|
|
if (!cn) continue;
|
|||
|
|
const ratio = 1 - titleDistance(tn, cn) / Math.max(tn.length, cn.length, 1);
|
|||
|
|
if (ratio > 0.8 && (!best || ratio > best._ratio)) { best = { ...v, _ratio: ratio }; }
|
|||
|
|
}
|
|||
|
|
return best;
|
|||
|
|
};
|
|||
|
|
let fixed = 0;
|
|||
|
|
for (const r of missing) {
|
|||
|
|
let title = null;
|
|||
|
|
try {
|
|||
|
|
const jd = JSON.parse(r.content_json);
|
|||
|
|
title = jd && jd.title != null ? String(jd.title) : null;
|
|||
|
|
} catch (e) { /* 单个解析失败跳过 */ }
|
|||
|
|
const hit = title ? find(title) : null;
|
|||
|
|
if (hit && hit.aweme_id) {
|
|||
|
|
d.prepare(`UPDATE account_video_source SET aweme_id=?, video_id=? WHERE id=?`).run(hit.aweme_id, hit.id, r.id);
|
|||
|
|
fixed++;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
// 补 analysis_json 完成（已上移至函数开头执行，保证每次启动都补录）
|
|||
|
|
if (fixed > 0) console.log(`[dsh-plugin-mcn] source 表回填 ${fixed}/${missing.length} 条 aweme_id（标题匹配修复错位）`);
|
|||
|
|
return fixed;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
export { initDb, backfillSourceAwemeIds, extractScriptTitle };
|