R76-R77: 修复解析入库aweme_id关联失败+告警机制—①server.js匹配同构规范化(normKey无空格紧凑+补全角问号,根因:文件夹名吞空格vs库标题留空格)②import加unmatched告警+video_id双键(匹配不到跳过不插脏)③解析prompt加入库自查(返回体unmatched含本视频即报失败)④全功能落库排查库内健康(双null=0/孤儿=0)
This commit is contained in:
maogeigei committed
1 parent
6b6dca7da8
commit
0fe170a04b
4 files changed
+48
-17
No files matched your search
@@ -74,7 +74,7 @@ function resolveAccountDir(name, root) {
|
||||
}
|
||||
|
||||
function importAccountIntoDb(db, account, dir) {
|
||||
const stats = { video_analysis: 0, video_source: 0, persona: 0, account_analysis: 0, videos_upserted: 0, skipped: [] };
|
||||
const stats = { video_analysis: 0, video_source: 0, persona: 0, account_analysis: 0, videos_upserted: 0, unmatched: 0, unmatched_folders: [], skipped: [] };
|
||||
const now = new Date().toISOString();
|
||||
const insAnalysis = db.prepare(`INSERT INTO account_video_analysis (video_id, aweme_id, analysis_time, content_json, summary) VALUES (?,?,?,?,?)`);
|
||||
const insSource = db.prepare(`INSERT INTO account_video_source (video_id, aweme_id, analysis_time, content_json, summary, analysis_json) VALUES (?,?,?,?,?,?)`);
|
||||
@@ -85,18 +85,23 @@ function importAccountIntoDb(db, account, dir) {
|
||||
// 清理历史误插脏数据：09-01 匹配缺陷（标点未规范化）产生 aweme_id=null 且 video_id=null 的无意义记录，重跑入库前先清
|
||||
db.prepare("DELETE FROM account_video_analysis WHERE aweme_id IS NULL AND video_id IS NULL").run();
|
||||
db.prepare("DELETE FROM account_video_source WHERE aweme_id IS NULL AND video_id IS NULL").run();
|
||||
// 规范化标题：去 # 标签 → 删全角/半角标点（xlsx 标题带「！」「“”」而文件夹名无，09-01 匹配失败根因）→ 下划线转空格（对齐文件夹 _标签 结构）→ 空格归一
|
||||
const normTitle = (s) => String(s || '').split('#')[0].replace(/[！!“”‘’"'`,.…。，、；：:;·]/g, '').replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
// 规范化标题：去 # 标签 → 删全角/半角标点（xlsx 标题带「！」「“”」而文件夹名无，09-01 匹配失败根因；09-04 补全角问号「？」——folder 生成时删 ? 而 title 未删导致残留干扰）→ 下划线转空格（对齐文件夹 _标签 结构）→ 空格归一
|
||||
const normTitle = (s) => String(s || '').split('#')[0].replace(/[！!？?“”‘’"'`,.…。，、；：:;·]/g, '').replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim();
|
||||
// 09-04 修复：文件夹名与 video_title 的规范化不对称——文件夹命名规则「标题去特殊符号」会把词间空格一并吞掉
|
||||
// （"小主人: 终于"→"小主人终于"），而 normTitle 只删标点保留空格（"小主人 终于"）→ 两侧精确/包含匹配双失效
|
||||
// → aweme_id=null 脏入库 → 前端视频详情（按 aweme_id 关联，要求 NOT NULL）查不到解析。
|
||||
// 治本：比较键统一为「无空格紧凑形态」（normTitle 后再删全部空白），folder 与 title 两侧同构；精确失败后走无空格包含匹配。
|
||||
const normKey = (s) => normTitle(s).replace(/\s+/g, '');
|
||||
// 0) aweme_id 权威映射：短视频表格.xlsx（子进程 python，最高优先级）→ account_videos 表（兜底）
|
||||
const xlsxList = readXlsxMap(dir);
|
||||
const xlsxByNorm = {};
|
||||
for (const r of xlsxList) { if (r.aweme_id && r.title) xlsxByNorm[normTitle(r.title)] = r.aweme_id; }
|
||||
for (const r of xlsxList) { if (r.aweme_id && r.title) xlsxByNorm[normKey(r.title)] = r.aweme_id; }
|
||||
const titleRows = db.prepare('SELECT aweme_id, video_title FROM account_videos WHERE video_title IS NOT NULL').all();
|
||||
const dbByNorm = {};
|
||||
for (const r of titleRows) { if (r.aweme_id && r.video_title) dbByNorm[normTitle(r.video_title)] = String(r.aweme_id); }
|
||||
// 匹配：精确优先；包含匹配要求双方标题 ≥10 字且按标题长度降序（最具体优先，防短标题误配）
|
||||
for (const r of titleRows) { if (r.aweme_id && r.video_title) dbByNorm[normKey(r.video_title)] = String(r.aweme_id); }
|
||||
// 匹配：无空格紧凑精确优先；包含匹配要求双方 ≥10 字符且按长度降序（最具体优先，防短标题误配）
|
||||
const matchAweme = (folder) => {
|
||||
const base = normTitle(folder);
|
||||
const base = normKey(folder);
|
||||
if (!base) return null;
|
||||
if (xlsxByNorm[base]) return xlsxByNorm[base];
|
||||
if (dbByNorm[base]) return dbByNorm[base];
|
||||
@@ -134,22 +139,33 @@ function importAccountIntoDb(db, account, dir) {
|
||||
let st; try { st = fs.statSync(fdir); } catch (e) { continue; }
|
||||
if (!st.isDirectory()) continue;
|
||||
const awemeId = matchAweme(folder);
|
||||
// 09-04：awemeId 命中后反查 account_videos.id 一并填 video_id（双键关联，前端 video_id/aweme_id 皆可命中；历史 source 行 video_id 多为 null 靠 aweme_id 兜底）
|
||||
let videoId = null;
|
||||
if (awemeId) { const vv = db.prepare('SELECT id FROM account_videos WHERE aweme_id=?').get(awemeId); videoId = vv ? vv.id : null; }
|
||||
const anPath = path.join(fdir, 'analysis.json');
|
||||
const coPath = path.join(fdir, 'content.json');
|
||||
if (fs.existsSync(coPath)) {
|
||||
const coExists = fs.existsSync(coPath), anExists = fs.existsSync(anPath);
|
||||
if (!coExists && !anExists) continue; // 空文件夹（半途写入残留）跳过，不计 unmatched
|
||||
// 09-04：aweme_id 匹配不到 → 记 unmatched 告警并跳过插入（不再制造双 null 脏记录、避免"入库假成功"；补 xlsx/视频列表后重跑 import 即恢复）
|
||||
if (!awemeId) { stats.unmatched++; stats.unmatched_folders.push(folder); }
|
||||
if (coExists) {
|
||||
const content = fs.readFileSync(coPath, 'utf8');
|
||||
const analysisJson = fs.existsSync(anPath) ? fs.readFileSync(anPath, 'utf8') : null;
|
||||
if (awemeId) db.prepare('DELETE FROM account_video_source WHERE aweme_id=?').run(awemeId);
|
||||
insSource.run(null, awemeId, now, content, '', analysisJson);
|
||||
stats.video_source++;
|
||||
const analysisJson = anExists ? fs.readFileSync(anPath, 'utf8') : null;
|
||||
if (awemeId) {
|
||||
db.prepare('DELETE FROM account_video_source WHERE aweme_id=?').run(awemeId);
|
||||
insSource.run(videoId, awemeId, now, content, '', analysisJson);
|
||||
stats.video_source++;
|
||||
}
|
||||
}
|
||||
// 拆解分析.md → account_video_analysis（功能四逐条拆解产物，与解析并存于同一文件夹；无则不入）
|
||||
const decon = fs.readdirSync(fdir).find((f) => f.endsWith('拆解分析.md'));
|
||||
if (decon) {
|
||||
const content = fs.readFileSync(path.join(fdir, decon), 'utf8');
|
||||
if (awemeId) db.prepare('DELETE FROM account_video_analysis WHERE aweme_id=?').run(awemeId);
|
||||
insAnalysis.run(null, awemeId, now, content, content.slice(0, 500));
|
||||
stats.video_analysis++;
|
||||
if (awemeId) {
|
||||
db.prepare('DELETE FROM account_video_analysis WHERE aweme_id=?').run(awemeId);
|
||||
insAnalysis.run(videoId, awemeId, now, content, content.slice(0, 500));
|
||||
stats.video_analysis++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user