// MCN热点雷达 - 本地 Web 服务(零依赖 Node 实现)
// 启动: node server.js [端口] → http://localhost:8900(默认;可传参指定端口,如 node server.js 9000)
// 功能: 技能导航 + 产出内容浏览(默认读桌面 MCNSkill项目)+ dsh 数据功能页(账号/视频/脚本/复盘/周榜,只读)
'use strict';
const http = require('http');
const fs = require('fs');
const path = require('path');
const os = require('os');
const { execFileSync } = require('child_process');
const { URL } = require('url');
const dsh = require('./dsh-data');
const { parseAttachment } = require('./scripts/attachment-parser');
// 09-01 默认端口由 8899 改为 8900:本机 8899 被小米 PC 管家 MiPCAudio.exe 系统服务占用(0.0.0.0 监听且自动复活),
// 此时 127.0.0.1:8899 绑定不生效 → 默认直接起在 8900;仍可 `node server.js <端口>` 显式换端口
const PORT_BASE = parseInt(process.argv[2], 10) || 8900;
const ROOT = __dirname;
const PUBLIC_DIR = path.join(ROOT, 'public');
const CONFIG_FILE = path.join(ROOT, 'config.json');
// ---- 默认根目录:桌面/MCNSkill项目 ----
const DEFAULT_ROOT = path.join(os.homedir(), 'Desktop', 'MCNSkill项目');
// ---- 配置读写(自定义根目录列表)----
function loadConfig() {
try {
if (fs.existsSync(CONFIG_FILE)) {
const c = JSON.parse(fs.readFileSync(CONFIG_FILE, 'utf8'));
if (Array.isArray(c.extraRoots)) return { extraRoots: c.extraRoots };
}
} catch (e) { /* 忽略损坏配置 */ }
return { extraRoots: [] };
}
function saveConfig(cfg) {
fs.writeFileSync(CONFIG_FILE, JSON.stringify(cfg, null, 2), 'utf8');
}
// ---- 允许读取的根目录白名单 ----
function allowedRoots() {
const cfg = loadConfig();
const list = [DEFAULT_ROOT].concat(cfg.extraRoots || []);
// 去重、只保留存在的目录
return list.filter((p, i) => p && fs.existsSync(p) && fs.statSync(p).isDirectory() && list.indexOf(p) === i);
}
// ---- 路径安全:必须落在某个允许根目录内 ----
function resolveSafe(rawPath) {
if (!rawPath) return null;
let p;
try { p = path.resolve(rawPath); } catch (e) { return null; }
for (const root of allowedRoots()) {
const r = path.resolve(root);
if (p === r || p.startsWith(r + path.sep)) {
return { abs: p, root: r };
}
}
return null;
}
// ---- 分析产物入库(09-01 新增):扫描账号产出目录 → 写工作台库 mcn-plugin.db ----
// 背景:账号分析技能产出只落盘文件系统未入库(俊希解析/人设卡全缺),工作台选题等读不到视频内容。
// 实现参照 dsh-plugin-mcn lib/imports.js 的 analyses 入库逻辑(dsh 接口 3080/3081 不可用时直写库)。
function resolveAccountDir(name, root) {
const safeName = String(name || '').replace(/[\\/:*?"<>|]/g, '_').trim();
if (!safeName) return null;
const roots = root ? [root] : allowedRoots();
for (const r of roots) {
const dir = path.join(r, safeName);
try { if (fs.statSync(dir).isDirectory()) return dir; } catch (e) { /* 忽略 */ }
}
return null;
}
function importAccountIntoDb(db, account, dir) {
const stats = { video_analysis: 0, video_source: 0, persona: 0, account_analysis: 0, videos_upserted: 0, skipped: [] };
const now = new Date().toISOString();
const insAnalysis = db.prepare(`INSERT INTO account_video_analysis (video_id, aweme_id, analysis_time, content_json, summary) VALUES (?,?,?,?,?)`);
const insSource = db.prepare(`INSERT INTO account_video_source (video_id, aweme_id, analysis_time, content_json, summary, analysis_json) VALUES (?,?,?,?,?,?)`);
const insPersona = db.prepare(`INSERT INTO account_persona (account_id, analysis_time, content_json, summary) VALUES (?,?,?,?)`);
const insAccAna = db.prepare(`INSERT INTO account_analysis (account_id, analysis_time, content_json, summary) VALUES (?,?,?,?)`);
const acc = db.prepare('SELECT id FROM hot_accounts WHERE account_name=? LIMIT 1').get(account);
const accountId = acc ? acc.id : null;
// 清理历史误插脏数据:09-01 匹配缺陷(标点未规范化)产生 aweme_id=null 且 video_id=null 的无意义记录,重跑入库前先清
db.prepare("DELETE FROM account_video_analysis WHERE aweme_id IS NULL AND video_id IS NULL").run();
db.prepare("DELETE FROM account_video_source WHERE aweme_id IS NULL AND video_id IS NULL").run();
// 规范化标题:去 # 标签 → 删全角/半角标点(xlsx 标题带「!」「“”」而文件夹名无,09-01 匹配失败根因)→ 下划线转空格(对齐文件夹 _标签 结构)→ 空格归一
const normTitle = (s) => String(s || '').split('#')[0].replace(/[!!“”‘’"'`,.…。,、;::;·]/g, '').replace(/[_-]+/g, ' ').replace(/\s+/g, ' ').trim();
// 0) aweme_id 权威映射:短视频表格.xlsx(子进程 python,最高优先级)→ account_videos 表(兜底)
const xlsxList = readXlsxMap(dir);
const xlsxByNorm = {};
for (const r of xlsxList) { if (r.aweme_id && r.title) xlsxByNorm[normTitle(r.title)] = r.aweme_id; }
const titleRows = db.prepare('SELECT aweme_id, video_title FROM account_videos WHERE video_title IS NOT NULL').all();
const dbByNorm = {};
for (const r of titleRows) { if (r.aweme_id && r.video_title) dbByNorm[normTitle(r.video_title)] = String(r.aweme_id); }
// 匹配:精确优先;包含匹配要求双方标题 ≥10 字且按标题长度降序(最具体优先,防短标题误配)
const matchAweme = (folder) => {
const base = normTitle(folder);
if (!base) return null;
if (xlsxByNorm[base]) return xlsxByNorm[base];
if (dbByNorm[base]) return dbByNorm[base];
const cands = [];
for (const [t, aw] of Object.entries(xlsxByNorm)) {
if (base.length >= 10 && t.length >= 10 && (base.includes(t) || t.includes(base))) cands.push([t.length, aw]);
}
if (!cands.length) {
for (const [t, aw] of Object.entries(dbByNorm)) {
if (base.length >= 10 && t.length >= 10 && (base.includes(t) || t.includes(base))) cands.push([t.length, aw]);
}
}
cands.sort((a, b) => b[0] - a[0]);
return cands.length ? cands[0][1] : null;
};
// 0.5) 视频列表入库:短视频表格.xlsx → account_videos(按 account_id+aweme_id 去重,补上工作台视频列表数据)
if (accountId && xlsxList.length) {
const insVideo = db.prepare(`INSERT INTO account_videos (account_id, aweme_id, video_title, video_url, like_count, comment_count, share_count, collect_count, play_count, duration, publish_time, tags, collected_time) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)`);
for (const v of xlsxList) {
const ex = db.prepare('SELECT id FROM account_videos WHERE account_id=? AND aweme_id=? LIMIT 1').get(accountId, v.aweme_id);
if (!ex) {
insVideo.run(accountId, v.aweme_id, v.title, v.url || null, v.like_count ?? null, v.comment_count ?? null, v.share_count ?? null, v.collect_count ?? null, v.play_count ?? null, v.duration ?? null, v.publish_time || null, v.tags || null, now);
stats.videos_upserted++;
}
}
}
// 1) 视频分析目录(功能三 Step5 产出;功能五产出在 视频对标/ 目录,由 dsh 导入机制处理)
// 字段映射(09-01 修正,对齐 dsh 语义):content.json→account_video_source.content_json(视频脚本);
// analysis.json→account_video_source.analysis_json(视频拆解,MCP 结构化分析);*拆解分析.md→account_video_analysis(视频分析,功能四产物)。
// 原实现把 analysis.json 误写 account_video_analysis 导致「拆解内容进了视频分析」+「视频拆解 tab 空」
const vaDir = path.join(dir, '视频分析');
if (fs.existsSync(vaDir)) {
for (const folder of fs.readdirSync(vaDir)) {
const fdir = path.join(vaDir, folder);
let st; try { st = fs.statSync(fdir); } catch (e) { continue; }
if (!st.isDirectory()) continue;
const awemeId = matchAweme(folder);
const anPath = path.join(fdir, 'analysis.json');
const coPath = path.join(fdir, 'content.json');
if (fs.existsSync(coPath)) {
const content = fs.readFileSync(coPath, 'utf8');
const analysisJson = fs.existsSync(anPath) ? fs.readFileSync(anPath, 'utf8') : null;
if (awemeId) db.prepare('DELETE FROM account_video_source WHERE aweme_id=?').run(awemeId);
insSource.run(null, awemeId, now, content, '', analysisJson);
stats.video_source++;
}
// 拆解分析.md → account_video_analysis(功能四逐条拆解产物,与解析并存于同一文件夹;无则不入)
const decon = fs.readdirSync(fdir).find((f) => f.endsWith('拆解分析.md'));
if (decon) {
const content = fs.readFileSync(path.join(fdir, decon), 'utf8');
if (awemeId) db.prepare('DELETE FROM account_video_analysis WHERE aweme_id=?').run(awemeId);
insAnalysis.run(null, awemeId, now, content, content.slice(0, 500));
stats.video_analysis++;
}
}
}
// 2) 人设卡 + 账号分析(去重,不删旧)
if (accountId) {
const personaPath = path.join(dir, account + '账号设定.md');
if (fs.existsSync(personaPath)) {
const content = fs.readFileSync(personaPath, 'utf8');
const dup = db.prepare('SELECT id FROM account_persona WHERE account_id=? AND content_json=? LIMIT 1').get(accountId, content);
if (!dup) { insPersona.run(accountId, now, content, content.slice(0, 200)); stats.persona++; }
else stats.skipped.push('persona(重复)');
}
const anaPath = path.join(dir, account + '账号数据分析.md');
if (fs.existsSync(anaPath)) {
const content = fs.readFileSync(anaPath, 'utf8');
const dup = db.prepare('SELECT id FROM account_analysis WHERE account_id=? AND content_json=? LIMIT 1').get(accountId, content);
if (!dup) { insAccAna.run(accountId, now, content, content.slice(0, 200)); stats.account_analysis++; }
else stats.skipped.push('account_analysis(重复)');
}
}
return { account, account_id: accountId, dir, xlsx_rows: xlsxList.length, title_map_size: Object.keys(dbByNorm).length, ...stats };
}
// 读短视频表格.xlsx → 视频映射数组(子进程调 python export_video_map.py,openpyxl 读 xlsx;失败返回 [])
function readXlsxMap(dir) {
const xlsxPath = path.join(dir, '短视频表格.xlsx');
if (!fs.existsSync(xlsxPath)) return [];
const py = process.env.PYTHON || 'python';
try {
const out = execFileSync(py, [path.join(ROOT, 'scripts', 'export_video_map.py'), xlsxPath], { encoding: 'utf8', timeout: 20000, windowsHide: true });
const arr = JSON.parse(out);
return Array.isArray(arr) ? arr : [];
} catch (e) {
log('xlsx 视频映射读取失败(用表内匹配兜底): ' + e.message);
return [];
}
}
// ---- 查找账号设定卡 SVG ----
// 产出规范:{产出根}/{达人昵称}/{达人昵称}账号设定卡.svg(功能六产出,与账号设定.md 同目录)
// 查找顺序:规范路径 → 根下直挂 → 账号目录内扫描(防目录名/账号名不一致)
function findAccountCard(name) {
const safeName = String(name || '').replace(/[\\/:*?"<>|]/g, '_').trim();
if (!safeName) return null;
for (const root of allowedRoots()) {
const p1 = path.join(root, safeName, safeName + '账号设定卡.svg');
const p2 = path.join(root, safeName + '账号设定卡.svg');
for (const c of [p1, p2]) {
try { if (fs.existsSync(c) && fs.statSync(c).isFile()) return c; } catch (e) { /* 忽略 */ }
}
}
for (const root of allowedRoots()) {
const dir = path.join(root, safeName);
try {
if (fs.statSync(dir).isDirectory()) {
const hit = fs.readdirSync(dir).find((f) => f.toLowerCase().endsWith('.svg') && f.includes('账号设定卡'));
if (hit) return path.join(dir, hit);
}
} catch (e) { /* 忽略 */ }
}
return null;
}
// ---- 目录树(递归,跳过隐藏项)----
function buildTree(absPath, depth) {
if (depth > 4) return null;
let entries;
try { entries = fs.readdirSync(absPath, { withFileTypes: true }); } catch (e) { return null; }
const dirs = [];
const files = [];
for (const ent of entries) {
if (ent.name.startsWith('.')) continue; // 隐藏文件
if (ent.name === 'node_modules' || ent.name === '.git') continue;
const full = path.join(absPath, ent.name);
try {
if (ent.isDirectory()) {
const child = buildTree(full, depth + 1);
if (child) dirs.push({ name: ent.name, path: full, type: 'dir', children: child });
} else if (ent.isFile()) {
const st = fs.statSync(full);
files.push({ name: ent.name, path: full, type: 'file', ext: path.extname(ent.name).toLowerCase(), size: st.size });
}
} catch (e) { /* 权限/占用跳过 */ }
}
dirs.sort((a, b) => a.name.localeCompare(b.name, 'zh'));
files.sort((a, b) => a.name.localeCompare(b.name, 'zh'));
return dirs.concat(files);
}
// ---- MIME ----
const MIME = {
'.html': 'text/html; charset=utf-8',
'.js': 'text/javascript; charset=utf-8',
'.css': 'text/css; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.svg': 'image/svg+xml',
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.gif': 'image/gif',
'.ico': 'image/x-icon',
'.woff2': 'font/woff2',
};
function mimeOf(p) {
return MIME[path.extname(p).toLowerCase()] || 'application/octet-stream';
}
// ---- 读取文本文件(按扩展名判定预览能力)----
const PREVIEW_EXTS = ['.md', '.markdown', '.txt', '.json', '.csv', '.log'];
function readText(absPath) {
const ext = path.extname(absPath).toLowerCase();
if (!PREVIEW_EXTS.includes(ext)) return null;
const buf = fs.readFileSync(absPath);
// BOM 去除
let text = buf.toString('utf8');
if (text.charCodeAt(0) === 0xFEFF) text = text.slice(1);
return text;
}
// ---- 简易日志 ----
function log(msg) {
console.log('[' + new Date().toLocaleTimeString('zh-CN', { hour12: false }) + '] ' + msg);
}
// ---------- 榜单数据更新任务(09-03 新增:热点数据页「更新榜单」按钮后台任务)----------
// 任务名固定用于防重识别(同名最近一次任务状态 = 是否有更新在跑 / 上次执行结果)
const RANK_UPDATE_NAME = '更新榜单数据';
const RANK_UPDATE_COOLDOWN_MS = 30 * 60 * 1000; // 防重复更新窗口:距上次成功落盘 < 30 分钟拦截(可 force 跳过)
// workbuddy 宿主库(automations/sessions/automation_runs),与 /api/run 同款路径解析
function wbDb() {
const { DatabaseSync } = require('node:sqlite');
return new DatabaseSync(process.env.WORKBUDDY_DB || path.join(os.homedir(), '.workbuddy', 'workbuddy.db'));
}
// 创建一次性后台任务(08-31 实测约定:scheduled_at 与 next_run_at 均取 +5s 未来,调度器按 next_run_at 扫描拾取;
// 模型跟随用户最近活跃手动会话选择,兜底 'auto')——逻辑与 /api/run 完全一致,供更新榜单等新入口复用
function createOnceAutomation(name, prompt, skillsArr) {
const db = wbDb();
try {
const now = Date.now();
const id = 'automation-' + now;
const cwd = 'D:\\AgentSkill\\mcn-workshop'; // 工作台触发的会话归入 mcn-workshop 空间分组
const d = new Date(now + 5 * 1000);
const pad = (n) => String(n).padStart(2, '0');
const scheduledAt = `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())}T${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}`;
const nextRunAt = now + 5 * 1000;
const uidRow = db.prepare("SELECT user_id FROM sessions WHERE user_id IS NOT NULL AND user_id <> '' ORDER BY created_at DESC LIMIT 1").get();
const ownerUserId = uidRow ? uidRow.user_id : '';
const modelRow = db.prepare(`SELECT model FROM sessions WHERE deleted_at IS NULL AND COALESCE(is_background_automation,0)=0 AND model IS NOT NULL AND model <> '' ORDER BY COALESCE(last_activity_at, updated_at) DESC LIMIT 1`).get();
const modelId = (modelRow && modelRow.model) ? modelRow.model : 'auto';
db.prepare(`INSERT INTO automations (id,name,prompt,status,schedule_type,scheduled_at,next_run_at,rrule,cwds,created_at,updated_at,skills_json,connector_ids_json,model_id,permission_mode,owner_user_id,owner_status)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`)
.run(id, name, prompt, 'ACTIVE', 'once', scheduledAt, nextRunAt, '', JSON.stringify([cwd]), now, now, JSON.stringify(Array.isArray(skillsArr) ? skillsArr : []), '[]', modelId, 'fullAccess', ownerUserId, 'confirmed');
return { id };
} finally {
db.close();
}
}
// 最近一次「更新榜单数据」任务状态(running/queued/pending = 进行中;done/error/review = 已结束)
function rankLastTaskState() {
try {
const db = wbDb();
try {
const row = db.prepare('SELECT id, status, deleted_at FROM automations WHERE name=? AND deleted_at IS NULL ORDER BY created_at DESC LIMIT 1').get(RANK_UPDATE_NAME);
if (!row) return null;
const st = db.prepare('SELECT running, last_error FROM automation_runtime_state WHERE automation_id=?').get(row.id);
const runs = db.prepare('SELECT status FROM automation_runs WHERE automation_id=? ORDER BY created_at DESC LIMIT 1').get(row.id);
let state = 'queued';
if (st && Number(st.running) === 1) state = 'running';
else if (runs) {
const s = String(runs.status || '').toUpperCase();
if (s === 'IN_PROGRESS') state = 'running';
else if (['DONE', 'COMPLETED', 'SUCCESS', 'FINISHED', 'ACCEPTED'].includes(s)) state = 'done';
else if (['ERROR', 'CANCELLED', 'FAILED', 'INTERRUPTED'].includes(s)) state = 'error';
else if (s === 'PENDING_REVIEW') state = 'review';
else state = 'pending';
}
return { id: row.id, state, error: (st && st.last_error) || null };
} finally { db.close(); }
} catch (e) { return null; }
}
// ---- 静态文件 / API ----
const server = http.createServer((req, res) => {
const u = new URL(req.url, 'http://localhost');
const p = decodeURIComponent(u.pathname);
const q = u.searchParams;
// 统一 JSON 响应
const sendJSON = (code, obj) => {
const body = JSON.stringify(obj);
res.writeHead(code, { 'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store' });
res.end(body);
};
const sendErr = (code, msg) => sendJSON(code, { error: msg });
// ---------- API ----------
if (p === '/api/roots') {
return sendJSON(200, {
defaultRoot: DEFAULT_ROOT,
extraRoots: loadConfig().extraRoots,
defaultExists: fs.existsSync(DEFAULT_ROOT),
});
}
if (p === '/api/roots' && req.method === 'POST') {
let body = '';
req.on('data', (c) => { body += c; if (body.length > 1e6) req.destroy(); });
req.on('end', () => {
try {
const { extraRoots } = JSON.parse(body || '{}');
if (!Array.isArray(extraRoots)) return sendErr(400, 'extraRoots 必须为数组');
const cleaned = extraRoots
.map((x) => (typeof x === 'string' ? x.trim() : ''))
.filter((x) => x && fs.existsSync(x) && fs.statSync(x).isDirectory());
saveConfig({ extraRoots: cleaned });
sendJSON(200, { extraRoots: cleaned });
} catch (e) {
sendErr(500, '配置保存失败: ' + e.message);
}
});
return;
}
if (p === '/api/tree') {
const rootParam = q.get('root') || DEFAULT_ROOT;
const safe = resolveSafe(rootParam);
if (!safe) return sendErr(403, '目录不在允许范围,请先在设置中添加');
const children = buildTree(safe.abs, 0) || [];
return sendJSON(200, { root: safe.abs, tree: children });
}
if (p === '/api/read') {
const safe = resolveSafe(q.get('path'));
if (!safe) return sendErr(403, '路径不在允许范围');
try {
const st = fs.statSync(safe.abs);
if (st.size > 5 * 1024 * 1024) return sendErr(413, '文件超过 5MB,不支持在线预览,请下载查看');
const text = readText(safe.abs);
if (text === null) return sendErr(415, '该类型不支持在线预览,请下载后用本地软件打开');
return sendJSON(200, {
name: path.basename(safe.abs),
path: safe.abs,
ext: path.extname(safe.abs).toLowerCase(),
size: st.size,
content: text,
});
} catch (e) {
return sendErr(500, '读取失败: ' + e.message);
}
}
if (p === '/api/download') {
const safe = resolveSafe(q.get('path'));
if (!safe) return sendErr(403, '路径不在允许范围');
try {
const st = fs.statSync(safe.abs);
const fname = encodeURIComponent(path.basename(safe.abs));
res.writeHead(200, {
'Content-Type': mimeOf(safe.abs),
'Content-Length': st.size,
'Content-Disposition': "attachment; filename*=UTF-8''" + fname,
'Cache-Control': 'no-store',
});
fs.createReadStream(safe.abs).pipe(res);
} catch (e) {
return sendErr(500, '下载失败: ' + e.message);
}
return;
}
if (p === '/api/file') {
// 内联输出(无 attachment 头):供
/