Files
dsh_shenxian/scripts/verify-cluster-fs.mjs
admin c70d5d860e feat(cluster): 集群化落地 —— Manager/Worker 拆分 + 归属租约 + 跨机验证(T08)
背景:把平台从「单机单进程」改造成「1 组 Manager + N 台 Worker + 共享归属状态」,
硬约束 = 全程兼容单例模式(deployMode 默认 local;生产切换前 47 一行未动)。

主要改动
1) 数据模型 v7(SQLite 与 PG 两方言同步):新增 dsh_hosts 注册表 +
   dsh_instances.{host_id,epoch,heartbeat_at,lease_until};claimInstance 原子抢占
   (UPDATE … WHERE host_id IS NULL OR lease_until < now)+ pinInstanceHost 钉住归属。
2) 租约与 fencing:src/supervisor/lease.ts(acquire/renew/release + stillHolder 判据 +
   ttl > 2×renew 硬校验);心跳里续租,失权即向 worker 下发更高 epoch(self-fencing)。
   ⚠️ release 只清租约(lease_until),**保留 host_id** —— host_id 是「用户数据在哪台」的锚点。
3) Worker agent(src/worker/agent.ts,子命令 dshs worker):实例生命周期 + 文件面 /fs/*
   + 幂等键(operationId)+ 鉴权(timingSafeEqual);Worker 不写控制面数据
   (apiKey/uid 由 Manager 随 launch 投递,R5 收窄)。
4) 远端 Spawner + LeasedSpawner:按 host 路由(**粘性优先**:有历史归属且那台 up 就留在原地,
   否则按容量选最空的)+ 容量准入 + deployMode=cluster 装配(systemd drop-in,可回滚)。
5) bwrap 修正:**所有挂载点的中间目录统一前置 + 去重 + 由外到内**(「就近创建」会在嵌套前缀下
   遮掉已绑挂载点 ⇒ bwrap: Can't chdir);且**只能用 --tmpfs**,用 --perms 会让 47 的
   bwrap 0.4.0 直接拒启动(沙箱全挂)。
6) 跨机隧道 src/worker/tunnel.ts:SSH ControlMaster + 动态 -R 转发;**自愈由 agent 本地
   20s 定时器驱动**(不能只放 /healthz —— 心跳本身经隧道进来,断了就没人触发它)。
7) 文件面按归属路由(RemoteUserFs):实例与文件必须落在同一台机器,否则实例看不到自己的文件。
8) 观测面:dshs doctor / dshs cluster status。

验证(本次均已实跑)
- test/lease.test.mjs:SQLite 10/10 == PG 10/10
- 组件级端到端 5 个:verify-cluster-{agent,lease,fs,migrate,live}.mjs
- 真跨机(47 Manager / 106 Worker,跨云 + 反向隧道)verify-cluster-cross.mjs 九步全绿
- 域名形态访问 verify-cluster-domain.mjs(<user>.域名 → Manager → 远端实例;越权 403)
- 冒烟 scripts/smoke-*:6/8,失败项与改动前基线完全相同(无回归)
- 生产切换与回滚剧本见 dsh-server-docs/交接单/T08-集群化落地-兼容单例模式.md §16
2026-09-15 18:47:02 +08:00

156 lines
7.2 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* T08 S5 · 跨机文件面验证。
*
* 关键设计:**Manager 的 `dataRoot` 故意与 worker 的 `dataRoot` 不同** ——
* 只有这样"文件面真的走了远端"才被证明;若两个 root 相同,本地实现也能碰巧通过。
*
* 验的是:
* ① 门户的路由(`/api/desktop/tree`、`/api/fs/*`)在 cluster 模式下照常工作(**路由零改动**);
* ② 文件**落在 worker 的 dataRoot 下**、且**不在** Manager 的 dataRoot 下;
* ③ 路径安全与本地**同源**(`bad_path` 走同一条 `resolveWithinRoot`);
* ④ `resolvePath` 返回的是**实例眼里的路径**(按 worker 的 dataRoot 算)。
*
* 运行:node scripts/verify-cluster-fs.mjs
*/
import { existsSync, mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { dirname, join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { buildServer } from '../lib/web/server.js'
import { buildWorkerAgent, AGENT_TOKEN_HEADER } from '../lib/worker/agent.js'
import { resolveConfig } from '../lib/config.js'
import { hashPassword } from '../lib/web/auth.js'
function assert(condition, message) {
if (!condition) throw new Error('ASSERT: ' + message)
}
const here = dirname(fileURLToPath(import.meta.url))
const fakeDsh = join(here, 'fake-dsh.mjs')
const TOKEN = 'verify-cluster-fs-token'
const workerRoot = mkdtempSync(join(tmpdir(), 'dsh-cfs-worker-'))
const managerRoot = mkdtempSync(join(tmpdir(), 'dsh-cfs-manager-'))
let agentApp
let agentHandle
let app
try {
// ── worker agent(dataRoot = workerRoot)────────────────────────────────
const agent = buildWorkerAgent(
resolveConfig({ port: 0, dbPath: ':memory:', dataRoot: workerRoot, dshCommand: [process.execPath, fakeDsh], clusterHostId: 'w-1' }),
{ hostId: 'w-1', token: TOKEN, port: 0, host: '127.0.0.1', instanceHost: '127.0.0.1', logLevel: 'warn' },
)
agentApp = agent.app
agentHandle = agent
await agentApp.listen({ host: '127.0.0.1', port: 0 })
const agentUrl = `http://127.0.0.1:${agentApp.server.address().port}`
console.log('worker -> dataRoot %s', workerRoot)
// ── Manager(dataRoot = managerRoot ≠ workerRoot;显式告知 worker 的 root)──
app = await buildServer(
resolveConfig({
port: 0,
dbPath: ':memory:',
dataRoot: managerRoot,
deployMode: 'cluster',
clusterAgentUrl: agentUrl,
clusterAgentToken: TOKEN,
clusterInstanceHost: '127.0.0.1',
clusterHostId: 'm-1',
clusterWorkerDataRoot: workerRoot,
}),
)
await app.listen({ port: 0 })
const base = `http://127.0.0.1:${app.server.address().port}`
console.log('manager -> dataRoot %s(与 worker 不同 ⇒ 能证明走远端)', managerRoot)
const json = async (path, { method = 'GET', body, cookie } = {}) => {
const res = await fetch(base + path, {
method,
headers: { ...(body ? { 'content-type': 'application/json' } : {}), ...(cookie ? { cookie } : {}) },
body: body ? JSON.stringify(body) : undefined,
})
const text = await res.text()
return { status: res.status, body: text ? JSON.parse(text) : null, setCookie: res.headers.get('set-cookie') }
}
await app.db.createUser({
id: 'u1',
username: 'bob',
passHash: await hashPassword('bobpass123'),
role: 'active',
homeDir: '/tmp/u1-home',
})
// 用户根必须建在 **worker 上**(这一步本身就走远端)
await app.userFs.initUserRoot('u1')
// ④ resolvePath = 实例眼里的路径(按 worker 的 dataRoot)
const resolved = app.userFs.resolvePath('u1', 'proj')
assert(resolved === join(workerRoot, 'users', 'u1', 'ws', 'proj'), `resolvePath 应按 worker 的 root 计算(实际 ${resolved})`)
console.log('④ resolvePath -> %s', resolved)
// ── ① 门户路由(零改动)──────────────────────────────────────────────
let r = await json('/api/auth/login', { method: 'POST', body: { username: 'bob', password: 'bobpass123' } })
assert(r.status === 200, 'login succeeds')
const cookie = r.setCookie.split(';')[0]
r = await json('/api/desktop/tree', { cookie })
assert(r.status === 200 && r.body.entries.length === 0, '空工作区列出 0 项')
r = await json('/api/fs/mkdir', { method: 'POST', cookie, body: { path: 'proj' } })
assert(r.status === 200, `mkdir 经远端成功(实际 ${r.status} ${JSON.stringify(r.body)})`)
r = await json('/api/fs/upload', {
method: 'POST',
cookie,
body: { path: 'proj', name: 'hello.txt', data: Buffer.from('hi there').toString('base64') },
})
assert(r.status === 200, `upload 经远端成功(实际 ${r.status})`)
r = await json('/api/desktop/tree', { cookie })
assert(r.status === 200 && r.body.entries.length === 1, '工作区里出现了 proj')
console.log('① 门户路由 -> tree/mkdir/upload 全部经远端通过')
// ── ② 文件真的落在 worker 上 ──────────────────────────────────────────
const onWorker = join(workerRoot, 'users', 'u1', 'ws', 'proj', 'hello.txt')
const onManager = join(managerRoot, 'users', 'u1', 'ws', 'proj', 'hello.txt')
assert(existsSync(onWorker), `文件应落在 worker:${onWorker}`)
assert(!existsSync(onManager), `文件不该出现在 Manager 本地:${onManager}`)
console.log('② 落点 -> worker 有、manager 无(确认走远端)')
// ── ③ 路径安全与本地同源 ──────────────────────────────────────────────
r = await json('/api/fs/mkdir', { method: 'POST', cookie, body: { path: '../evil' } })
assert(r.status === 400 && r.body.error === 'bad_path', `越界路径应 400 bad_path(实际 ${r.status} ${JSON.stringify(r.body)})`)
for (const bad of ['..', '../../etc']) {
let threw = false
try {
app.userFs.resolvePath('u1', bad)
} catch (err) {
threw = err.code === 'bad_path'
}
assert(threw, `resolvePath(${bad}) 应抛 bad_path`)
}
console.log('③ 路径安全 -> bad_path 与本地同源(走同一个 resolveWithinRoot)')
// 下载回读(readFile 经远端)—— 注意该路由回的是**原始字节**,不是 JSON
const dl = await fetch(`${base}/api/fs/download?path=proj/hello.txt`, { headers: { cookie } })
assert(dl.status === 200, `download 经远端成功(实际 ${dl.status})`)
const downloaded = await dl.text()
assert(downloaded === 'hi there', `下载内容应为上传的原文(实际 ${JSON.stringify(downloaded)})`)
console.log(' 下载回读 -> readFile 经远端成功(内容逐字一致)')
console.log('\nOK: 跨机文件面(RemoteUserFs → agent /fs/*)通过')
console.log(' ✓ 门户路由零改动 ✓ 落在 worker ✓ 路径安全同源 ✓ resolvePath 按 worker 计算')
} finally {
await app?.close()
await agentHandle?.stop()
await new Promise((r) => setTimeout(r, 300))
for (const dir of [workerRoot, managerRoot]) {
try {
rmSync(dir, { recursive: true, force: true })
} catch {
/* best-effort */
}
}
}