Files
admin c70d5d860e feat(cluster): 集群化落地 —— Manager/Worker 拆分 + 归属租约 + 跨机验证(T08)
背景:把平台从「单机单进程」改造成「1 组 Manager + N 台 Worker + 共享归属状态」,
硬约束 = 全程兼容单例模式(deployMode 默认 local;生产切换前 47 一行未动)。

主要改动
1) 数据模型 v7(SQLite 与 PG 两方言同步):新增 dsh_hosts 注册表 +
   dsh_instances.{host_id,epoch,heartbeat_at,lease_until};claimInstance 原子抢占
   (UPDATE … WHERE host_id IS NULL OR lease_until < now)+ pinInstanceHost 钉住归属。
2) 租约与 fencing:src/supervisor/lease.ts(acquire/renew/release + stillHolder 判据 +
   ttl > 2×renew 硬校验);心跳里续租,失权即向 worker 下发更高 epoch(self-fencing)。
   ⚠️ release 只清租约(lease_until),**保留 host_id** —— host_id 是「用户数据在哪台」的锚点。
3) Worker agent(src/worker/agent.ts,子命令 dshs worker):实例生命周期 + 文件面 /fs/*
   + 幂等键(operationId)+ 鉴权(timingSafeEqual);Worker 不写控制面数据
   (apiKey/uid 由 Manager 随 launch 投递,R5 收窄)。
4) 远端 Spawner + LeasedSpawner:按 host 路由(**粘性优先**:有历史归属且那台 up 就留在原地,
   否则按容量选最空的)+ 容量准入 + deployMode=cluster 装配(systemd drop-in,可回滚)。
5) bwrap 修正:**所有挂载点的中间目录统一前置 + 去重 + 由外到内**(「就近创建」会在嵌套前缀下
   遮掉已绑挂载点 ⇒ bwrap: Can't chdir);且**只能用 --tmpfs**,用 --perms 会让 47 的
   bwrap 0.4.0 直接拒启动(沙箱全挂)。
6) 跨机隧道 src/worker/tunnel.ts:SSH ControlMaster + 动态 -R 转发;**自愈由 agent 本地
   20s 定时器驱动**(不能只放 /healthz —— 心跳本身经隧道进来,断了就没人触发它)。
7) 文件面按归属路由(RemoteUserFs):实例与文件必须落在同一台机器,否则实例看不到自己的文件。
8) 观测面:dshs doctor / dshs cluster status。

验证(本次均已实跑)
- test/lease.test.mjs:SQLite 10/10 == PG 10/10
- 组件级端到端 5 个:verify-cluster-{agent,lease,fs,migrate,live}.mjs
- 真跨机(47 Manager / 106 Worker,跨云 + 反向隧道)verify-cluster-cross.mjs 九步全绿
- 域名形态访问 verify-cluster-domain.mjs(<user>.域名 → Manager → 远端实例;越权 403)
- 冒烟 scripts/smoke-*:6/8,失败项与改动前基线完全相同(无回归)
- 生产切换与回滚剧本见 dsh-server-docs/交接单/T08-集群化落地-兼容单例模式.md §16
2026-09-15 18:47:02 +08:00

184 lines
7.2 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* T08 S2 · 实例归属租约单测。
*
* 刻意**不 mock 时钟**:走真实 SQL(SqliteAdapter(':memory:'))并用**极短 TTL** 制造过期,
* 这样测到的是"SQL 的原子抢占真的成立",而不是"我的假时钟算对了"。
*
* 两个后端都跑(同 T08 S1 的做法):默认 SQLite(内存库,每用例一份);
* 设 `LEASETEST_PG_URL=postgres://…` 时改跑 PG —— 用来验证两套实现语义一致。
* 运行:node --test test/lease.test.mjs
* LEASETEST_PG_URL=postgres://dshs:[email protected]:15432/dshs_smoke node --test test/lease.test.mjs
*/
import { test } from 'node:test'
import assert from 'node:assert/strict'
import pg from 'pg'
import { SqliteAdapter } from '../lib/db/sqlite.js'
import { openPgAdapter } from '../lib/db/pg.js'
import { InstanceLease, stillHolder, DEFAULT_LEASE_TTL_MS } from '../lib/supervisor/lease.js'
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
const PG_URL = process.env.LEASETEST_PG_URL
/** PG 侧:每个用例前清空三张表(该库专供本测试,清空是安全的)。 */
async function resetPg() {
const client = new pg.Client({ connectionString: PG_URL })
await client.connect()
await client.query('DELETE FROM dsh_instances')
await client.query('DELETE FROM dsh_hosts')
await client.query('DELETE FROM users')
await client.end()
}
/** 每个用例一个独立后端(SQLite = 新内存库;PG = 清空后的专用库)。 */
async function freshDb() {
const db = PG_URL === undefined ? new SqliteAdapter(':memory:', 100000) : await openPgAdapter(PG_URL, 100000)
if (PG_URL !== undefined) await resetPg()
await db.createUser({
id: 'u1',
username: 'alice',
passHash: 'x',
role: 'active',
homeDir: '/tmp/u1',
})
return db
}
/** 短 TTL 的租约(ttl=60ms > 2×20ms,满足不变量)。 */
function shortLease(db, hostId) {
return new InstanceLease(db, hostId, { ttlMs: 60, renewMs: 20 })
}
test('lease: 默认时序满足 ttl > 2×renew 不变量', () => {
assert.ok(DEFAULT_LEASE_TTL_MS > 2 * 10_000, '默认 30s TTL 必须 > 2×10s 续租')
})
test('lease: 违反不变量时构造即抛(fail-loud,防抖动误判)', async () => {
const db = await freshDb()
assert.throws(() => new InstanceLease(db, 'w-a', { ttlMs: 100, renewMs: 60 }), /ttlMs/)
await db.close()
})
test('lease: 首次抢占成功,epoch 从 1 开始', async () => {
const db = await freshDb()
const a = new InstanceLease(db, 'w-a', { ttlMs: 5000, renewMs: 1000 })
const r = await a.acquire('u1')
assert.equal(r.ok, true)
assert.equal(r.epoch, 1)
assert.ok(r.leaseUntil > Date.now(), '租约应在未来')
assert.deepEqual(a.holdings().get('u1'), { epoch: 1, hostId: 'w-a' })
await db.close()
})
test('lease: 未过期时他人抢占失败 —— 单写者保证', async () => {
const db = await freshDb()
const a = new InstanceLease(db, 'w-a', { ttlMs: 5000, renewMs: 1000 })
const b = new InstanceLease(db, 'w-b', { ttlMs: 5000, renewMs: 1000 })
assert.equal((await a.acquire('u1')).ok, true)
const r = await b.acquire('u1')
assert.equal(r.ok, false, '有人在管 ⇒ 必须退让')
assert.equal(r.holder, 'w-a')
assert.equal(b.holdings().has('u1'), false, '失败不得写入本地持有记录')
await db.close()
})
test('lease: 续租必须带 epoch —— 旧持有者续租失败(fencing 生效)', async () => {
const db = await freshDb()
const a = shortLease(db, 'w-a')
assert.equal((await a.acquire('u1')).ok, true)
const staleEpoch = a.holdings().get('u1').epoch
await sleep(90) // 让租约过期
const b = shortLease(db, 'w-b')
const taken = await b.acquire('u1')
assert.equal(taken.ok, true, '过期后可被抢占')
assert.equal(taken.epoch, staleEpoch + 1, 'epoch 必须递增')
// 老持有者拿着旧 epoch 续租 ⇒ 必须失败(否则就脑裂双写了)
assert.equal(await a.renew('u1'), false)
assert.equal(a.holdings().has('u1'), false, '失权后必须清掉本地记录(供 self-fence)')
// 直接调 DB 层也一样:epoch 不匹配 → 不更新
assert.equal(await db.renewInstanceLease('u1', 'w-a', staleEpoch, 60), false)
assert.equal(await b.renew('u1'), true, '新持有者续租成功')
await db.close()
})
test('lease: 释放后归零,可再次抢占且 epoch 继续递增', async () => {
const db = await freshDb()
const a = new InstanceLease(db, 'w-a', { ttlMs: 5000, renewMs: 1000 })
await a.acquire('u1')
assert.equal(await a.release('u1'), true)
assert.equal(a.holdings().has('u1'), false)
const inst = await db.findUserInstance('u1', 'main')
assert.equal(inst.hostId, null, '释放 = 归属清空')
assert.equal(stillHolder(inst, 'w-a', 1), false)
const b = new InstanceLease(db, 'w-b', { ttlMs: 5000, renewMs: 1000 })
const r = await b.acquire('u1')
assert.equal(r.ok, true)
assert.equal(r.epoch, 2, 'epoch 单调递增(不复用)')
await db.close()
})
test('lease: release 也带 epoch 校验 —— 老持有者不能清掉新持有者的归属', async () => {
const db = await freshDb()
const a = shortLease(db, 'w-a')
await a.acquire('u1')
await sleep(90)
const b = shortLease(db, 'w-b')
await b.acquire('u1')
// a 试图释放(它本地已失权 ⇒ release 返回 false 且不动 DB)
assert.equal(await a.release('u1'), false)
const inst = await db.findUserInstance('u1', 'main')
assert.equal(inst.hostId, 'w-b', '新持有者的归属不能被误清')
await db.close()
})
test('lease: stillHolder 判据(hostId + epoch 双匹配)', async () => {
const db = await freshDb()
const a = new InstanceLease(db, 'w-a', { ttlMs: 5000, renewMs: 1000 })
await a.acquire('u1')
const inst = await db.findUserInstance('u1', 'main')
assert.equal(stillHolder(inst, 'w-a', 1), true)
assert.equal(stillHolder(inst, 'w-a', 2), false, 'epoch 不符 = 已失权')
assert.equal(stillHolder(inst, 'w-b', 1), false, '换了机器 = 已失权')
assert.equal(stillHolder(undefined, 'w-a', 1), false)
await db.close()
})
test('lease: 对账视图 —— mine() 只回本机、expiredAll() 回全局过期', async () => {
const db = await freshDb()
await db.createUser({ id: 'u2', username: 'bob', passHash: 'x', role: 'active', homeDir: '/tmp/u2' })
const a = shortLease(db, 'w-a')
const b = shortLease(db, 'w-b')
await a.acquire('u1')
await b.acquire('u2')
assert.deepEqual((await a.mine()).map((i) => i.userId), ['u1'], '一次拿回整机(本机只有 u1)')
assert.deepEqual((await b.mine()).map((i) => i.userId), ['u2'])
assert.equal((await a.expiredAll()).length, 0, '刚认领未过期')
await sleep(90)
assert.equal((await a.expiredAll()).length, 2, '过期后两台都进清单(供巡检,不自动接管)')
assert.equal((await a.expiredHere()).length, 1, 'expiredHere 只回自己名下')
await db.close()
})
test('lease: renewAll 回传失权清单(调用方据此 self-fence)', async () => {
const db = await freshDb()
const a = shortLease(db, 'w-a')
await a.acquire('u1')
assert.deepEqual(await a.renewAll(), [], '正常时无人失权')
await sleep(90)
const b = shortLease(db, 'w-b')
await b.acquire('u1') // 抢走
assert.deepEqual(await a.renewAll(), ['u1'], 'a 必须知道自己已失权')
await db.close()
})