feat(cluster): 集群化落地 —— Manager/Worker 拆分 + 归属租约 + 跨机验证(T08)

背景:把平台从「单机单进程」改造成「1 组 Manager + N 台 Worker + 共享归属状态」,
硬约束 = 全程兼容单例模式(deployMode 默认 local;生产切换前 47 一行未动)。

主要改动
1) 数据模型 v7(SQLite 与 PG 两方言同步):新增 dsh_hosts 注册表 +
   dsh_instances.{host_id,epoch,heartbeat_at,lease_until};claimInstance 原子抢占
   (UPDATE … WHERE host_id IS NULL OR lease_until < now)+ pinInstanceHost 钉住归属。
2) 租约与 fencing:src/supervisor/lease.ts(acquire/renew/release + stillHolder 判据 +
   ttl > 2×renew 硬校验);心跳里续租,失权即向 worker 下发更高 epoch(self-fencing)。
   ⚠️ release 只清租约(lease_until),**保留 host_id** —— host_id 是「用户数据在哪台」的锚点。
3) Worker agent(src/worker/agent.ts,子命令 dshs worker):实例生命周期 + 文件面 /fs/*
   + 幂等键(operationId)+ 鉴权(timingSafeEqual);Worker 不写控制面数据
   (apiKey/uid 由 Manager 随 launch 投递,R5 收窄)。
4) 远端 Spawner + LeasedSpawner:按 host 路由(**粘性优先**:有历史归属且那台 up 就留在原地,
   否则按容量选最空的)+ 容量准入 + deployMode=cluster 装配(systemd drop-in,可回滚)。
5) bwrap 修正:**所有挂载点的中间目录统一前置 + 去重 + 由外到内**(「就近创建」会在嵌套前缀下
   遮掉已绑挂载点 ⇒ bwrap: Can't chdir);且**只能用 --tmpfs**,用 --perms 会让 47 的
   bwrap 0.4.0 直接拒启动(沙箱全挂)。
6) 跨机隧道 src/worker/tunnel.ts:SSH ControlMaster + 动态 -R 转发;**自愈由 agent 本地
   20s 定时器驱动**(不能只放 /healthz —— 心跳本身经隧道进来,断了就没人触发它)。
7) 文件面按归属路由(RemoteUserFs):实例与文件必须落在同一台机器,否则实例看不到自己的文件。
8) 观测面:dshs doctor / dshs cluster status。

验证(本次均已实跑)
- test/lease.test.mjs:SQLite 10/10 == PG 10/10
- 组件级端到端 5 个:verify-cluster-{agent,lease,fs,migrate,live}.mjs
- 真跨机(47 Manager / 106 Worker,跨云 + 反向隧道)verify-cluster-cross.mjs 九步全绿
- 域名形态访问 verify-cluster-domain.mjs(<user>.域名 → Manager → 远端实例;越权 403)
- 冒烟 scripts/smoke-*:6/8,失败项与改动前基线完全相同(无回归)
- 生产切换与回滚剧本见 dsh-server-docs/交接单/T08-集群化落地-兼容单例模式.md §16
This commit is contained in:
admin committed 2026-09-15 18:47:02 +08:00
1 parent 68c0a320ed
commit c70d5d860e
47 files changed
+5367 -14

No files matched your search

+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env bash
# 切换后功能验证 v2(在 47 上跑)
# ① 既有用户 guest:应留 w-47,且**工作区有历史数据**(文件在 47 的盘上)
# ② 新用户:应落 w-106,且**建的文件真的出现在 106 的盘上**(文件面路由已修)
# 判据:实例页必须带 <base href="/"(门户页不算);归属看 PG;文件落盘看两台机器磁盘
set -uo pipefail
PG() { PGPASSWORD=dshs_cluster_2026 /usr/bin/psql -h 127.0.0.1 -p 15432 -U dshs -d dshs -tAc "$1"; }
M=http://127.0.0.1:3080
DOMAIN=alotbuy.com
T47=dshs-worker-47-c4b7e19f
T106=dshs-worker-7f3a91c05e
mksess() {
local u="$1" T H
T=$(openssl rand -hex 32); H=$(printf '%s' "$T" | sha256sum | cut -d' ' -f1)
PG "insert into sessions (token_hash,user_id,created_at,expires_at,ip,user_agent)
select '$H', id, (extract(epoch from now())*1000)::bigint, (extract(epoch from now())*1000)::bigint+1800000, '127.0.0.1','switch-verify' from users where username='$u'" >/dev/null
printf '%s' "$T"
}
owner() { PG "select u.username||' -> '||coalesce(i.host_id,'NULL')||' epoch='||i.epoch from users u left join dsh_instances i on i.user_id=u.id where u.username='$1'"; }
agent() { curl -s -m 6 -H "x-dsh-agent-token: $2" "http://127.0.0.1:$1/healthz" | grep -o '"instances":[0-9]*'; }
isapp() { grep -q '<base href=' <<<"$1" && echo "✓实例页" || echo "✗非实例页"; }
echo "############ ① 既有用户 guest(应留 w-47 + 工作区有数据) ############"
GT=$(mksess guest); GC="sid=$GT"
E=$(curl -s -m 60 -X POST -b "$GC" -H 'content-type: application/json' -d '{}' "$M/api/dsh/enter")
echo " enter → $(head -c 130 <<<"$E")"
sleep 3
echo " 归属: $(owner guest) ← 期望 w-47"
echo " w-47=$(agent 19100 $T47) w-106=$(agent 19000 $T106)"
echo " 工作区首项: $(curl -s -m 10 -b "$GC" "$M/api/desktop/tree" | head -c 150)"
URL=$(grep -o '"url":"[^"]*"' <<<"$E" | head -1 | cut -d'"' -f4)
PAGE=$(curl -s -m 25 -L -b "$GC" -H "Host: guest.$DOMAIN" "$M/" | head -c 300)
echo " 实例页: $(isapp "$PAGE")"
echo " 47 盘上 guest 工作区条目: $(ls /var/lib/dshs/users/4092b965-2f68-4977-9989-68b3966f7df0/ws 2>/dev/null | wc -l) 项(>0 = 数据在 47 ✓)"
echo
echo "############ ② 新用户(应落 w-106 + 文件真的写到 106 盘) ############"
NU="swtest$(date +%H%M%S)"
AT=$(mksess admin); AC="sid=$AT"
echo " register=$(curl -s -m 15 -X POST -H 'content-type: application/json' -d "{\"username\":\"$NU\",\"password\":\"SwitchTest123\"}" "$M/api/auth/register" -o /dev/null -w '%{http_code}')"
NU_ID=$(PG "select id from users where username='$NU'")
echo " approve=$(curl -s -m 15 -X POST -b "$AC" "$M/api/admin/users/$NU_ID/approve" -o /dev/null -w '%{http_code}')"
NC=$(curl -s -m 15 -X POST -H 'content-type: application/json' -d "{\"username\":\"$NU\",\"password\":\"SwitchTest123\"}" -D - "$M/api/auth/login" -o /dev/null | grep -i '^set-cookie' | head -1 | grep -oP 'sid=[^;]+')
echo " mkdir=$(curl -s -m 15 -X POST -b "$NC" -H 'content-type: application/json' -d '{"path":"proj"}' "$M/api/fs/mkdir" -o /dev/null -w '%{http_code}')"
echo " upload=$(curl -s -m 20 -X POST -b "$NC" -H 'content-type: application/json' -d "{\"path\":\"proj\",\"name\":\"hello.txt\",\"data\":\"$(printf 'hi-from-switch' | base64 -w0)\"}" "$M/api/fs/upload" -o /dev/null -w '%{http_code}')"
echo " launch=$(curl -s -m 60 -X POST -b "$NC" -H 'content-type: application/json' -d '{"folder":"proj"}' "$M/api/dsh/launch" -o /dev/null -w '%{http_code}')"
sleep 4
echo " 归属: $(owner "$NU") ← 期望 w-106"
echo " w-106=$(agent 19000 $T106) w-47=$(agent 19100 $T47)"
echo " --- 文件落盘取证(这才是文件面路由修好的证据) ---"
echo " 47 盘: $(ls /var/lib/dshs/users/$NU_ID/ws/proj/hello.txt 2>/dev/null && echo 存在 || echo '不存在 ✓(不应在 47)')"
echo " 106 盘: $(ssh -n -i /root/.ssh/dshworker_ed25519 -o BatchMode=yes -o StrictHostKeyChecking=accept-new [email protected] "ls /var/lib/dshs/users/$NU_ID/ws/proj/hello.txt 2>/dev/null" 2>/dev/null && echo 存在✓ || echo '不存在 ✗')"
NU_PAGE=$(curl -s -m 25 -L -b "$NC" -H "Host: $NU.$DOMAIN" "$M/" | head -c 300)
echo " 实例页(Host: $NU.$DOMAIN): $(isapp "$NU_PAGE")"
echo
echo "############ 收尾 ############"
curl -s -m 40 -X POST -b "$GC" "$M/api/dsh/stop" -o /dev/null -w " guest stop=%{http_code}\n"
curl -s -m 40 -X POST -b "$NC" "$M/api/dsh/stop" -o /dev/null -w " newuser stop=%{http_code}\n"
PG "delete from sessions where user_agent='switch-verify'" >/dev/null
echo " 残留临时 session: $(PG "select count(*) from sessions where user_agent='switch-verify'")(应 0)"
echo " 新用户记录: $NU / $NU_ID"