初始提交:DSH 多租户平台(dshs)
This commit is contained in:
commit
43976fea6a
167 files changed
+24456
No files matched your search
@@ -0,0 +1,115 @@
|
||||
/**
|
||||
* 档案 20 · 崩溃退避与熔断策略单测(纯函数,不 spawn 任何进程)。
|
||||
* 运行:node --test test/crash-policy.test.mjs(含在 npm test 中)
|
||||
*/
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
|
||||
import {
|
||||
backoffDelayMs,
|
||||
breakerActive,
|
||||
breakerCooldownMs,
|
||||
breakerUntil,
|
||||
decideCrashAction,
|
||||
openBreaker,
|
||||
pruneHistory,
|
||||
} from '../lib/supervisor/crash-policy.js'
|
||||
|
||||
const CFG = { baseDelayMs: 1000, maxDelayMs: 30000, windowMs: 600000, maxRestartsInWindow: 5, stableResetMs: 60000 }
|
||||
|
||||
test('backoffDelayMs: 指数退避并在上限处封顶', () => {
|
||||
assert.equal(backoffDelayMs(0, CFG), 1000)
|
||||
assert.equal(backoffDelayMs(1, CFG), 2000)
|
||||
assert.equal(backoffDelayMs(2, CFG), 4000)
|
||||
assert.equal(backoffDelayMs(3, CFG), 8000)
|
||||
assert.equal(backoffDelayMs(4, CFG), 16000)
|
||||
assert.equal(backoffDelayMs(5, CFG), 30000, '超过上限应封顶到 maxDelayMs')
|
||||
assert.equal(backoffDelayMs(50, CFG), 30000, '极大值也不溢出')
|
||||
})
|
||||
|
||||
test('pruneHistory: 丢弃窗口外的时间戳', () => {
|
||||
const now = 1_000_000
|
||||
assert.deepEqual(pruneHistory([now - 700_000, now - 100_000, now - 1_000], now, CFG.windowMs), [
|
||||
now - 100_000,
|
||||
now - 1_000,
|
||||
])
|
||||
})
|
||||
|
||||
test('decideCrashAction: 窗口内未超限 → restart(带退避与尝试号)', () => {
|
||||
const now = 1_000_000
|
||||
const d = decideCrashAction([now - 5_000], 0, now, CFG)
|
||||
assert.equal(d.action, 'restart')
|
||||
assert.equal(d.delayMs, 1000)
|
||||
assert.equal(d.attempt, 1)
|
||||
assert.equal(d.windowRestarts, 2)
|
||||
})
|
||||
|
||||
test('decideCrashAction: 达到窗口上限 → circuit-open(熔断)', () => {
|
||||
const now = 1_000_000
|
||||
const history = [now - 50_000, now - 40_000, now - 30_000, now - 20_000, now - 10_000] // 恰好 5 次
|
||||
const d = decideCrashAction(history, 5, now, CFG)
|
||||
assert.equal(d.action, 'circuit-open')
|
||||
assert.equal(d.windowRestarts, 5)
|
||||
})
|
||||
|
||||
test('decideCrashAction: 窗口外的历史不计入熔断', () => {
|
||||
const now = 1_000_000
|
||||
const history = [now - 900_000, now - 800_000, now - 700_000, now - 650_000, now - 610_000] // 全部 > windowMs
|
||||
const d = decideCrashAction(history, 5, now, CFG)
|
||||
assert.equal(d.action, 'restart', '窗口外的旧崩溃不应触发熔断')
|
||||
assert.equal(d.windowRestarts, 1)
|
||||
})
|
||||
|
||||
test('decideCrashAction: 稳定后 streak 归零 → 退避回到 base', () => {
|
||||
const now = 2_000_000
|
||||
// 历史上有 3 次(窗口内),但 streak=0(已稳定过),退避应回到 baseDelayMs
|
||||
const history = [now - 300_000, now - 200_000, now - 100_000]
|
||||
const d = decideCrashAction(history, 0, now, CFG)
|
||||
assert.equal(d.action, 'restart')
|
||||
assert.equal(d.delayMs, 1000)
|
||||
assert.equal(d.windowRestarts, 4)
|
||||
})
|
||||
|
||||
test('decideCrashAction: 第 5 次(窗口内已有 4 次)仍允许重启,第 6 次熔断', () => {
|
||||
const now = 3_000_000
|
||||
const four = [now - 4000, now - 3000, now - 2000, now - 1000]
|
||||
const d5 = decideCrashAction(four, 4, now, CFG)
|
||||
assert.equal(d5.action, 'restart')
|
||||
assert.equal(d5.windowRestarts, 5)
|
||||
const five = [...four, now]
|
||||
const d6 = decideCrashAction(five, 5, now, CFG)
|
||||
assert.equal(d6.action, 'circuit-open')
|
||||
})
|
||||
|
||||
/* ── 档案 78:熔断冷却(防「崩溃循环可无限重来」)────────────────────────── */
|
||||
|
||||
const BCFG = { baseCooldownMs: 600000, maxCooldownMs: 21600000 }
|
||||
|
||||
test('档案 78 breakerCooldownMs: 指数加长并在上限封顶', () => {
|
||||
assert.equal(breakerCooldownMs(1, BCFG), 600000)
|
||||
assert.equal(breakerCooldownMs(2, BCFG), 1200000)
|
||||
assert.equal(breakerCooldownMs(3, BCFG), 2400000)
|
||||
assert.equal(breakerCooldownMs(99, BCFG), 21600000)
|
||||
})
|
||||
|
||||
test('档案 78 openBreaker/breakerActive/breakerUntil: opens 递增、冷却期内 active', () => {
|
||||
const t0 = 1_000_000
|
||||
const b1 = openBreaker(undefined, t0)
|
||||
assert.deepEqual(b1, { openedAt: t0, opens: 1 })
|
||||
assert.equal(breakerActive(b1, t0 + 599_999, BCFG), true)
|
||||
assert.equal(breakerActive(b1, t0 + 600_000, BCFG), false)
|
||||
assert.equal(breakerUntil(b1, BCFG), t0 + 600000)
|
||||
const b2 = openBreaker(b1, t0 + 600_000)
|
||||
assert.equal(b2.opens, 2)
|
||||
assert.equal(breakerUntil(b2, BCFG), t0 + 600_000 + 1_200_000)
|
||||
assert.equal(breakerActive(undefined, t0, BCFG), false)
|
||||
})
|
||||
|
||||
test('档案 78 回归:熔断后冷却期内应拒绝、冷却过后才放行一次预算', () => {
|
||||
const now = 5_000_000
|
||||
const history = [now - 5000, now - 4000, now - 3000, now - 2000, now - 1000]
|
||||
assert.equal(decideCrashAction(history, 5, now, CFG).action, 'circuit-open')
|
||||
const b = openBreaker(undefined, now)
|
||||
assert.equal(breakerActive(b, now + 1000, BCFG), true) // 冷却期内 → 拒绝重来
|
||||
assert.equal(breakerActive(b, now + 600_000, BCFG), false) // 冷却过后 → 放行一次干净预算
|
||||
})
|
||||
@@ -0,0 +1,214 @@
|
||||
// DB adapter regression tests (node:test). Runs against the built `lib/`
|
||||
// output — `npm test` builds first. Covers the constraint-mapping and
|
||||
// transaction semantics shared by both backends:
|
||||
// - unique / foreign-key errors converge on UniqueViolationError /
|
||||
// ForeignKeyViolationError
|
||||
// - the "disable-all-then-enable-one" credential upsert keeps exactly one
|
||||
// enabled key under concurrent writes
|
||||
// - concurrent createUser / audit writes land cleanly
|
||||
// The Postgres suite self-skips unless DSHS_TEST_DB_URL is set
|
||||
// (mirrors the CI e2e self-skip convention).
|
||||
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { SqliteAdapter } from '../lib/db/sqlite.js'
|
||||
import { openPgAdapter } from '../lib/db/pg.js'
|
||||
import { ForeignKeyViolationError, UniqueViolationError } from '../lib/db/errors.js'
|
||||
|
||||
const user = (id, username) => ({ id, username, passHash: 'x', role: 'active', homeDir: `/home/${username}` })
|
||||
|
||||
function register(backend, makeAdapter) {
|
||||
test(`${backend}: duplicate username → UniqueViolationError`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
await assert.rejects(() => db.createUser(user('b', 'alice')), UniqueViolationError)
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: session for unknown user → ForeignKeyViolationError`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => db.createSession({ tokenHash: 't', userId: 'missing', expiresAt: Date.now() + 1000 }),
|
||||
ForeignKeyViolationError,
|
||||
)
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: concurrent setCredentialKey keeps exactly one enabled`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
await Promise.all(
|
||||
Array.from({ length: 5 }, (_, i) => db.setCredentialKey('a', `k${i}`, `ref${i}`)),
|
||||
)
|
||||
const keys = await db.listCredentialKeys('a')
|
||||
assert.equal(keys.filter((k) => k.enabled).length, 1, 'exactly one key enabled')
|
||||
const ref = await db.getEnabledCredentialKeyRef('a')
|
||||
assert.ok(ref !== null && ref.startsWith('ref'), 'enabled key has a ref')
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: selectCredentialKey flips the enabled key`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
const k1 = await db.setCredentialKey('a', 'k1', 'r1')
|
||||
await db.setCredentialKey('a', 'k2', 'r2')
|
||||
assert.equal(await db.getEnabledCredentialKeyRef('a'), 'r2')
|
||||
assert.equal(await db.selectCredentialKey('a', k1.id), true)
|
||||
assert.equal(await db.getEnabledCredentialKeyRef('a'), 'r1')
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: concurrent createUser`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await Promise.all(Array.from({ length: 20 }, (_, i) => db.createUser(user(`u${i}`, `user${i}`))))
|
||||
assert.equal((await db.listPublicUsers()).length, 20)
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: getOrCreateWorkspace is idempotent`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
const w1 = await db.getOrCreateWorkspace('a', 'proj/one')
|
||||
const w2 = await db.getOrCreateWorkspace('a', 'proj/one')
|
||||
assert.equal(w1.id, w2.id)
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: createUser assigns a unique uid`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
const a = await db.createUser(user('a', 'alice'))
|
||||
const b = await db.createUser(user('b', 'bob'))
|
||||
assert.ok(a.uid !== null && a.uid !== undefined, 'first user has a uid')
|
||||
assert.notEqual(a.uid, b.uid, 'uids are unique across users')
|
||||
assert.equal((await db.listUsersWithoutUid()).length, 0, 'no unassigned uids remain')
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: concurrent audit writes`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await Promise.all(Array.from({ length: 10 }, (_, i) => db.audit('system', 'test', `detail-${i}`)))
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: upsertInstance round-trips folder + patch and is idempotent`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
await db.upsertInstance({ id: 'dsh-a', userId: 'a', role: 'main', status: 'starting', folder: '/ws/proj', patch: '- insert:\n' })
|
||||
const first = await db.findInstance('dsh-a')
|
||||
assert.equal(first.folder, '/ws/proj')
|
||||
assert.equal(first.patch, '- insert:\n')
|
||||
assert.equal(first.status, 'starting')
|
||||
assert.ok(first.startedAt > 0, 'started_at stamped')
|
||||
|
||||
// Same deterministic id → overwrite, not a duplicate row.
|
||||
await db.upsertInstance({ id: 'dsh-a', userId: 'a', role: 'main', status: 'running', folder: '/ws/other' })
|
||||
const second = await db.findInstance('dsh-a')
|
||||
assert.equal(second.folder, '/ws/other')
|
||||
assert.equal(second.patch, null, 'omitted patch clears the column')
|
||||
assert.equal((await db.listInstancesByRole('main')).filter((i) => i.userId === 'a').length, 1)
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: setInstanceStatus records the exit outcome`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
await db.upsertInstance({ id: 'dsh-a', userId: 'a', role: 'main', status: 'running', folder: '/ws' })
|
||||
assert.equal(await db.setInstanceStatus('dsh-a', 'crashed', { exitCode: 137, lastError: 'OOMKilled' }), true)
|
||||
const row = await db.findInstance('dsh-a')
|
||||
assert.equal(row.status, 'crashed')
|
||||
assert.equal(row.exitCode, 137)
|
||||
assert.equal(row.lastError, 'OOMKilled')
|
||||
assert.ok(row.lastExit > 0, 'last_exit stamped')
|
||||
assert.equal(await db.setInstanceStatus('dsh-missing', 'stopped'), false, 'unknown id reports no change')
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: instances are scoped by user and role, and cascade on user delete`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
await db.createUser(user('b', 'bob'))
|
||||
await db.upsertInstance({ id: 'dsh-a', userId: 'a', role: 'main', status: 'running', folder: '/ws' })
|
||||
await db.upsertInstance({ id: 'dsh-a-watchdog', userId: 'a', role: 'watchdog', status: 'starting' })
|
||||
await db.upsertInstance({ id: 'dsh-b', userId: 'b', role: 'main', status: 'running', folder: '/ws' })
|
||||
|
||||
assert.equal((await db.findUserInstance('a', 'main')).id, 'dsh-a')
|
||||
assert.equal((await db.findUserInstance('a', 'watchdog')).id, 'dsh-a-watchdog')
|
||||
assert.equal((await db.listInstancesByRole('main')).length, 2)
|
||||
assert.equal((await db.listInstancesByRole('watchdog')).length, 1)
|
||||
|
||||
await db.deleteUserInstances('a')
|
||||
assert.equal(await db.findUserInstance('a', 'main'), undefined)
|
||||
assert.equal((await db.listInstancesByRole('main')).length, 1, "b's instance survives")
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: hasActiveSession reflects unexpired vs expired sessions`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await db.createUser(user('a', 'alice'))
|
||||
assert.equal(await db.hasActiveSession('a'), false, 'no session → inactive')
|
||||
await db.createSession({ tokenHash: 't1', userId: 'a', expiresAt: Date.now() + 60_000 })
|
||||
assert.equal(await db.hasActiveSession('a'), true, 'unexpired session → active')
|
||||
await db.createSession({ tokenHash: 't2', userId: 'a', expiresAt: Date.now() - 1000 })
|
||||
assert.equal(await db.hasActiveSession('a'), true, 'at least one unexpired session → active')
|
||||
await db.deleteSession('t1')
|
||||
assert.equal(await db.hasActiveSession('a'), false, 'only expired session left → inactive')
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
|
||||
test(`${backend}: instance for unknown user → ForeignKeyViolationError`, async () => {
|
||||
const db = await makeAdapter()
|
||||
try {
|
||||
await assert.rejects(
|
||||
() => db.upsertInstance({ id: 'dsh-ghost', userId: 'missing', role: 'main', status: 'starting' }),
|
||||
ForeignKeyViolationError,
|
||||
)
|
||||
} finally {
|
||||
await db.close()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
register('sqlite', () => new SqliteAdapter(':memory:', 100000))
|
||||
|
||||
const pgUrl = process.env.DSHS_TEST_DB_URL
|
||||
if (pgUrl) {
|
||||
register('pg', () => openPgAdapter(pgUrl, 100000))
|
||||
} else {
|
||||
test('pg: skipped — set DSHS_TEST_DB_URL to run', { skip: 'no DSHS_TEST_DB_URL' }, () => {})
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
// K8sSpawner unit tests against fake API clients. Covers the six defects fixed
|
||||
// in Stage 4 without needing a cluster: 404-vs-500 error handling, full
|
||||
// teardown of every generated object, idempotent Secret/ConfigMap, and the
|
||||
// file-sidecar Pod + init container shape.
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { K8sSpawner } from '../lib/supervisor/k8s-spawner.js'
|
||||
import { resolveConfig } from '../lib/config.js'
|
||||
import { AlreadyRunningError } from '../lib/supervisor/spawner.js'
|
||||
import { FILE_SERVICE_PORT } from '../lib/web/file-service.js'
|
||||
|
||||
function notFound() {
|
||||
const err = new Error('not found')
|
||||
err.code = 404
|
||||
return err
|
||||
}
|
||||
function conflict() {
|
||||
const err = new Error('already exists')
|
||||
err.code = 409
|
||||
return err
|
||||
}
|
||||
|
||||
/** A tiny in-memory fake for the K8s API surface K8sSpawner touches. */
|
||||
function makeClients() {
|
||||
const pods = new Map()
|
||||
const services = new Map()
|
||||
const secrets = new Map()
|
||||
const configMaps = new Map()
|
||||
const policies = new Map()
|
||||
const jobs = new Map()
|
||||
const deletes = []
|
||||
|
||||
return {
|
||||
pods, services, secrets, configMaps, policies, jobs, deletes,
|
||||
core: {
|
||||
async readNamespacedPod({ name }) { return pods.get(name) ?? (() => { throw notFound() })() },
|
||||
async createNamespacedPod({ body }) {
|
||||
// Stored pods report Ready so `waitForPodReady` returns immediately.
|
||||
const pod = { ...body, status: { phase: 'Running', conditions: [{ type: 'Ready', status: 'True' }] } }
|
||||
pods.set(body.metadata.name, pod)
|
||||
return pod
|
||||
},
|
||||
async deleteNamespacedPod({ name }) { deletes.push(`pod:${name}`); pods.delete(name) },
|
||||
async readNamespacedSecret({ name }) { return secrets.get(name) ?? (() => { throw notFound() })() },
|
||||
async createNamespacedSecret({ body }) { secrets.set(body.metadata.name, body); return body },
|
||||
async deleteNamespacedSecret({ name }) { deletes.push(`secret:${name}`); secrets.delete(name) },
|
||||
async readNamespacedConfigMap({ name }) { return configMaps.get(name) ?? (() => { throw notFound() })() },
|
||||
async createNamespacedConfigMap({ body }) { configMaps.set(body.metadata.name, body); return body },
|
||||
async deleteNamespacedConfigMap({ name }) { deletes.push(`configmap:${name}`); configMaps.delete(name) },
|
||||
async readNamespacedService({ name }) { return services.get(name) ?? (() => { throw notFound() })() },
|
||||
async createNamespacedService({ body }) { services.set(body.metadata.name, body); return body },
|
||||
async deleteNamespacedService({ name }) { deletes.push(`service:${name}`); services.delete(name) },
|
||||
},
|
||||
networking: {
|
||||
async readNamespacedNetworkPolicy({ name }) { return policies.get(name) ?? (() => { throw notFound() })() },
|
||||
async createNamespacedNetworkPolicy({ body }) { policies.set(body.metadata.name, body); return body },
|
||||
async deleteNamespacedNetworkPolicy({ name }) { deletes.push(`np:${name}`); policies.delete(name) },
|
||||
},
|
||||
batch: {
|
||||
async createNamespacedJob({ body }) { jobs.set(body.metadata.name, body); return body },
|
||||
async deleteNamespacedJob({ name }) { deletes.push(`job:${name}`); jobs.delete(name) },
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
function spawner(clients) {
|
||||
const config = resolveConfig({ deployMode: 'k8s', controlPlaneImage: 'acr/cp:tag', dshImage: 'acr/dsh:tag' })
|
||||
const db = { findUserInstance: async () => undefined }
|
||||
return new K8sSpawner(config, db, async () => 'sk-test', async () => 100042, clients)
|
||||
}
|
||||
|
||||
test('k8s: launch creates DSH Pod/Service/NP and calls ensureFileService first', async () => {
|
||||
const clients = makeClients()
|
||||
const s = spawner(clients)
|
||||
const inst = await s.launch('u1', '/ws/proj')
|
||||
assert.equal(inst.id, 'dsh-u1')
|
||||
assert.ok(clients.pods.has('dsh-files-u1'), 'files Pod created first')
|
||||
assert.ok(clients.pods.has('dsh-u1'), 'dsh Pod created')
|
||||
assert.ok(clients.services.has('dsh-u1'), 'dsh Service created')
|
||||
assert.ok(clients.policies.has('dsh-u1'), 'dsh NetworkPolicy created')
|
||||
|
||||
const filesPod = clients.pods.get('dsh-files-u1')
|
||||
assert.equal(filesPod.spec.containers[0].args[0], 'file-service', 'files container runs the file-service subcommand')
|
||||
assert.equal(filesPod.spec.initContainers.length, 1, 'files Pod carries the user-dir init container')
|
||||
const filesMounts = filesPod.spec.containers[0].volumeMounts
|
||||
assert.equal(filesMounts.length, 2, 'files container mounts its subPath + /tmp')
|
||||
assert.equal(filesMounts[0].name, 'data')
|
||||
assert.equal(filesMounts[0].subPath, 'u1', 'files container mounts <pvc>/u1 via subPath')
|
||||
assert.equal(filesMounts[1].name, 'tmp', 'files container mounts emptyDir /tmp (read-only rootfs)')
|
||||
assert.equal(filesPod.spec.containers[0].securityContext.readOnlyRootFilesystem, true, 'files container rootfs is read-only')
|
||||
const initMount = filesPod.spec.initContainers[0].volumeMounts[0]
|
||||
assert.equal(initMount.name, 'data-root', 'init container mounts the PVC root (no subPath)')
|
||||
assert.equal(initMount.subPath, undefined, 'init container has no subPath')
|
||||
|
||||
await assert.rejects(() => s.launch('u1', '/ws/proj'), AlreadyRunningError)
|
||||
})
|
||||
|
||||
test('k8s: endpointFor returns the Pod IP (not Service DNS) for a Running Pod', async () => {
|
||||
const clients = makeClients()
|
||||
const s = spawner(clients)
|
||||
assert.equal(await s.endpointFor('u1'), undefined, 'absent Pod → undefined')
|
||||
clients.pods.set('dsh-u1', { status: { phase: 'Running', podIP: '10.42.0.7' } })
|
||||
assert.deepEqual(await s.endpointFor('u1'), { host: '10.42.0.7', port: 8081 })
|
||||
clients.pods.set('dsh-u1', { status: { phase: 'Pending' } })
|
||||
assert.equal(await s.endpointFor('u1'), undefined, 'non-Running Pod → undefined')
|
||||
})
|
||||
|
||||
test('k8s: stop deletes every generated object (Pod/Service/NP/Job/ConfigMap)', async () => {
|
||||
const clients = makeClients()
|
||||
const s = spawner(clients)
|
||||
await s.launch('u1', '/ws/proj', '- insert:\n')
|
||||
// Pre-seed the patch ConfigMap + watchdog Job as a prior launch would have.
|
||||
clients.configMaps.set('dsh-u1-patch', { metadata: { name: 'dsh-u1-patch' } })
|
||||
clients.jobs.set('dsh-u1-watchdog', { metadata: { name: 'dsh-u1-watchdog' } })
|
||||
|
||||
await s.stop('u1')
|
||||
const names = clients.deletes
|
||||
for (const expected of ['pod:dsh-u1', 'service:dsh-u1', 'np:dsh-u1', 'job:dsh-u1-watchdog', 'configmap:dsh-u1-patch']) {
|
||||
assert.ok(names.includes(expected), `stop deletes ${expected}`)
|
||||
}
|
||||
// stop() must NOT touch the files Pod (it lives independently of the DSH).
|
||||
assert.ok(clients.pods.has('dsh-files-u1'), 'files Pod survives a DSH stop')
|
||||
})
|
||||
|
||||
test('k8s: a 404 is swallowed, but 403/500 propagate', async () => {
|
||||
const clients = makeClients()
|
||||
const s = spawner(clients)
|
||||
// endpointFor must not mask a real failure as "not running".
|
||||
clients.core.readNamespacedPod = async () => { const e = new Error('forbidden'); e.code = 403; throw e }
|
||||
await assert.rejects(() => s.endpointFor('u1'), (err) => err.code === 403, '403 surfaces')
|
||||
clients.core.readNamespacedPod = async () => { throw notFound() }
|
||||
assert.equal(await s.endpointFor('u1'), undefined, '404 is "no Pod"')
|
||||
})
|
||||
|
||||
test('k8s: ensureSecret is idempotent (no 409 on relaunch)', async () => {
|
||||
const clients = makeClients()
|
||||
const s = spawner(clients)
|
||||
// First call: read → 404 → create.
|
||||
await s.ensureFileService('u1') // exercises the image gate, not the secret
|
||||
await s.launch('u1', '/ws')
|
||||
// Simulate a stale Secret still present after a stop that skipped it.
|
||||
clients.secrets.set('dsh-key-u1', { metadata: { name: 'dsh-key-u1' } })
|
||||
clients.pods.delete('dsh-u1')
|
||||
await s.launch('u1', '/ws') // read → present → no create, no throw
|
||||
assert.ok(clients.secrets.has('dsh-key-u1'), 'secret still present, no 409')
|
||||
})
|
||||
|
||||
test('k8s: ensureFileService fails loudly without a control-plane image', async () => {
|
||||
const config = resolveConfig({ deployMode: 'k8s', controlPlaneImage: '', dshImage: 'acr/dsh:tag' })
|
||||
const s = new K8sSpawner(config, { findUserInstance: async () => undefined }, async () => null, async () => 100042, makeClients())
|
||||
await assert.rejects(() => s.ensureFileService('u1'), /CONTROL_PLANE_IMAGE/, 'missing image is a hard failure')
|
||||
})
|
||||
|
||||
test('k8s: files NetworkPolicy only allows the control plane on the file port', async () => {
|
||||
const clients = makeClients()
|
||||
const s = spawner(clients)
|
||||
await s.ensureFileService('u1')
|
||||
const np = clients.policies.get('dsh-files-u1')
|
||||
assert.equal(np.spec.ingress[0]._from[0].podSelector.matchLabels.app, 'dsh-orchestrator')
|
||||
assert.equal(np.spec.ingress[0].ports[0].port, FILE_SERVICE_PORT)
|
||||
// No 443 egress for the files sidecar — it never reaches the LLM API.
|
||||
assert.equal(np.spec.egress.some((rule) => (rule.ports ?? []).some((p) => p.port === 443)), false)
|
||||
})
|
||||
@@ -0,0 +1,117 @@
|
||||
// Leader election + reconcile planning, without a cluster. `planReconcile` is
|
||||
// pure; `LeaderElector` is driven through a fake CoordinationV1Api and a fake
|
||||
// clock so the acquire / back-off / take-over state machine is deterministic.
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { LeaderElector } from '../lib/supervisor/leader.js'
|
||||
import { planReconcile } from '../lib/supervisor/reconcile.js'
|
||||
|
||||
// The real client deserializes V1MicroTime back to an ISO *string*, so fake the
|
||||
// same shape here to exercise the normalize-on-read path.
|
||||
const lease = (holder, renewMs, transitions = 0, rv = '1') => ({
|
||||
metadata: { resourceVersion: rv },
|
||||
spec: {
|
||||
holderIdentity: holder,
|
||||
renewTime: new Date(renewMs).toISOString(),
|
||||
leaseTransitions: transitions,
|
||||
},
|
||||
})
|
||||
|
||||
test('reconcile: relaunch desired mains with a missing/stopped Pod, delete orphans', () => {
|
||||
const desired = [
|
||||
{ id: 'dsh-a', userId: 'a', role: 'main', folder: '/ws', patch: null },
|
||||
{ id: 'dsh-b', userId: 'b', role: 'main', folder: '/ws', patch: null },
|
||||
]
|
||||
const live = [
|
||||
{ name: 'dsh-a', userId: 'a', running: true, crashed: false },
|
||||
{ name: 'dsh-orphan', userId: 'x', running: true, crashed: false },
|
||||
]
|
||||
const plan = planReconcile(desired, live)
|
||||
assert.deepEqual(plan.launch.map((i) => i.userId), ['b'], 'missing main is relaunched')
|
||||
assert.deepEqual(plan.delete.map((p) => p.userId), ['x'], 'orphan Pod is deleted')
|
||||
})
|
||||
|
||||
test('reconcile: a crashed (non-running) Pod still counts as absent', () => {
|
||||
const plan = planReconcile(
|
||||
[{ id: 'dsh-a', userId: 'a', role: 'main', folder: '/ws', patch: null }],
|
||||
[{ name: 'dsh-a', userId: 'a', running: false, crashed: true }],
|
||||
)
|
||||
assert.equal(plan.launch.length, 1, 'crashed Pod → relaunch')
|
||||
assert.equal(plan.delete.length, 0)
|
||||
})
|
||||
|
||||
test('leader: first candidate creates the lease and leads', async () => {
|
||||
let created = false
|
||||
let patched = []
|
||||
const coordination = {
|
||||
async createNamespacedLease({ body }) {
|
||||
created = true
|
||||
return body
|
||||
},
|
||||
async readNamespacedLease() { throw Object.assign(new Error('nf'), { code: 404 }) },
|
||||
async replaceNamespacedLease({ body }) { patched.push(body) },
|
||||
}
|
||||
const started = []
|
||||
const elector = new LeaderElector({ namespace: 'dsh', identity: 'pod-1', coordination, now: () => 1_000_000 })
|
||||
elector.setLeadershipCallbacks((f) => started.push(f))
|
||||
await elector.start()
|
||||
assert.equal(created, true, 'first candidate creates the lease')
|
||||
assert.equal(elector.isLeader, true)
|
||||
assert.equal(started[0].holder, 'pod-1')
|
||||
assert.equal(started[0].operationId, 0)
|
||||
elector.stop()
|
||||
})
|
||||
|
||||
test('leader: a second candidate backs off while a live holder leads', async () => {
|
||||
const coordination = {
|
||||
async createNamespacedLease() { throw Object.assign(new Error('exists'), { code: 409 }) },
|
||||
async readNamespacedLease() { return lease('pod-1', 10_000_000) }, // renew 1s ago, live
|
||||
async replaceNamespacedLease() { throw new Error('should not replace') },
|
||||
}
|
||||
const elector = new LeaderElector({
|
||||
namespace: 'dsh',
|
||||
identity: 'pod-2',
|
||||
coordination,
|
||||
now: () => 11_000_000, // 1s after the holder's renew, still within 15s
|
||||
})
|
||||
await elector.start()
|
||||
assert.equal(elector.isLeader, false, 'live holder → back off')
|
||||
elector.stop()
|
||||
})
|
||||
|
||||
test('leader: a stale lease is taken over with a bumped transition', async () => {
|
||||
let patched
|
||||
const coordination = {
|
||||
async createNamespacedLease() { throw Object.assign(new Error('exists'), { code: 409 }) },
|
||||
async readNamespacedLease() { return lease('pod-1', 10_000_000, 3, 'rv-9') }, // renew 20s ago, expired
|
||||
async replaceNamespacedLease({ body }) { patched = body },
|
||||
}
|
||||
const started = []
|
||||
const elector = new LeaderElector({
|
||||
namespace: 'dsh',
|
||||
identity: 'pod-2',
|
||||
coordination,
|
||||
now: () => 30_000_000,
|
||||
})
|
||||
elector.setLeadershipCallbacks((f) => started.push(f))
|
||||
await elector.start()
|
||||
assert.equal(elector.isLeader, true, 'expired lease → take over')
|
||||
assert.equal(patched.metadata.resourceVersion, 'rv-9', 'CAS on the read resourceVersion')
|
||||
assert.equal(patched.spec.leaseTransitions, 4, 'transition bumps')
|
||||
assert.equal(started[0].operationId, 4, 'fencing token carries the new transition')
|
||||
elector.stop()
|
||||
})
|
||||
|
||||
test('leader: re-acquiring our own lease renews rather than bumps', async () => {
|
||||
let patched
|
||||
const coordination = {
|
||||
async createNamespacedLease() { throw Object.assign(new Error('exists'), { code: 409 }) },
|
||||
async readNamespacedLease() { return lease('pod-1', 30_000_000, 0, 'rv-2') },
|
||||
async replaceNamespacedLease({ body }) { patched = body },
|
||||
}
|
||||
const elector = new LeaderElector({ namespace: 'dsh', identity: 'pod-1', coordination, now: () => 30_000_000 })
|
||||
await elector.start()
|
||||
assert.equal(elector.isLeader, true)
|
||||
assert.equal(patched.spec.leaseTransitions, 0, 'renewing preserves transitions (does not bump)')
|
||||
elector.stop()
|
||||
})
|
||||
@@ -0,0 +1,120 @@
|
||||
// LocalUserFs regression tests (node:test, against built lib/ — `npm test`
|
||||
// builds first). Focus: the symlink-escape guard layered on top of lexical
|
||||
// path containment. A link planted inside the workspace (`ws/link -> /etc`)
|
||||
// passes the prefix check, so every operation must reject symlink components
|
||||
// before touching the filesystem.
|
||||
//
|
||||
// Symlink creation is privilege-gated on Windows (junctions work unprivileged,
|
||||
// file links need dev mode), so the link-based cases self-skip when the FS
|
||||
// refuses to create one.
|
||||
|
||||
import { test } from 'node:test'
|
||||
import assert from 'node:assert/strict'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { LocalUserFs } from '../lib/fs/local-user-fs.js'
|
||||
import { UserFsError } from '../lib/fs/user-fs.js'
|
||||
|
||||
const USER = 'u1'
|
||||
|
||||
function makeUser(t) {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), 'dsh-lufs-'))
|
||||
t.after(() => rmSync(dataRoot, { recursive: true, force: true }))
|
||||
const ws = join(dataRoot, 'users', USER, 'ws')
|
||||
mkdirSync(ws, { recursive: true })
|
||||
return { fs: new LocalUserFs((id) => join(dataRoot, 'users', id)), dataRoot, ws }
|
||||
}
|
||||
|
||||
/** Create a link; false when the platform refuses (e.g. unprivileged Windows). */
|
||||
function tryLink(target, path, type) {
|
||||
try {
|
||||
symlinkSync(target, path, type)
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function isBadPath(err) {
|
||||
return err instanceof UserFsError && err.code === 'bad_path'
|
||||
}
|
||||
|
||||
test('upload/mkdir/listDir work through nested directories', async (t) => {
|
||||
const { fs } = makeUser(t)
|
||||
await fs.mkdir(USER, 'proj')
|
||||
assert.equal(await fs.upload(USER, 'proj', 'hello.txt', Buffer.from('hi')), 'hello.txt')
|
||||
const entries = await fs.listDir(USER, 'proj')
|
||||
assert.deepEqual(entries.map((e) => e.name).sort(), ['hello.txt'])
|
||||
})
|
||||
|
||||
test('lexical traversal is still rejected with bad_path', async (t) => {
|
||||
const { fs } = makeUser(t)
|
||||
await assert.rejects(() => fs.upload(USER, '../outside', 'x.txt', Buffer.from('x')), isBadPath)
|
||||
await assert.rejects(() => fs.listDir(USER, '../../etc'), isBadPath)
|
||||
})
|
||||
|
||||
test('upload through an in-workspace directory symlink is rejected', async (t) => {
|
||||
const { fs, dataRoot, ws } = makeUser(t)
|
||||
const outside = join(dataRoot, 'outside')
|
||||
mkdirSync(outside)
|
||||
const type = process.platform === 'win32' ? 'junction' : 'dir'
|
||||
if (!tryLink(outside, join(ws, 'link'), type)) {
|
||||
t.skip('symlink creation unavailable on this host')
|
||||
return
|
||||
}
|
||||
await assert.rejects(() => fs.upload(USER, 'link', 'escaped.txt', Buffer.from('x')), isBadPath)
|
||||
await assert.rejects(() => fs.mkdir(USER, 'link/sub'), isBadPath)
|
||||
await assert.rejects(() => fs.listDir(USER, 'link'), isBadPath)
|
||||
await assert.rejects(() => fs.isDirectory(USER, 'link'), isBadPath)
|
||||
})
|
||||
|
||||
test('upload onto an existing symlinked filename does not follow it', async (t) => {
|
||||
const { fs, ws } = makeUser(t)
|
||||
writeFileSync(join(ws, 'real.txt'), 'original')
|
||||
if (!tryLink(join(ws, 'real.txt'), join(ws, 'alias.txt'), 'file')) {
|
||||
t.skip('symlink creation unavailable on this host')
|
||||
return
|
||||
}
|
||||
await assert.rejects(() => fs.upload(USER, '', 'alias.txt', Buffer.from('evil')), isBadPath)
|
||||
assert.equal(readFileSync(join(ws, 'real.txt'), 'utf8'), 'original', 'target untouched')
|
||||
})
|
||||
|
||||
test('missing path components end the walk and surface as not_found', async (t) => {
|
||||
const { fs } = makeUser(t)
|
||||
await assert.rejects(
|
||||
() => fs.isDirectory(USER, 'ghost/deep'),
|
||||
(err) => err instanceof UserFsError && err.code === 'not_found',
|
||||
)
|
||||
})
|
||||
|
||||
|
||||
// ─── 档案 56:readFile(门户/实例页「我的文件」下载) ───────────────
|
||||
test("readFile: 返回文件名与内容", async (t) => {
|
||||
const { fs, ws } = makeUser(t)
|
||||
writeFileSync(join(ws, "报告.md"), "# 内容\n")
|
||||
const out = await fs.readFile(USER, "报告.md")
|
||||
assert.equal(out.name, "报告.md")
|
||||
assert.equal(out.data.toString("utf8"), "# 内容\n")
|
||||
})
|
||||
|
||||
test("readFile: 目录→not_a_file;缺失→not_found;超限→too_large", async (t) => {
|
||||
const { fs, ws } = makeUser(t)
|
||||
mkdirSync(join(ws, "dir"))
|
||||
await assert.rejects(() => fs.readFile(USER, "dir"), (e) => e instanceof UserFsError && e.code === "not_a_file")
|
||||
await assert.rejects(() => fs.readFile(USER, "ghost.txt"), (e) => e instanceof UserFsError && e.code === "not_found")
|
||||
writeFileSync(join(ws, "big.bin"), Buffer.alloc(4096))
|
||||
await assert.rejects(() => fs.readFile(USER, "big.bin", 1024), (e) => e instanceof UserFsError && e.code === "too_large")
|
||||
})
|
||||
|
||||
test("readFile: 路径逃逸被拒", async (t) => {
|
||||
const { fs } = makeUser(t)
|
||||
await assert.rejects(() => fs.readFile(USER, "../outside.txt"), isBadPath)
|
||||
await assert.rejects(() => fs.readFile(USER, "../../etc/passwd"), isBadPath)
|
||||
})
|
||||
|
||||
test("readFile: 符号链接逃逸被拒(不跟随工作区内的链接)", async (t) => {
|
||||
const { fs, ws } = makeUser(t)
|
||||
if (!tryLink("/etc/passwd", join(ws, "link.txt"), "file")) return
|
||||
await assert.rejects(() => fs.readFile(USER, "link.txt"), isBadPath)
|
||||
})
|
||||
Reference in new issue
Block a user