初始提交：DSH 多租户平台（dshs）

2026-09-13 16:18:10 +08:00
commit 43976fea6a
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dsh
@@ -0,0 +1,19 @@
+# CloudNativePG Cluster — 控制面数据层（Phase 2 测试：1 实例）。
+# 生产 HA 用 instances: 3（见 docs/k8s.md §4.5）。storageClass 用拓扑感知
+# WaitForFirstConsumer，避免 ESSD 云盘与 Pod 跨可用区挂载失败。
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: dsh-pg
+  namespace: dsh
+spec:
+  instances: 1
+  # ghcr.io 在阿里云被墙/极慢，Postgres 镜像走国内镜像源。
+  imageName: ghcr.m.daocloud.io/cloudnative-pg/postgresql:18.4-system-trixie
+  storage:
+    size: 20Gi
+    storageClass: alicloud-disk-topology-alltype
+  bootstrap:
+    initdb:
+      database: dsh
+      owner: dsh
@@ -0,0 +1,104 @@
+# 控制面 Deployment + Service（Phase 3：deployMode=k8s 起每用户 DSH Pod）。
+# 依赖：secret `dsh-pg`（key: url = Postgres DSN）、`dsh-secret`（key: key =
+# 共享加密密钥）、imagePullSecret `dsh-acr-pull`、ServiceAccount
+# `dsh-orchestrator`（deploy/03-rbac.yaml）。镜像由 CI push 到 ACR。
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: dsh-orchestrator
+  template:
+    metadata:
+      labels:
+        app: dsh-orchestrator
+    spec:
+      imagePullSecrets:
+        - name: dsh-acr-pull
+      serviceAccountName: dsh-orchestrator
+      containers:
+        - name: orchestrator
+          image: registry.example.com/dsh/dshs:0.2.0
+          imagePullPolicy: Always
+          args: ["--host", "0.0.0.0"]
+          env:
+            - name: DSHS_DB_URL
+              valueFrom:
+                secretKeyRef:
+                  name: dsh-pg
+                  key: url
+            - name: DSHS_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: dsh-secret
+                  key: key
+            - name: DSHS_SECURE_COOKIES
+              value: "true"
+            - name: DSHS_BASE_DOMAIN
+              value: "dsh.example.com"
+            - name: DSHS_COOKIE_DOMAIN
+              value: ".dsh.example.com"
+            - name: DSHS_DEPLOY_MODE
+              value: "k8s"
+            - name: DSHS_NAMESPACE
+              value: "dsh"
+            - name: DSHS_DSH_IMAGE
+              value: "registry.example.com/dsh/dsh:0.1.1-rc.2"
+            - name: DSHS_CONTROL_PLANE_IMAGE
+              value: "registry.example.com/dsh/dshs:0.2.0"
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          ports:
+            - containerPort: 3080
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            seccompProfile:
+              type: RuntimeDefault
+            readOnlyRootFilesystem: true
+          resources:
+            requests:
+              cpu: "250m"
+              memory: "256Mi"
+            limits:
+              cpu: "1"
+              memory: "1Gi"
+      volumes:
+        - name: tmp
+          emptyDir: {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+spec:
+  selector:
+    app: dsh-orchestrator
+  ports:
+    - port: 3080
+      targetPort: 3080
+  type: ClusterIP
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+spec:
+  minAvailable: 2
+  selector:
+    matchLabels:
+      app: dsh-orchestrator
@@ -0,0 +1,41 @@
+# 控制面 ServiceAccount + RBAC（docs/k8s.md §5.3）：dsh-orchestrator 在 dsh
+# 命名空间里建/删每用户 Pod/Service/NetworkPolicy/Secret/Job/PVC，读事件，
+# lease 供 leader election。
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "services", "secrets", "persistentvolumeclaims", "configmaps", "events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["networkpolicies"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: dsh-orchestrator
+subjects:
+  - kind: ServiceAccount
+    name: dsh-orchestrator
+    namespace: dsh
@@ -0,0 +1,14 @@
+# 每用户 DSH 共享的 RWX 卷（docs/k8s.md §4.3/§4.7）。ACK 上 RWX 用阿里云 NAS
+# （需先有 alicloud-nas StorageClass + NAS 文件系统；单机测试阶段可先不建，
+# K8sSpawner 的 Pod 会因挂不上卷而 Pending，但生命周期逻辑可验）。
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: dsh-users
+  namespace: dsh
+spec:
+  accessModes: ["ReadWriteMany"]
+  resources:
+    requests:
+      storage: 50Gi
+  storageClassName: alicloud-nas
@@ -0,0 +1,16 @@
+# 阿里云 NAS StorageClass（ACK + CNFS）。每用户 DSH 共享的 RWX 卷走 NAS
+# （deploy/04-pvc.yaml 引用 alicloud-nas）。通过 CNFS CR（storage.alibabacloud.com/v1beta1
+# ContainerNetworkFileSystem，控制台「容器网络文件系统」创建，名为 `nas`）引用
+# 已就绪的 NAS 文件系统，provisioner 自动在 `/` 下为每个 PVC 建独立子目录。
+# 见 docs/k8s-deploy.md §5.5：ACK 上 Longhorn 被 NAS/CNFS 取代。
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: alicloud-nas
+provisioner: nasplugin.csi.alibabacloud.com
+reclaimPolicy: Delete
+parameters:
+  volumeAs: subpath
+  containerNetworkFileSystem: nas
+  path: "/"
+  archiveOnDelete: "false"
@@ -0,0 +1,42 @@
+# 命名空间配额 + 默认资源区间（docs/k8s.md §3.6 / Phase 3「上线即配」）。
+# 防单个用户/大量用户耗尽集群。数值按 2×4C8G + 1 自动扩节点起步规模估计，
+# 上量后按 §11.1 对象数监控调。
+apiVersion: v1
+kind: ResourceQuota
+metadata:
+  name: dsh-quota
+  namespace: dsh
+spec:
+  hard:
+    requests.cpu: "6"
+    requests.memory: "12Gi"
+    limits.cpu: "12"
+    limits.memory: "24Gi"
+    count/pods: "60"
+    count/services: "80"
+    count/networkpolicies: "120"
+    count/jobs: "40"
+    count/secrets: "120"
+    count/configmaps: "120"
+---
+apiVersion: v1
+kind: LimitRange
+metadata:
+  name: dsh-limits
+  namespace: dsh
+spec:
+  limits:
+    # 默认值覆盖没有显式声明资源的 Pod；上限兜底防失控。
+    - type: Container
+      default:
+        cpu: "500m"
+        memory: "512Mi"
+      defaultRequest:
+        cpu: "250m"
+        memory: "256Mi"
+      max:
+        cpu: "4"
+        memory: "8Gi"
+      min:
+        cpu: "10m"
+        memory: "16Mi"
@@ -0,0 +1,17 @@
+# Pod Security Admission：命名空间 enforce=restricted（docs/k8s.md 选型定案
+# §0 / Phase 3）。准入时拒绝 privileged/hostPath/hostNetwork/提权 capability。
+#
+# ⚠️ 时序（严格，docs/k8s.md §4.9）：必须在 08-bootstrap.yaml 的权限提升 Job
+# 跑完 **之后** 再 apply 本文件——先打 restricted 标签，非 root 的 bootstrap
+# Job 就写不了 PVC 根，用户目录永远建不出来。
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: dsh
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    pod-security.kubernetes.io/audit: restricted
+    pod-security.kubernetes.io/audit-version: latest
+    pod-security.kubernetes.io/warn: restricted
+    pod-security.kubernetes.io/warn-version: latest
@@ -0,0 +1,37 @@
+# 一次性权限提升 Job：把共享 RWX PVC 根 chmod 1777（world-writable + sticky），
+# 让后续各用户的非 root initContainer 能在根下建自己的 <userId>/ 目录
+# （docs/k8s.md §4.9 第 1 步）。initContainer 挂的是 PVC **根**，不做 subPath。
+#
+# ⚠️ 只在 namespace 打 PSA restricted **之前** apply（见 07-psa.yaml 头注释）。
+# 跑完可删，或留着（restartPolicy Never + 一次性效果，幂等）。
+#
+# 用控制面镜像（node:22-slim，已推 ACR）而非 busybox：集群拉不动 docker.io。
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dsh-users-bootstrap
+  namespace: dsh
+spec:
+  ttlSecondsAfterFinished: 300
+  template:
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      imagePullSecrets:
+        - name: dsh-acr-pull
+      containers:
+        - name: chmod-root
+          image: registry.example.com/dsh/dshs:0.2.0
+          command: ["sh", "-c", "chmod 1777 /mnt"]
+          securityContext:
+            runAsUser: 0   # 需 root 才能 chmod；本 Job 跑在 PSA restricted 之前
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          volumeMounts:
+            - name: data-root
+              mountPath: /mnt
+      volumes:
+        - name: data-root
+          persistentVolumeClaim:
+            claimName: dsh-users
@@ -0,0 +1,61 @@
+# 控制面 NetworkPolicy（docs/k8s.md §4.2 尾部）。若 namespace 走 default-deny
+# （Cilium/Terway DataPath V2 下可能默认放行，则本文件为显式白名单、无害）。
+# ingress：Traefik/Ingress → 3080；egress：Postgres:5432 + K8s API:443 + DNS:53
+# + 每用户 DSH sidecar:8081 + 每用户 file sidecar:8082。控制面不回调任何公网。
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: dsh-orchestrator
+  namespace: dsh
+spec:
+  podSelector:
+    matchLabels:
+      app: dsh-orchestrator
+  policyTypes: [Ingress, Egress]
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: ingress-nginx
+        - namespaceSelector: {}
+      ports:
+        - port: 3080
+  egress:
+    - to:
+        - podSelector:
+            matchLabels:
+              cnpg.io/cluster: dsh-pg
+      ports:
+        - port: 5432
+    - to:
+        - podSelector:
+            matchLabels:
+              app: dsh
+      ports:
+        - port: 8081
+    - to:
+        - podSelector:
+            matchLabels:
+              app: dsh-files
+      ports:
+        - port: 8082
+    - to:
+        - namespaceSelector: {}
+          podSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      ports:
+        - port: 53
+          protocol: UDP
+        - port: 53
+          protocol: TCP
+    # K8s API server（托管的 control plane 在集群外，走公网/内网 API endpoint，
+    # 端口 6443/443）。这里放全出口 6443+443 以便访问 API server；如需更严，
+    # 按集群 API endpoint IP/CIDR 收窄。
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+            except: [169.254.169.254/32]
+      ports:
+        - port: 6443
+        - port: 443
@@ -0,0 +1,41 @@
+# 给每个节点下发 NFS 客户端 sunrpc 调优参数（用户提供的腾讯云侧经验值）：
+#   options sunrpc tcp_slot_table_entries=128
+#   options sunrpc tcp_max_slot_table_entries=128
+# 写到 /etc/modprobe.d/sunrpc.conf，节点下次加载 sunrpc 模块时生效（持久）。
+# 高并发 NFS 下可避免 "RPC: task blocked"。放在 kube-system，跑完可删。
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: sunrpc-tuning
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: sunrpc-tuning
+  template:
+    metadata:
+      labels:
+        app: sunrpc-tuning
+    spec:
+      hostPID: false
+      containers:
+        - name: write
+          image: registry.example.com/dsh/dshs:0.2.0
+          command:
+            - sh
+            - -ec
+            - |
+              mkdir -p /host/etc/modprobe.d
+              printf 'options sunrpc tcp_slot_table_entries=128\noptions sunrpc tcp_max_slot_table_entries=128\n' > /host/etc/modprobe.d/sunrpc.conf
+              echo "wrote sunrpc.conf:"; cat /host/etc/modprobe.d/sunrpc.conf
+              sleep infinity
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: modprobe
+              mountPath: /host/etc/modprobe.d
+      volumes:
+        - name: modprobe
+          hostPath:
+            path: /etc/modprobe.d
+            type: DirectoryOrCreate