ks8: 昇腾虚拟化方案
一、需求
背景：昇腾卡
二、方案实现
- volcalno配置
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override: |
actions: enqueue, allocate, preempt, backfill, reclaim
tiers:
- plugins:
- name: priority
- name: conformance
- name: overcommit
arguments:
overcommit-factor: 2.0
- plugins:
- name: gang
enablePreemptable: false
enableJobStarving: false
- name: drf
enablePreemptable: false
- name: predicates
- name: proportion
- name: nodeorder
- name: binpack
功能点：
①用户-租户: 创建k8s同名命名空间（注意名字命名规则，k8s限制）并绑定显卡配额，分配用个数，例如租户zhangshan, 配额npu:2
②创建优先级：引导用户去[清单执行器]执行器执行
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: ai-high
value: 1000
preemptionPolicy: PreemptLowerPriority
description: "AI high priority for volcano preempt tests"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: ai-low
value: 100
preemptionPolicy: PreemptLowerPriority
description: "AI low priority for volcano preempt tests"
③ 创建队列： 用户绑定空间和配额时同时创建
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
name: default
spec:
weight: 1
reclaimable: true
capability:
huawei.com/Ascend310P: "4"
---
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
name: tenant-a
spec:
weight: 1
reclaimable: true
capability:
huawei.com/Ascend310P: "2"
④ 推理任务修改：
apiVersion: v1
kind: Namespace
metadata:
name: ns-tenant-a
---
apiVersion: batch.volcano.sh/v1alpha1
kind: Job
metadata:
name: job-c-1
namespace: ns-tenant-a // 租户
spec:
schedulerName: volcano
queue: tenant-a // 队列选择
priorityClassName: ai-low // 优先级
minAvailable: 1
tasks:
- replicas: 1
name: worker
template:
spec:
priorityClassName: ai-low // 队列选择
restartPolicy: Never
containers:
- name: worker
image: 10.30.15.90:5000/aarch64/mindie:2.3.1
command: ["sleep", "3600"]
resources:
requests:
huawei.com/Ascend310P: "1"
limits:
huawei.com/Ascend310P: "1"
---
三、NPU测试
3.1 310P测试
3.1.1 ✔️ Hami软切测试
✔️ 软切单卡测试
- 测试脚本
apiVersion: v1 kind: Pod metadata: name: gpu-pod annotations: huawei.com/vnpu-mode: "hami-core" spec: containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐 - 测试输出
[root@gpu-pod HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 45 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1284 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
- 测试脚本
✔️ 软切双卡测试
测试脚本
apiVersion: v1 kind: Pod metadata: name: gpu-pod annotations: huawei.com/vnpu-mode: "hami-core" spec: containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "2" huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐测试输出
[root@gpu-pod HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 46 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1665 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32800 310P3 | OK | NA 45 0 / 0 | | 1 1 | 0000:82:00.0 | 0 1284 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
3.1.2 ⚠️ Hami硬切测试
✔️硬切单卡测试
测试脚本：
apiVersion: v1 kind: Pod metadata: name: gpu-pod spec: runtimeClassName: ascend schedulerName: hami-scheduler containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "1024"测试输出
+--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310Pvir02 | OK | NA 46 0 / 0 | | 0 0 | 0000:82:00.0 | 0 367 / 10923 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
⚠️硬切双卡测试
双卡限卡限显存 ❌
- 测试脚本
apiVersion: v1 kind: Pod metadata: name: gpu-pod spec: runtimeClassName: ascend schedulerName: hami-scheduler containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "2" huawei.com/Ascend310P-memory: "1024"- 测试输出
[root@10-30-15-89 hami]# kubectl apply -f test.yaml Error from server: error when creating "test.yaml": admission webhook "vgpu.hami.io" denied the request: vNPU nor supported for multiple devices 3.双卡限卡无显存✔️
测试脚本
apiVersion: v1 kind: Pod metadata: name: gpu-pod spec: runtimeClassName: ascend schedulerName: hami-scheduler containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "2"测试输出
[root@gpu-pod HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 47 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1665 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32800 310P3 | OK | NA 45 0 / 0 | | 1 1 | 0000:82:00.0 | 0 1284 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
3.1.3 ✔️ Volcano 软切测试
- ✔️ 软切单卡测试
✔️ 单卡限制显存
- 测试代码
apiVersion: v1 kind: Pod metadata: name: gpu-pod annotations: huawei.com/vnpu-mode: "hami-core" spec: containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐 - 测试输出
[root@gpu-pod HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 46 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1284 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
- 测试代码
✔️ 软切双卡测试
- 双卡限制显存
- 测试代码
apiVersion: v1 kind: Pod metadata: name: gpu-pod annotations: huawei.com/vnpu-mode: "hami-core" spec: containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "2" huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐 - 测试输出
[root@gpu-pod HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 47 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1664 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32800 310P3 | OK | NA 45 0 / 0 | | 1 1 | 0000:82:00.0 | 0 1285 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
- 测试代码
- ✔️ 双卡无限制显存
- 测试代码
apiVersion: v1 kind: Pod metadata: name: gpu-pod annotations: huawei.com/vnpu-mode: "hami-core" spec: containers: - name: ubuntu-container image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend310P: "2" #huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐 - 测试输出
[root@gpu-pod HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 46 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1665 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32800 310P3 | OK | NA 45 0 / 0 | | 1 1 | 0000:82:00.0 | 0 1284 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
- 测试代码
- 双卡限制显存
3.1.4 ❓ Volcano硬切测试
3.1.5 ⚠️ Volcano 软切vcjob
- 创建优先级
apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: ai-high value: 1000 preemptionPolicy: PreemptLowerPriority description: "AI high priority for volcano preempt tests" --- apiVersion: scheduling.k8s.io/v1 kind: PriorityClass metadata: name: ai-low value: 100 preemptionPolicy: PreemptLowerPriority description: "AI low priority for volcano preempt tests" - 创建队列
- ⚠️ Volcano 软切测试单个job
- Volcano 软切单卡
- 测试代码
apiVersion: v1 kind: Namespace metadata: name: ns-demo --- apiVersion: batch.volcano.sh/v1alpha1 kind: Job metadata: name: job-a-4 namespace: ns-demo spec: schedulerName: volcano queue: default priorityClassName: ai-low minAvailable: 1 tasks: - replicas: 1 name: worker template: metadata: annotations: volcano.sh/preemptable: "true" spec: priorityClassName: ai-low restartPolicy: Never containers: - name: worker image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["sleep", "3600"] resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "4096" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "4096" - 测试输出
[root@10-30-15-89 customer_test]# kubectl get vcjob,pg -n ns-demo NAME STATUS MINAVAILABLE RUNNINGS AGE job.batch.volcano.sh/job-a-4 Running 1 1 20s NAME STATUS MINMEMBER RUNNINGS AGE podgroup.scheduling.volcano.sh/job-a-4-d13b476a-1c16-47ee-85e5-49730bd7d804 Running 1 1 20s [root@10-30-15-89 customer_test]# kubectl get vcjob,pod,pg -n ns-demo NAME STATUS MINAVAILABLE RUNNINGS AGE job.batch.volcano.sh/job-a-4 Running 1 1 42s NAME READY STATUS RESTARTS AGE pod/job-a-4-worker-0 1/1 Running 0 42s NAME STATUS MINMEMBER RUNNINGS AGE podgroup.scheduling.volcano.sh/job-a-4-d13b476a-1c16-47ee-85e5-49730bd7d804 Running 1 1 42s [root@10-30-15-89 customer_test]# [root@10-30-15-89 customer_test]# [root@10-30-15-89 customer_test]# [root@10-30-15-89 customer_test]# kubectl -n ns-demo exec -it pod/job-a-4-worker-0 -- bash Welcome to 4.19.90-89.25.v2401.ky10.aarch64 System information as of time: Sun Aug 9 15:26:18 CST 2026 System load: 5.85 Memory used: 18.3% Swap used: 0.0% Usage On: 76% Users online: 0 [root@job-a-4-worker-0 HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 47 0 / 0 | | 0 0 | 0000:82:00.0 | 0 1665 / 44278 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
- 测试代码
- Volcano 软切双卡
❌ Volcano 软切双卡限内存
- 测试代码
apiVersion: v1 kind: Namespace metadata: name: ns-demo --- apiVersion: batch.volcano.sh/v1alpha1 kind: Job metadata: name: job-a-4 namespace: ns-demo spec: schedulerName: volcano queue: default priorityClassName: ai-low minAvailable: 1 tasks: - replicas: 1 name: worker template: metadata: annotations: volcano.sh/preemptable: "true" spec: priorityClassName: ai-low restartPolicy: Never containers: - name: worker image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["sleep", "3600"] resources: requests: huawei.com/Ascend310P: "2" huawei.com/Ascend310P-memory: "4096" limits: huawei.com/Ascend310P: "2" huawei.com/Ascend310P-memory: "4096" - 测试输出
[root@10-30-15-89 customer_test]# kubectl get vcjob,pod,pg -n ns-demo NAME STATUS MINAVAILABLE RUNNINGS AGE job.batch.volcano.sh/job-a-4 Pending 1 13s NAME READY STATUS RESTARTS AGE pod/job-a-4-worker-0 0/1 Pending 0 13s NAME STATUS MINMEMBER RUNNINGS AGE podgroup.scheduling.volcano.sh/job-a-4-fa1653d9-1b86-4231-8b7d-e85444e3663f Inqueue 1 13s
- 测试代码
✔️ Volcano 软切双卡无限内存
- 测试代码
apiVersion: v1 kind: Namespace metadata: name: ns-demo --- apiVersion: batch.volcano.sh/v1alpha1 kind: Job metadata: name: job-a-4 namespace: ns-demo spec: schedulerName: volcano queue: default priorityClassName: ai-low minAvailable: 1 tasks: - replicas: 1 name: worker template: metadata: annotations: volcano.sh/preemptable: "true" spec: priorityClassName: ai-low restartPolicy: Never containers: - name: worker image: 10.30.15.90:5000/aarch64/mindie:2.3.1 command: ["sleep", "3600"] resources: requests: huawei.com/Ascend310P: "2" # huawei.com/Ascend310P-memory: "4096" limits: huawei.com/Ascend310P: "2" # huawei.com/Ascend310P-memory: "4096" - 测试输出
[root@10-30-15-89 customer_test]# kubectl get vcjob,pod,pg -n ns-demo NAME STATUS MINAVAILABLE RUNNINGS AGE job.batch.volcano.sh/job-a-4 Running 1 1 11s NAME READY STATUS RESTARTS AGE pod/job-a-4-worker-0 1/1 Running 0 10s NAME STATUS MINMEMBER RUNNINGS AGE podgroup.scheduling.volcano.sh/job-a-4-dadb2a88-2765-46df-be48-e22345f1777f Running 1 1 11s [root@10-30-15-89 customer_test]# kubectl -n ns-demo exec -it pod/job-a-4-worker-0 -- bash Welcome to 4.19.90-89.25.v2401.ky10.aarch64 System information as of time: Sun Aug 9 15:42:49 CST 2026 System load: 9.87 Memory used: 18.3% Swap used: 0.0% Usage On: 76% Users online: 0 [root@job-a-4-worker-0 HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32768 310P3 | OK | NA 48 0 / 0 | | 0 0 | 0000:81:00.0 | 0 1697 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32768 310P3 | OK | NA 46 0 / 0 | | 1 1 | 0000:81:00.0 | 0 1243 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32768 | +===============================+=================+======================================================+
- 测试代码
⚠️ Volcano 软切其他测试
# 20 ❌ 卡无限测试输出❌ [root@10-30-15-89 customer_test]# kubectl get vcjob,pod,pg -n ns-demo NAME STATUS MINAVAILABLE RUNNINGS AGE job.batch.volcano.sh/job-a-4 Pending 1 14s NAME READY STATUS RESTARTS AGE pod/job-a-4-worker-0 0/1 Pending 0 14s NAME STATUS MINMEMBER RUNNINGS AGE podgroup.scheduling.volcano.sh/job-a-4-eab57f0c-c339-4777-ade9-c42c77cc6aa1 Inqueue 1 14s Warning FailedScheduling 3m16s volcano 0/1 nodes are unavailable: 1 no ascend device available. ------- # ✔️4卡测试成功✔️ [root@10-30-15-89 customer_test]# kubectl get vcjob,pod,pg -n ns-demo NAME STATUS MINAVAILABLE RUNNINGS AGE job.batch.volcano.sh/job-a-4 Running 1 1 8s NAME READY STATUS RESTARTS AGE pod/job-a-4-worker-0 1/1 Running 0 7s NAME STATUS MINMEMBER RUNNINGS AGE podgroup.scheduling.volcano.sh/job-a-4-eccb17f2-882b-451e-8fa8-ca4ed6fc1510 Running 1 1 8s [root@10-30-15-89 customer_test]# [root@10-30-15-89 customer_test]# [root@10-30-15-89 customer_test]# kubectl -n ns-demo exec -it pod/job-a-4-worker-0 -- bash Welcome to 4.19.90-89.25.v2401.ky10.aarch64 System information as of time: Sun Aug 9 15:55:04 CST 2026 System load: 5.31 Memory used: 18.4% Swap used: 0.0% Usage On: 76% Users online: 0 [root@job-a-4-worker-0 HwHiAiUser]# npu-smi info +--------------------------------------------------------------------------------------------------------+ | npu-smi 26.0.rc1 Version: 26.0.rc1 | +-------------------------------+-----------------+------------------------------------------------------+ | NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) | | Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) | +===============================+=================+======================================================+ | 32768 310P3 | OK | NA 47 0 / 0 | | 0 0 | 0000:81:00.0 | 0 1696 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32768 310P3 | OK | NA 46 0 / 0 | | 1 1 | 0000:81:00.0 | 0 1244 / 43693 | +===============================+=================+======================================================+ | 32800 310P3 | OK | NA 47 0 / 0 | | 0 2 | 0000:82:00.0 | 0 1665 / 44278 | +-------------------------------+-----------------+------------------------------------------------------+ | 32800 310P3 | OK | NA 45 0 / 0 | | 1 3 | 0000:82:00.0 | 0 1284 / 43693 | +===============================+=================+======================================================+ +-------------------------------+-----------------+------------------------------------------------------+ | NPU Chip | Process id | Process name | Process memory(MB) | +===============================+=================+======================================================+ | No running processes found in NPU 32768 | +===============================+=================+======================================================+ | No running processes found in NPU 32800 | +===============================+=================+======================================================+
- Volcano 软切单卡
3.1.6 ⚠️ Volcano 软切Deploy
基础环境：
cat << EOF > queue-deployment.yaml
---
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
name: default
spec:
weight: 1
reclaimable: true
capability:
huawei.com/Ascend310P: "4"
---
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
name: tenant-a
spec:
weight: 1
reclaimable: true
capability:
huawei.com/Ascend310P: "2"
---
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
name: tenant-b
spec:
weight: 1
reclaimable: true
capability:
huawei.com/Ascend310P: "2"
EOF
cat << EOF > namespce-deploynent.yaml
---
apiVersion: v1
kind: Namespace
metadata:
name: ns-demo
---
apiVersion: v1
kind: Namespace
metadata:
name: ns-tenant-a
---
apiVersion: v1
kind: Namespace
metadata:
name: ns-tenant-b
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: npu-quota
namespace: ns-tenant-a
spec:
hard:
requests.huawei.com/Ascend310P: "2"
limits.huawei.com/Ascend310P: "2"
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: npu-quota
namespace: ns-tenant-b
spec:
hard:
requests.huawei.com/Ascend310P: "2"
limits.huawei.com/Ascend310P: "2"
EOF
cat << EOF > priorty-deploynent.yaml
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: ai-high
value: 1000
preemptionPolicy: PreemptLowerPriority
description: "AI high priority for volcano preempt tests"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: ai-low
value: 100
preemptionPolicy: PreemptLowerPriority
description: "AI low priority for volc
EOF
多卡测试
- 测试代码
apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-a-4 namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-low --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-a-4 namespace: ns-demo labels: app: infer-a-4 spec: replicas: 1 selector: matchLabels: app: infer-a-4 template: metadata: labels: app: infer-a-4 annotations: scheduling.k8s.io/group-name: infer-a-4 volcano.sh/preemptable: "true" spec: schedulerName: volcano priorityClassName: ai-low containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "7" limits: huawei.com/Ascend310P: "7" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory - 测试输出
Events: Type Reason Age From Message ---- ------ ---- ---- ------- Warning FailedScheduling 71s volcano 0/1 nodes are unavailable: 1 no ascend device available. [root@10-30-15-89 deploy]# kubectl get pods -n ns-demo -w NAME READY STATUS RESTARTS AGE infer-a-1-7d9bf894f8-5qfbz 0/1 Pending 0 93s infer-a-2-557848c56-c2dfw 0/1 Pending 0 93s infer-a-3-84f98d4db6-bhtp9 0/1 Pending 0 92s infer-a-4-59fc88cb56-567bc 0/1 Pending 0 92s
- 测试代码
单卡测试
- 测试代码
mkdir -p deploy cat << EOF > deploy/case-a.whole.yaml --- apiVersion: v1 kind: Namespace metadata: name: ns-demo --- apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-a-1 namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-low --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-a-1 namespace: ns-demo labels: app: infer-a-1 spec: replicas: 1 selector: matchLabels: app: infer-a-1 template: metadata: labels: app: infer-a-1 annotations: scheduling.k8s.io/group-name: infer-a-1 volcano.sh/preemptable: "true" huawei.com/vnpu-mode: "hami-core" spec: schedulerName: volcano runtimeClassName: ascend priorityClassName: ai-low containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory --- apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-a-2 namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-low --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-a-2 namespace: ns-demo labels: app: infer-a-2 spec: replicas: 1 selector: matchLabels: app: infer-a-2 template: metadata: labels: app: infer-a-2 annotations: scheduling.k8s.io/group-name: infer-a-2 volcano.sh/preemptable: "true" huawei.com/vnpu-mode: "hami-core" spec: schedulerName: volcano runtimeClassName: ascend priorityClassName: ai-low containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory --- apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-a-3 namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-low --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-a-3 namespace: ns-demo labels: app: infer-a-3 spec: replicas: 1 selector: matchLabels: app: infer-a-3 template: metadata: labels: app: infer-a-3 annotations: scheduling.k8s.io/group-name: infer-a-3 volcano.sh/preemptable: "true" huawei.com/vnpu-mode: "hami-core" spec: schedulerName: volcano runtimeClassName: ascend priorityClassName: ai-low containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory --- apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-a-4 namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-low --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-a-4 namespace: ns-demo labels: app: infer-a-4 spec: replicas: 1 selector: matchLabels: app: infer-a-4 template: metadata: labels: app: infer-a-4 annotations: scheduling.k8s.io/group-name: infer-a-4 volcano.sh/preemptable: "true" huawei.com/vnpu-mode: "hami-core" spec: schedulerName: volcano runtimeClassName: ascend priorityClassName: ai-low containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory EOF cat << EOF > deploy/case-a.pend.yaml apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-a-5 namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-low --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-a-5 namespace: ns-demo labels: app: infer-a-5 spec: replicas: 1 selector: matchLabels: app: infer-a-5 template: metadata: labels: app: infer-a-5 annotations: scheduling.k8s.io/group-name: infer-a-5 volcano.sh/preemptable: "true" huawei.com/vnpu-mode: "hami-core" spec: schedulerName: volcano runtimeClassName: ascend priorityClassName: ai-low containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory EOF cat << EOF > deploy/case-b.hight.yaml apiVersion: scheduling.volcano.sh/v1beta1 kind: PodGroup metadata: name: infer-b-high namespace: ns-demo spec: minMember: 1 queue: default priorityClassName: ai-high --- apiVersion: apps/v1 kind: Deployment metadata: name: infer-b-high namespace: ns-demo labels: app: infer-b-high spec: replicas: 1 selector: matchLabels: app: infer-b-high template: metadata: labels: app: infer-b-high annotations: scheduling.k8s.io/group-name: infer-b-high huawei.com/vnpu-mode: "hami-core" spec: schedulerName: volcano runtimeClassName: ascend priorityClassName: ai-high containers: - name: vllm image: 10.30.15.90:5000/aarch64/vllm-ascend:0.22.1 imagePullPolicy: IfNotPresent command: - vllm - serve - /models/Qwen2.5-0.5B - --host - 0.0.0.0 - --port - "8000" - --dtype - float16 ports: - name: http containerPort: 8000 readinessProbe: httpGet: path: /v1/models port: http initialDelaySeconds: 30 periodSeconds: 10 failureThreshold: 30 resources: requests: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" limits: huawei.com/Ascend310P: "1" huawei.com/Ascend310P-memory: "3072" volumeMounts: - name: model mountPath: /models/Qwen2.5-0.5B readOnly: true volumes: - name: model hostPath: path: /exports/bdbbf9e5-bfee-4d64-b9e7-4c89ff27d0e6/data/hub/Qwen/Qwen2.5-0.5B type: Directory EOF - 测试输出
[root@10-30-15-89 hami]# kubectl get pg -n ns-demo NAME STATUS MINMEMBER RUNNINGS AGE infer-a-1 Running 1 1 6m42s infer-a-2 Running 1 1 6m42s infer-a-3 Running 1 1 6m42s infer-a-4 Running 1 1 6m42s [root@10-30-15-89 hami]# kubectl get pg -n ns-demo -w NAME STATUS MINMEMBER RUNNINGS AGE infer-a-1 Running 1 1 6m44s infer-a-2 Running 1 1 6m44s infer-a-3 Running 1 1 6m44s infer-a-4 Running 1 1 6m44s infer-a-5 1 0s infer-a-5 Inqueue 1 0s infer-b-high 1 0s infer-b-high Inqueue 1 1s infer-a-3 Completed 1 10m infer-b-high Inqueue 1 2s infer-a-3 Completed 1 10m infer-b-high Running 1 3s infer-a-3 Completed 1 10m infer-a-3 Completed 1 10m infer-b-high Running 1 1 7s ^C[root@10-30-15-89 hami]# [root@10-30-15-89 hami]# kubectl get pg -n ns-demo -w NAME STATUS MINMEMBER RUNNINGS AGE infer-a-1 Running 1 1 11m infer-a-2 Running 1 1 11m infer-a-3 Completed 1 11m infer-a-4 Running 1 1 11m infer-a-5 Inqueue 1 3m49s infer-b-high Running 1 1 40s
- 测试代码