查看 NPU 信息

拉取镜像

docker pull qingcheng-ai-cn-beijing.cr.volces.com/public/chitu-ascend_a2:v0.6.0

下载模型

mkdir Qwen3.6-27B && cd Qwen3.6-27B
modelscope download --model Qwen/Qwen3.6-27B --local_dir ./

创建容器

docker run \
  --name chitu \
  -itd \
  --net=host \
  --shm-size=500g \
  --device /dev/davinci0 \
  --device /dev/davinci1 \
  --device /dev/davinci2 \
  --device /dev/davinci3 \
  --device /dev/davinci4 \
  --device /dev/davinci5 \
  --device /dev/davinci6 \
  --device /dev/davinci7 \
  --device /dev/davinci_manager \
  --device /dev/devmm_svm \
  --device /dev/hisi_hdc \
  --entrypoint=bash \
  -v /usr/local/dcmi:/usr/local/dcmi \
  -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
  -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
  -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
  -v /etc/ascend_install.info:/etc/ascend_install.info \
  -v /mnt/Qwen3.6-27B:/Qwen3.6-27B \
  qingcheng-ai-cn-beijing.cr.volces.com/public/chitu-ascend_a2:v0.6.0 

进入容器

docker exec -it chitu bash

启动服务

export WORLD_SIZE=2
torchrun --nnodes 1 \
    --nproc_per_node 2 \
    --master_port=22525 \
    -m chitu \
    serve.port=21002 \
    infer.cache_type=paged \
    infer.pp_size=1 \
    infer.tp_size=2 \
    models=Qwen3.6-27B \
    models.ckpt_dir=/Qwen3.6-27B \
    infer.mla_absorb=absorb-without-precomp \
    infer.raise_lower_bit_float_to=bfloat16 \
    infer.max_batch_size=1 \
    infer.max_seq_len=4096 \
    request.max_new_tokens=1024 \
    infer.use_cuda_graph=True \
    infer.attn_type=npu

推理测试

curl localhost:21002/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "messages": [
      {
        "role": "system",
        "content": "You are a helpful assistant."
      },
      {
        "role": "user",
        "content": "What is machine learning?"
      }
    ],
    "enable_thinking": false
  }'

参考

赤兔推理框架开源链接