Vllm Qwen3.6-35b-a3b
Материал из Мои проекты
nvidia-smi
Wed Jul 22 08:13:42 2026
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 610.43.02 KMD Version: 610.43.02 CUDA UMD Version: 13.3 |
+-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA GeForce RTX 3090 On | 00000000:01:00.0 Off | N/A |
| 77% 58C P8 32W / 370W | 22870MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
| 1 NVIDIA GeForce RTX 3090 On | 00000000:81:00.0 Off | N/A |
| 65% 64C P8 38W / 350W | 22882MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
| 2 NVIDIA GeForce RTX 3090 On | 00000000:82:00.0 Off | N/A |
| 64% 47C P8 26W / 370W | 22882MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
| 3 NVIDIA GeForce RTX 3090 On | 00000000:C1:00.0 Off | N/A |
| 73% 54C P8 35W / 370W | 22876MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
| 4 NVIDIA GeForce RTX 3090 On | 00000000:C2:00.0 Off | N/A |
| 70% 57C P8 28W / 370W | 22075MiB / 24576MiB | 0% Default |
| | | N/A |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 1294156 C VLLM::Worker_TP0 22860MiB |
| 1 N/A N/A 1294157 C VLLM::Worker_TP1 22872MiB |
| 2 N/A N/A 1294158 C VLLM::Worker_TP2 22872MiB |
| 3 N/A N/A 1294159 C VLLM::Worker_TP3 22866MiB |
| 4 N/A N/A 1293408 C VLLM::EngineCore 17682MiB |
| 4 N/A N/A 1293522 C VLLM::EngineCore 1720MiB |
| 4 N/A N/A 1293557 C VLLM::EngineCore 1982MiB |
| 4 N/A N/A 3608522 C+G ./qdrant 408MiB |Создать папку проекта и перейти в нее. Создать директорию для моделей.
$ mkdir vllm $ cd vllm $ mkdir models
Активировать виртуальное окружение
$ source ../patents/bin/activate
Скачать модели локально, это нужно, чтобы контейнеры стартовали быстрее
$ pip install -U "huggingface_hub[cli]" $ hf download BAAI/bge-m3 --local-dir ./models/bge-m3
$ hf download Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4 --local-dir ./models/Qwen2.5-14B-Instruct-GPTQ-Int4 $ hf download BAAI/bge-reranker-v2-m3 --local-dir ./models/bge-reranker-v2-m3 $ hf download Qwen/Qwen3.6-35B-A3B --local-dir ./models/Qwen3.6-35B-A3B
cat docker-compose.yml
services:
vllm-embed:
image: vllm/vllm-openai:latest
container_name: vllm-embed
shm_size: '1gb'
mem_limit: 4g
ports:
- "8001:8000"
environment:
- VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
- OMP_NUM_THREADS=4
- MKL_NUM_THREADS=4
- TRANSFORMERS_OFFLINE=1
- HF_HUB_OFFLINE=1
env_file: .env
volumes:
- ./models/bge-m3:/app/models/bge-m3:ro
networks:
- localai_default
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ['4']
capabilities: [gpu]
restart: unless-stopped
command: >
/app/models/bge-m3
--served-model-name bge-m3
--tensor-parallel-size "1"
--gpu-memory-utilization "0.15"
--max-model-len 8192
vllm-rerank:
image: vllm/vllm-openai:latest
container_name: vllm-rerank
shm_size: '1gb'
mem_limit: 8g
ports:
- "8003:8000"
environment:
- VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
- OMP_NUM_THREADS=4
- MKL_NUM_THREADS=4
- TRANSFORMERS_OFFLINE=1
- HF_HUB_OFFLINE=1
env_file: .env
volumes:
- ./models/bge-reranker-v2-m3:/app/models/bge-reranker-v2-m3:ro
networks:
- localai_default
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ['4']
capabilities: [gpu]
restart: unless-stopped
command: >
/app/models/bge-reranker-v2-m3
--served-model-name bge-reranker-v2-m3
--tensor-parallel-size "1"
--gpu-memory-utilization "0.15"
--max-model-len 4096
vllm-qwen25-14b-keyword:
image: vllm/vllm-openai:latest
container_name: vllm-qwen25-14b-keyword
environment:
- VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
volumes:
- ./models/Qwen2.5-14B-Instruct-GPTQ-Int4:/app/models/Qwen2.5-14B-Instruct-GPTQ-Int4:ro
ports:
- "8005:8000"
ipc: host
shm_size: 20g
networks:
- localai_default
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ['4']
capabilities: [gpu]
restart: unless-stopped
command: >
/app/models/Qwen2.5-14B-Instruct-GPTQ-Int4
--served-model-name Qwen2.5-14B
--tensor-parallel-size "1"
--dtype auto
--quantization gptq_marlin
--gpu-memory-utilization "0.9"
--enable-prefix-caching
--enable-chunked-prefill
--trust-remote-code
--max-model-len 8192
vllm-qwen3.6-35b-a3b:
image: vllm/vllm-openai:latest
container_name: vllm-qwen3.6-35b-a3b
environment:
- VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1
- OMP_NUM_THREADS=4
- MKL_NUM_THREADS=4
- TRANSFORMERS_OFFLINE=1
- HF_HUB_OFFLINE=1
volumes:
- ./models/Qwen3.6-35B-A3B:/app/models/Qwen3.6-35B-A3B:ro
ports:
- "8002:8000"
ipc: host
shm_size: 40g
networks:
- localai_default
deploy:
resources:
reservations:
devices:
- driver: nvidia
device_ids: ['0', '1', '2', '3']
capabilities: [gpu]
restart: unless-stopped
command: >
/app/models/Qwen3.6-35B-A3B
--served-model-name Qwen3.6-35B-A3B
--tensor-parallel-size "4"
--dtype bfloat16
--gpu-memory-utilization "0.92"
--enable-prefix-caching
--enable-chunked-prefill
--trust-remote-code
--reasoning-parser qwen3
--enable-auto-tool-choice
--tool-call-parser qwen3_coder
networks:
localai_default:
driver: bridge