Tensor-LLM 1.0.0rc4 量化推理QwQ-32B int4

1
python /app/tensorrt_llm/examples/models/core/qwen/convert_checkpoint.py     --output_dir /workspace/models/QwQ-32B-checkpoint     --model_dir /workspace/models/QwQ-32B     --dtype float16     --tp_size 1     --pp_size 1     --workers 8     --use_parallel_embedding --use_weight_only --weight_only_precision int4

1
{
2
  "tensor_parallel": 1,
3
  "pipeline_parallel": 1,
4
  "max_batch_size": 1,
5
  "max_input_len": 4096,
6
  "max_seq_len": 8192,
7
  "max_num_tokens": 4096,
8
  "max_beam_width": 1,
9
  "enable_weight_only_quantization": true,
10
  "gpt_attention_plugin": "float16",
11
  "remove_input_padding": true,
12
  "context_fmha": true,
13
  "log_level": "info"
14
}

1
trtllm-build     --output_dir /workspace/models/QwQ-32B-engine     --checkpoint_dir /workspace/models/QwQ-32B-checkpoint     --build_config /workspace/models/build_config.json     --workers 8     --cluster_key A100-PCIe-40GB

1
trtllm-serve serve /workspace/models/QwQ-32B-engine --tokenizer /workspace/models/QwQ-32B --host 0.0.0.0 --port 8000 --tp_size 1 --pp_size 1 --max_batch_size 1 --max_seq_len 8192 --max_beam_width 1 --max_num_tokens 4096 --log_level info --num_postprocess_workers 8

1
[TensorRT-LLM][INFO] Memory usage when calculating max tokens in paged kv cache: total: 39.49 GiB, available: 20.44 GiB

1
python benchmarks/cpp/prepare_dataset.py --stdout --tokenizer Qwen/QwQ-32B token-norm-dist --input-mean 4096 --output-mean 1024 --input-stdev 0 --output-stdev 0 --num-requests 3000 > /tmp/synthetic_128_128.txt

1
trtllm-bench --model Qwen/QwQ-32B --model_path /workspace/models/QwQ-32B throughput --dataset /tmp/synthetic_128_128.txt --engine_dir /workspace/models/QwQ-32B-engine

1
head_dim = getattr(
2
            hf_config, "head_dim",
3
            hf_config.hidden_size // hf_config.num_attention_heads)