Cannot be loaded in vllm
#3
by
mratsim
- opened
The error is
(APIServer pid=1) File "/usr/local/lib/python3.12/dist-packages/pydantic/_internal/_dataclasses.py", line 123, in __init__
(APIServer pid=1) s.__pydantic_validator__.validate_python(ArgsKwargs(args, kwargs), self_instance=s)
(APIServer pid=1) pydantic_core._pydantic_core.ValidationError: 1 validation error for ModelConfig
(APIServer pid=1) Value error, Unknown quantization method: . Must be one of ['awq', 'deepspeedfp', 'tpu_int8', 'fp8', 'ptpc_fp8', 'fbgemm_fp8', 'modelopt', 'modelopt_fp4', 'bitblas', 'gguf', 'gptq_marlin_24', 'gptq_marlin', 'gptq_bitblas', 'awq_marlin', 'gptq', 'compressed-tensors', 'bitsandbytes', 'hqq', 'experts_int8', 'ipex', 'quark', 'moe_wna16', 'torchao', 'auto-round', 'rtn', 'inc', 'mxfp4', 'petit_nvfp4']. [type=value_error, input_value=ArgsKwargs((), {'model': ...rocessor_plugin': None}), input_type=ArgsKwargs]
For information, I can load NVFP4 models quantized through llmcompressor properly like my own https://huggingface.co/mratsim/Seed-OSS-36B-Instruct-NVFP4
I also ran into model loading issues. I found that the model was quantized using an older version of ModelOpt. In your case, the problem likely stems from the config.json using "modelopt" under the "quant_library" field, rather than the "quant_method" field expected by vLLM.
I updated the config.json as shown below, and now vLLM loads the model correctly:
{
"architectures": [
"Qwen3MoeForCausalLM"
],
"attention_bias": false,
"attention_dropout": 0.0,
"bos_token_id": 151643,
"decoder_sparse_step": 1,
"dtype": "bfloat16",
"eos_token_id": 151645,
"head_dim": 128,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 12288,
"max_position_embeddings": 262144,
"max_window_layers": 94,
"mlp_only_layers": [],
"model_type": "qwen3_moe",
"moe_intermediate_size": 1536,
"norm_topk_prob": true,
"num_attention_heads": 64,
"num_experts": 128,
"num_experts_per_tok": 8,
"num_hidden_layers": 94,
"num_key_value_heads": 4,
"output_router_logits": false,
"rms_norm_eps": 1e-06,
"rope_scaling": null,
"rope_theta": 5000000,
"router_aux_loss_coef": 0.001,
"sliding_window": null,
"tie_word_embeddings": false,
"transformers_version": "4.53.3",
"use_cache": true,
"use_sliding_window": false,
"vocab_size": 151936,
"quantization_config": {
"config_groups": {
"group_0": {
"input_activations": {
"dynamic": false,
"num_bits": 4,
"type": "float",
"group_size": 16
},
"weights": {
"dynamic": false,
"num_bits": 4,
"type": "float",
"group_size": 16
},
"targets": [
"Linear"
]
}
},
"ignore": [
"model.layers.0.mlp.gate",
"model.layers.1.mlp.gate",
"model.layers.10.mlp.gate",
"model.layers.11.mlp.gate",
"model.layers.12.mlp.gate",
"model.layers.13.mlp.gate",
"model.layers.14.mlp.gate",
"model.layers.15.mlp.gate",
"model.layers.16.mlp.gate",
"model.layers.17.mlp.gate",
"model.layers.18.mlp.gate",
"model.layers.19.mlp.gate",
"model.layers.2.mlp.gate",
"model.layers.20.mlp.gate",
"model.layers.21.mlp.gate",
"model.layers.22.mlp.gate",
"model.layers.23.mlp.gate",
"model.layers.24.mlp.gate",
"model.layers.25.mlp.gate",
"model.layers.26.mlp.gate",
"model.layers.27.mlp.gate",
"model.layers.28.mlp.gate",
"model.layers.29.mlp.gate",
"model.layers.3.mlp.gate",
"model.layers.30.mlp.gate",
"model.layers.31.mlp.gate",
"model.layers.32.mlp.gate",
"model.layers.33.mlp.gate",
"model.layers.34.mlp.gate",
"model.layers.35.mlp.gate",
"model.layers.36.mlp.gate",
"model.layers.37.mlp.gate",
"model.layers.38.mlp.gate",
"model.layers.39.mlp.gate",
"model.layers.4.mlp.gate",
"model.layers.40.mlp.gate",
"model.layers.41.mlp.gate",
"model.layers.42.mlp.gate",
"model.layers.43.mlp.gate",
"model.layers.44.mlp.gate",
"model.layers.45.mlp.gate",
"model.layers.46.mlp.gate",
"model.layers.47.mlp.gate",
"model.layers.48.mlp.gate",
"model.layers.49.mlp.gate",
"model.layers.5.mlp.gate",
"model.layers.50.mlp.gate",
"model.layers.51.mlp.gate",
"model.layers.52.mlp.gate",
"model.layers.53.mlp.gate",
"model.layers.54.mlp.gate",
"model.layers.55.mlp.gate",
"model.layers.56.mlp.gate",
"model.layers.57.mlp.gate",
"model.layers.58.mlp.gate",
"model.layers.59.mlp.gate",
"model.layers.6.mlp.gate",
"model.layers.60.mlp.gate",
"model.layers.61.mlp.gate",
"model.layers.62.mlp.gate",
"model.layers.63.mlp.gate",
"model.layers.64.mlp.gate",
"model.layers.65.mlp.gate",
"model.layers.66.mlp.gate",
"model.layers.67.mlp.gate",
"model.layers.68.mlp.gate",
"model.layers.69.mlp.gate",
"model.layers.7.mlp.gate",
"model.layers.70.mlp.gate",
"model.layers.71.mlp.gate",
"model.layers.72.mlp.gate",
"model.layers.73.mlp.gate",
"model.layers.74.mlp.gate",
"model.layers.75.mlp.gate",
"model.layers.76.mlp.gate",
"model.layers.77.mlp.gate",
"model.layers.78.mlp.gate",
"model.layers.79.mlp.gate",
"model.layers.8.mlp.gate",
"model.layers.80.mlp.gate",
"model.layers.81.mlp.gate",
"model.layers.82.mlp.gate",
"model.layers.83.mlp.gate",
"model.layers.84.mlp.gate",
"model.layers.85.mlp.gate",
"model.layers.86.mlp.gate",
"model.layers.87.mlp.gate",
"model.layers.88.mlp.gate",
"model.layers.89.mlp.gate",
"model.layers.9.mlp.gate",
"model.layers.90.mlp.gate",
"model.layers.91.mlp.gate",
"model.layers.92.mlp.gate",
"model.layers.93.mlp.gate",
"lm_head"
],
"quant_algo": "NVFP4",
"kv_cache_scheme": {
"dynamic": false,
"num_bits": 8,
"type": "float"
},
"producer": {
"name": "modelopt",
"version": "0.33.0"
},
"quant_method": "modelopt"
}
}