ValueError: Parameter model.layers.16.mlp.down_proj.qweight not found in the model.

[fukatani]

I trained llama-3.1 by QLORA as below.

import torch
from time import time
from datasets import load_dataset
from peft import LoraConfig, PeftModel, prepare_model_for_kbit_training
from transformers import (
    AutoConfig,
    AutoModelForCausalLM,
    AutoTokenizer,
    BitsAndBytesConfig,
    AutoTokenizer,
    TrainingArguments,
)
from trl import SFTTrainer,setup_chat_format

model_id = "/kaggle/input/llama-3.1/transformers/8b/2"

compute_dtype = torch.bfloat16
bnb_config = BitsAndBytesConfig(
        load_in_4bit=True,
        bnb_4bit_quant_type="nf4",
        bnb_4bit_compute_dtype=compute_dtype,
        bnb_4bit_use_double_quant=True)

model_config = AutoConfig.from_pretrained(
    model_id,
    trust_remote_code=True,
    max_new_tokens=1024
)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    trust_remote_code=True,
    config=model_config,
    quantization_config=bnb_config,
    device_map='auto',
)
tokenizer = AutoTokenizer.from_pretrained(model_id)
model, tokenizer = setup_chat_format(model, tokenizer)
model = prepare_model_for_kbit_training(model)
peft_config = LoraConfig(
        lora_alpha=64,
        lora_dropout=0.05,
        r=4,
        bias="none",
        task_type="CAUSAL_LM",
        target_modules= ["q_proj", "k_proj", "v_proj", "o_proj",
                      "gate_proj", "up_proj", "down_proj",]
)
training_arguments = TrainingArguments(
        output_dir="./results_llama3_sft/",
        evaluation_strategy="steps",
        do_eval=True,
        optim="paged_adamw_8bit",
        per_device_train_batch_size=8,
        gradient_accumulation_steps=2,
        per_device_eval_batch_size=8,
        log_level="debug",
        save_steps=20,
        logging_steps=1,
        learning_rate=1e-4,
        eval_steps=1,
        max_steps=20,
        num_train_epochs=5,
        warmup_steps=3,
        lr_scheduler_type="linear",
)

# prepare my dataset here.
trainer = SFTTrainer(
        model=model,
        train_dataset=dataset['train'],
        eval_dataset=dataset['test'],
        peft_config=peft_config,
        dataset_text_field="text",
        max_seq_length=512,
        tokenizer=tokenizer,
        args=training_arguments,
)

trainer.train()
# Load the LoRA model
lora_model = PeftModel.from_pretrained(model, "./results_llama3_sft/")

# Merge LoRA weights into the base model
lora_model = lora_model.merge_and_unload()  # This merges LoRA weights into the base model

# Save the merged model
lora_model.save_pretrained("./results_llama3_sft_merged/")

# Save tokenizer
tokenizer.save_pretrained("./results_llama3_sft_merged/")

Run inference

import vllm

llm = vllm.LLM(
    "./results_llama3_sft_merged/",
    quantization="bitsandbytes",
    load_format="bitsandbytes",
    dtype="half", 
    pipeline_parallel_size=2, 
    gpu_memory_utilization=0.95, 
    trust_remote_code=True,
    enforce_eager=True,
    max_model_len=8192,
    disable_log_stats=True
)

I got

[rank0]: Traceback (most recent call last):
[rank0]:   File "/kaggle/working/run_vllm.py", line 25, in <module>
[rank0]:     llm = vllm.LLM(
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/entrypoints/llm.py", line 214, in __init__
[rank0]:     self.llm_engine = LLMEngine.from_engine_args(
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 564, in from_engine_args
[rank0]:     engine = cls(
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 325, in __init__
[rank0]:     self.model_executor = executor_class(
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/executor/distributed_gpu_executor.py", line 26, in __init__
[rank0]:     super().__init__(*args, **kwargs)
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 47, in __init__
[rank0]:     self._init_executor()
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/executor/multiproc_gpu_executor.py", line 111, in _init_executor
[rank0]:     self._run_workers("load_model",
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/executor/multiproc_gpu_executor.py", line 185, in _run_workers
[rank0]:     driver_worker_output = driver_worker_method(*args, **kwargs)
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/worker/worker.py", line 183, in load_model
[rank0]:     self.model_runner.load_model()
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1016, in load_model
[rank0]:     self.model = get_model(model_config=self.model_config,
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/model_executor/model_loader/__init__.py", line 19, in get_model
[rank0]:     return loader.load_model(model_config=model_config,
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 1111, in load_model
[rank0]:     self._load_weights(model_config, model)
[rank0]:   File "/opt/conda/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 1065, in _load_weights
[rank0]:     raise ValueError(
[rank0]: ValueError: Parameter model.layers.16.mlp.down_proj.qweight not found in the model.

I use vllm==0.6.2
Any Suggestion or help is highly appreciated.

[NikyParfenov]

Hello!

I have the same problem with Mistral-Nemo-Instruct-2407 after I quantized it by BitsAndBytes. Without parallelization it works well.

os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["CUDA_VISIBLE_DEVICES"] = "1, 2, 3, 4"
os.environ["TOKENIZERS_PARALLELISM"] = "True"
os.environ["VLLM_USE_MODELSCOPE"] = "True"


def vllm_call(model_id, prompts, devices=1):

    sampling_params = SamplingParams(temperature=0.1, top_p=0.95)
    llm = LLM(model=model_id, 
              quantization='bitsandbytes', 
              load_format='bitsandbytes',
              max_model_len=4000,
              gpu_memory_utilization=0.95,
              pipeline_parallel_size=devices,
            #   tensor_parallel_size=devices,
              enforce_eager=None)
    outputs = llm.generate(prompts, sampling_params)
    return outputs
    
    
if __name__ == "__main__":

    model_id = "./Mistral-Nemo-Instruct-2407_bab-4bit-double"
    
    prompts = [
        "Hello, my name is",
        "The president of the United States is",
        "The capital of France is",
        "The future of AI is",
    ]

    outputs = vllm_call(model_id, prompts, devices=4)

    for output in outputs:
        prompt = output.prompt
        generated_text = output.outputs[0].text
        print(f"Prompt: {prompt}, Generated text: {generated_text}")

[rank0]: Traceback (most recent call last):
[rank0]:   File "/home/ubuntu/quantization/quantization.py", line 73, in <module>
[rank0]:     outputs = vllm_call(model_ids[3], prompts, devices=4)
[rank0]:   File "/home/ubuntu/quantization/quantization.py", line 50, in vllm_call
[rank0]:     llm = LLM(model=model_id, 
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/entrypoints/llm.py", line 214, in __init__
[rank0]:     self.llm_engine = LLMEngine.from_engine_args(
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 564, in from_engine_args
[rank0]:     engine = cls(
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/engine/llm_engine.py", line 325, in __init__
[rank0]:     self.model_executor = executor_class(
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/executor/distributed_gpu_executor.py", line 26, in __init__
[rank0]:     super().__init__(*args, **kwargs)
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/executor/executor_base.py", line 47, in __init__
[rank0]:     self._init_executor()
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/executor/multiproc_gpu_executor.py", line 111, in _init_executor
[rank0]:     self._run_workers("load_model",
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/executor/multiproc_gpu_executor.py", line 185, in _run_workers
[rank0]:     driver_worker_output = driver_worker_method(*args, **kwargs)
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/worker/worker.py", line 183, in load_model
[rank0]:     self.model_runner.load_model()
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/worker/model_runner.py", line 1016, in load_model
[rank0]:     self.model = get_model(model_config=self.model_config,
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/model_executor/model_loader/__init__.py", line 19, in get_model
[rank0]:     return loader.load_model(model_config=model_config,
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 1111, in load_model
[rank0]:     self._load_weights(model_config, model)
[rank0]:   File "/home/ubuntu/quantization/venv310/lib/python3.10/site-packages/vllm/model_executor/model_loader/loader.py", line 1065, in _load_weights
[rank0]:     raise ValueError(
[rank0]: ValueError: Parameter model.layers.10.mlp.down_proj.qweight not found in the model.
ERROR 10-09 17:37:26 multiproc_worker_utils.py:120] Worker VllmWorkerProcess pid 3491644 died, exit code: -15
INFO 10-09 17:37:26 multiproc_worker_utils.py:124] Killing local vLLM worker processes
/usr/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 3 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '

[nick-tonjum]

Same issue :P

[echo-yi]

I'm facing the same issue. Any updates?