Converting to GGUF after training a LoRA and merging it back to LLama 3.2 at 8bit.

[TByte007]

After training an 8bit LoRA and merging it back to LLama 3.2 loaded as 8bit then trying to convert it to GGUF with convert_hf_to_gguf.py I get this error:
Can not map tensor 'model.layers.0.mlp.down_proj.SCB'

What am I messing up (again) and is that the right way to do it? This is the python script that does the merge:

from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
import torch

def merge_lora_with_base_model():
    # Load the base model in 8-bit
    base_model_name = "meta-llama/Llama-3.2-1B"  # Changed to match fine-tuning script
    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_name,
        load_in_8bit=True,
        device_map="auto",
        torch_dtype=torch.float16,
    )
    
    # Load the LoRA model
    lora_path = "g:/ML/tmp/llama-3.2-1B-finetuned-LoRA"  # This path matches the save path in fine-tuning script
    model = PeftModel.from_pretrained(base_model, lora_path)
    
    # Merge LoRA weights with base model
    model = model.merge_and_unload()
    
    # Save the merged model
    output_path = "g:/ML/tmp/merged-llama-model-8bit"
    model.save_pretrained(output_path)
    
    # Save the tokenizer
    tokenizer = AutoTokenizer.from_pretrained(base_model_name)
    tokenizer.save_pretrained(output_path)
    
    print(f"Merged model saved to: {output_path}")

if __name__ == "__main__":
    merge_lora_with_base_model() 

Thanks in advance!

[TByte007]

If I merge it directly with the large model it works:

def merge_lora_with_base_model_32bit():
    # Load the base model in 32-bit (full precision)
    base_model_name = "meta-llama/Llama-3.2-1B"
    base_model = AutoModelForCausalLM.from_pretrained(
        base_model_name,
        device_map="auto",
        torch_dtype=torch.float32,
    )
    
    # Load the LoRA model
    lora_path = "g:/ML/tmp/llama-3.2-1B-finetuned-LoRA"
    model = PeftModel.from_pretrained(base_model, lora_path)
    
    # Merge LoRA weights with base model
    model = model.merge_and_unload()
    
    # Save the merged model
    output_path = "g:/ML/tmp/merged-llama-model-32bit"
    model.save_pretrained(output_path)
    
    # Save the tokenizer
    tokenizer = AutoTokenizer.from_pretrained(base_model_name)
    tokenizer.save_pretrained(output_path)
    
    print(f"Merged model saved to: {output_path}")