some nits in lora (#208)

Author: awni

mlx_lm/examples/lora_config.yaml

@@ -1,5 +1,5 @@
 # The path to the local model directory or Hugging Face repo.
-model: "mlx_model"
+model: "mlx-community/Llama-3.2-1B-Instruct"
 
 # Whether or not to train (boolean)
 train: true
@@ -17,7 +17,7 @@ optimizer: adamw
 #     bias_correction: true
 
 # Directory with {train, valid, test}.jsonl files
-data: "/path/to/training/data"
+data: "mlx-community/WikiSQL"
 
 # The PRNG seed
 seed: 0

mlx_lm/fuse.py

@@ -75,7 +75,9 @@ def main() -> None:
     model = load_adapters(model, args.adapter_path)
 
     fused_linears = [
-        (n, m.fuse()) for n, m in model.named_modules() if hasattr(m, "fuse")
+        (n, m.fuse(de_quantize=args.de_quantize))
+        for n, m in model.named_modules()
+        if hasattr(m, "fuse")
     ]
 
     if fused_linears:

mlx_lm/lora.py

@@ -65,7 +65,7 @@
     "config": None,
     "grad_checkpoint": False,
     "lr_schedule": None,
-    "lora_parameters": {"rank": 8, "dropout": 0.0, "scale": 10.0},
+    "lora_parameters": {"rank": 8, "dropout": 0.0, "scale": 20.0},
     "mask_prompt": False,
     "wandb": None,
 }

mlx_lm/tuner/lora.py

@@ -52,9 +52,8 @@ def fuse(self, de_quantize: bool = False):
         output_dims, input_dims = weight.shape
         fused_linear = nn.Linear(input_dims, output_dims, bias=bias)
 
-        lora_b = (self.scale * self.lora_b.T).astype(dtype)
-        lora_a = self.lora_a.T.astype(dtype)
-        fused_linear.weight = weight + lora_b @ lora_a
+        delta = ((self.scale * self.lora_b.T) @ self.lora_a.T).astype(dtype)
+        fused_linear.weight = weight + delta
         if bias:
             fused_linear.bias = linear.bias