use random-hadamard, add correctness tests

Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>

Author: kylesayrs

examples/transform/llama3_example.py

@@ -58,7 +58,7 @@ def tokenize(sample):
 #   * quantize the weights to 4 bit with GPTQ with a group size 128
 recipe = [
     TransformModifier(),
-    GPTQModifier(targets="Linear", scheme="W4A16", ignore=["lm_head"])
+    GPTQModifier(targets="Linear", scheme="W4A16", ignore=["lm_head"]),
 ]
 
 # Apply algorithms.

src/llmcompressor/modifiers/transform/__init__.py

@@ -1,3 +1,3 @@
 # flake8: noqa
 
-from .transform import TransformModifier
+from .transform import TransformModifier

src/llmcompressor/modifiers/transform/template/quip.py

@@ -1,10 +1,9 @@
-from compressed_tensors.transform import TransformArgs, TransformScheme, TransformConfig
-
+from compressed_tensors.transform import TransformArgs, TransformConfig, TransformScheme
 
 QUIP = TransformConfig(
     config_groups={
         "v": TransformScheme(
-            type="hadamard",
+            type="random-hadamard",
             apply=[
                 TransformArgs(
                     targets=["Linear"],
@@ -21,7 +20,7 @@
             randomize=True,
         ),
         "u": TransformScheme(
-            type="hadamard",
+            type="random-hadamard",
             apply=[
                 TransformArgs(
                     targets=["Linear"],
@@ -32,10 +31,10 @@
                     targets=["Linear"],
                     location="output",  # non-mergable
                     inverse=True,
-                    ignore="lm_head"
+                    ignore="lm_head",
                 ),
             ],
             randomize=True,
         ),
     }
-)
+)

src/llmcompressor/modifiers/transform/template/spinquant.py

@@ -1,5 +1,4 @@
-from compressed_tensors.transform import TransformArgs, TransformScheme, TransformConfig
-
+from compressed_tensors.transform import TransformArgs, TransformConfig, TransformScheme
 
 LLAMA_SPINQUANT = TransformConfig(
     transform_groups={
@@ -62,4 +61,4 @@
             ],
         ),
     }
-)
+)

src/llmcompressor/modifiers/transform/transform.py

@@ -1,12 +1,13 @@
 from typing import Dict, Optional
 
+from compressed_tensors.transform import TransformScheme, apply_transform_config
+
 from llmcompressor.core import State
 from llmcompressor.modifiers import Modifier
 
-from compressed_tensors.transform import TransformConfig, TransformScheme, TransformFactory
-
 from .template.quip import QUIP
 
+
 class TransformModifier(Modifier):
     preset_config: Optional[str] = None
     config_groups: Optional[Dict[str, TransformScheme]] = None
@@ -18,11 +19,8 @@ def on_initialize(self, state: State, **kwargs):
             # import config template and customize to model
             pass
 
-        
-        #config = TransformConfig(config_groups=self.config_groups)
+        # config = TransformConfig(config_groups=self.config_groups)
         config = QUIP
 
-        # TODO: use CT-provided apply_transform_config
-        for name, scheme in config.config_groups.items():
-            factory = TransformFactory.from_scheme(scheme, name=name)
-            factory.apply_to_model(state.model)
+        apply_transform_config(state.model, config)
+        breakpoint()