File tree Expand file tree Collapse file tree 3 files changed +10
-34
lines changed Expand file tree Collapse file tree 3 files changed +10
-34
lines changed Original file line number Diff line number Diff line change 6
6
from llmcompressor .transformers import oneshot
7
7
8
8
# Select model and load it.
9
+
10
+ # This script takes about 48 hours on 1xA100 to complete.
11
+ # Future improvements will reduce this runtime (#1561, #1558).
12
+
9
13
# For DeepSeek-R1, we require a full precision model in order to properly calibrate
10
14
# `DeepSeek-R1-0528-BF16` is a DeepSeek-V3 FP8 model which has been converted to BF16
15
+
11
16
model_id = "unsloth/DeepSeek-R1-0528-BF16"
12
17
config = AutoConfig .from_pretrained (model_id )
13
18
del config .quantization_config # fp8 qconfig no longer appplies to bf16 model
Original file line number Diff line number Diff line change 1
- import torch
1
+ from compressed_tensors . utils import replace_module
2
2
from transformers import PreTrainedModel
3
3
4
4
from llmcompressor .modeling .deepseek_v3 import replace as replace_DeepseekV3MoE
5
- from llmcompressor .utils .module import module_bfs
6
5
7
6
__all__ = ["prepare_for_calibration" ]
8
7
12
11
13
12
14
13
def prepare_for_calibration (model : PreTrainedModel ) -> PreTrainedModel :
15
- def replace ( module : torch . nn . Module ) -> torch . nn . Module :
14
+ for name , module in model . named_modules () :
16
15
cls_name = module .__class__ .__name__
17
16
if cls_name in replacements :
18
- return replacements [cls_name ](module )
19
- else :
20
- return module
17
+ new_module = replacements [cls_name ](module )
18
+ replace_module (model , name , new_module )
21
19
22
- return module_bfs ( model , replace , progress = True )
20
+ return model
Load Diff This file was deleted.
You can’t perform that action at this time.
0 commit comments