Updated p4d results using new int_mm kernel (#61)

Author: cpuhrsch

experiments/eval_combo.py

@@ -7,11 +7,6 @@
 import segment_anything_fast
 
 torch._dynamo.config.cache_size_limit = 50000
-# torch._inductor.config.fx_graph_cache = True # seems to slow performance
-torch._inductor.config.epilogue_fusion = False
-torch._inductor.config.coordinate_descent_tuning = True
-torch._inductor.config.coordinate_descent_check_all_directions = True
-torch._inductor.config.force_fuse_int_mm_with_mul = True
 
 def unbind_jagged(device, data, sizes, offsets):
     if data is None:
@@ -175,7 +170,8 @@ def build_results(batched_data_iter,
                   use_compile,
                   use_compile_decoder,
                   use_nested_tensor,
-                  pad_input_image_batch):
+                  pad_input_image_batch,
+                  use_fullgraph=False):
 
     # TODO: Re-enable this for datapoints
     assert not use_compile_decoder
@@ -197,7 +193,7 @@ def build_results(batched_data_iter,
             if batch_idx == 0:
                 with torch.autograd.profiler.record_function("compilation and warmup"):
                     if str(use_compile) != "False":
-                        predictor.model.image_encoder = torch.compile(predictor.model.image_encoder, mode=use_compile, fullgraph=True,)
+                        predictor.model.image_encoder = torch.compile(predictor.model.image_encoder, mode=use_compile, fullgraph=use_fullgraph)
                     # Run first batch a few times for warmup and exclude it from the final timings
                     for _ in range(3):
                         _ = batch_runner(predictor, batch, batch_size, pad_input_image_batch)
@@ -293,10 +289,17 @@ def run(
     profile_top=False,
     memory_path=None,
     use_local_sam_fork=False,
+    use_compiler_settings=False,
 ):
-    from torch._inductor import config as tritonconfig
-    tritonconfig.triton.unique_kernel_names = True
-    tritonconfig.epilogue_fusion_first = epilogue_fusion_first
+    from torch._inductor import config as inductorconfig
+    inductorconfig.triton.unique_kernel_names = True
+    inductorconfig.epilogue_fusion_first = epilogue_fusion_first
+
+    if use_compiler_settings:
+        # inductorconfig.fx_graph_cache = True # seems to slow performance
+        inductorconfig.epilogue_fusion = False
+        inductorconfig.coordinate_descent_tuning = True
+        inductorconfig.coordinate_descent_check_all_directions = True
 
     if use_half is not None:
         if use_half == "float16":
@@ -336,6 +339,7 @@ def run(
     if compress == "dynamic_quant":
         from segment_anything_fast.dynamic_quant import apply_dynamic_quant
         apply_dynamic_quant(predictor.model.image_encoder)
+        inductorconfig.force_fuse_int_mm_with_mul = True
     elif compress == "static_quant":
         from segment_anything_fast.static_quant import apply_static_quant
         apply_static_quant(predictor.model.image_encoder)

experiments/p4d_results/results_bs8.csv

@@ -5,5 +5,5 @@ compile,2.652463134129842,codesign,2.2.0.dev20231023+cu121,vit_b,8,7916,19,54.71
 SDPA,2.148758562405904,sdpa-decoder,2.2.0.dev20231023+cu121,vit_b,8,4679,11,73.1570663251564,13.669219533153035,0.5355346808697282,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,619,4952,None,None
 Triton,2.0386854648590087,local-fork,2.2.0.dev20231023+cu121,vit_b,8,1703,4,85.53658249838097,11.690904298391018,0.5339075529136259,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,619,4952,None,None
 NT,1.9225259701410928,local-fork,2.2.0.dev20231023+cu121,vit_b,8,2797,6,92.11983959049361,10.85542489484747,0.5337810700594795,max-autotune,torch.bfloat16,None,False,False,True,True,True,32,619,4952,None,None
-int8,4.885190570354462,local-fork,2.2.0.dev20231023+cu121,vit_b,8,2710,6,91.04449841914705,10.983640059130643,0.5331727804156572,max-autotune,torch.bfloat16,dynamic_quant,False,False,True,True,True,32,619,4952,None,None
+int8,3.0942590634028115,local-fork,2.2.0.dev20231026+cu121,vit_b,8,2712,6,90.34645860305645,11.068502467745533,0.5330263012027209,max-autotune,torch.bfloat16,dynamic_quant,False,False,True,True,True,32,619,4952,None,None
 sparse,3.841790223121643,local-fork,2.2.0.dev20231023+cu121,vit_b,8,3217,7,81.4912293589238,12.271259224665185,0.4783508911148021,max-autotune,torch.bfloat16,sparse,False,False,True,True,True,32,619,4952,None,None

experiments/p4d_results/results_bs8_vit_h.csv

@@ -5,5 +5,5 @@ compile,7.968364950021108,codesign,2.2.0.dev20231024+cu121,vit_h,8,12358,30,19.6
 SDPA,5.843019040425618,sdpa-decoder,2.2.0.dev20231024+cu121,vit_h,8,7947,19,21.92026495560376,45.61988653081299,0.581191777206921,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,619,4952,None,None
 Triton,9.09047209819158,local-fork,2.2.0.dev20231024+cu121,vit_h,8,4550,11,22.874989934428537,43.71586623060877,0.5820036887609843,max-autotune,torch.bfloat16,None,False,False,False,True,True,32,619,4952,None,None
 NT,5.455243261655172,local-fork,2.2.0.dev20231024+cu121,vit_h,8,4550,11,23.206823845253847,43.09077393219044,0.5809004559961229,max-autotune,torch.bfloat16,None,False,False,True,True,True,32,619,4952,None,None
-int8,8.623369554678598,local-fork,2.2.0.dev20231024+cu121,vit_h,8,3239,7,25.099473512089347,39.841473149559995,0.5820724009353484,max-autotune,torch.bfloat16,dynamic_quant,False,False,True,True,True,32,619,4952,None,None
+int8,6.994769084453583,local-fork,2.2.0.dev20231026+cu121,vit_h,8,4167,10,24.87583443921619,40.19965651578395,0.5819033780783904,max-autotune,torch.bfloat16,dynamic_quant,False,False,True,True,True,32,619,4952,None,None
 sparse,5.597406772772471,local-fork,2.2.0.dev20231024+cu121,vit_h,8,7055,17,24.900183397177024,40.16034677533225,0.5289167514647479,max-autotune,torch.bfloat16,sparse,False,False,True,True,True,32,619,4952,None,None

setup.py

@@ -7,8 +7,8 @@
     version='0.2',
     packages=packages,
     install_requires=[
-        'torch>=2.2.0.dev20231019',
-        'torchvision>=0.17.0.dev20231019',
+        'torch>=2.2.0.dev20231026',
+        'torchvision>=0.17.0.dev20231026',
         'diskcache',
         'pycocotools',
         'scipy',