Fix onnxrt calibration zero-size array error (#1296)

yuwenzho · web-flow · commit 93094bd3025b · 2022-09-28T17:52:28.000+08:00
diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml
@@ -33,7 +33,7 @@ quantization:                                        # optional. tuning constrai
           width: 224
           mean_value: [0.485, 0.456, 0.406]
   op_wise: {
-     'Gather_*': {
+     'Gather_.*?': {
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml
@@ -33,7 +33,7 @@ quantization:                                        # optional. tuning constrai
           width: 224
           mean_value: [0.485, 0.456, 0.406]
   op_wise: {
-     'Gather_*': {
+     'Gather_.*?': {
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
diff --git a/examples/onnxrt/object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1.yaml b/examples/onnxrt/object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1.yaml
@@ -24,9 +24,9 @@ model:                                               # mandatory. used to specif
 quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
   approach: post_training_static_quant
   op_wise: {
-    'Preprocessor/*': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
+    'Preprocessor/.*?': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
     'add': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
-    'Postprocessor/*': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
+    'Postprocessor/.*?': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
  
   }
   calibration:
diff --git a/examples/onnxrt/object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1_qdq.yaml b/examples/onnxrt/object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1_qdq.yaml
@@ -24,9 +24,9 @@ model:                                               # mandatory. used to specif
 quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
   approach: post_training_static_quant
   op_wise: {
-    'Preprocessor/*': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
+    'Preprocessor/.*?': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
     'add': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
-    'Postprocessor/*': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
+    'Postprocessor/.*?': {'weight': {'dtype': ['fp32']}, 'activation': {'dtype': ['fp32']}},
  
   }
   calibration:
diff --git a/examples/onnxrt/object_detection/onnx_model_zoo/tiny_yolov3/quantization/ptq/tiny_yolov3.yaml b/examples/onnxrt/object_detection/onnx_model_zoo/tiny_yolov3/quantization/ptq/tiny_yolov3.yaml
@@ -24,7 +24,7 @@ model:                                               # mandatory. used to specif
 quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
   approach: post_training_static_quant
   op_wise: {
-    'TFNodes/yolo_evaluation*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'TFNodes/yolo_evaluation.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'conv2d_10': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'conv2d_13': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
   }
diff --git a/examples/onnxrt/object_detection/onnx_model_zoo/yolov3/quantization/ptq/yolov3.yaml b/examples/onnxrt/object_detection/onnx_model_zoo/yolov3/quantization/ptq/yolov3.yaml
@@ -24,7 +24,7 @@ model:                                               # mandatory. used to specif
 quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
   approach: post_training_static_quant
   op_wise: {
-    'TFNodes/yolo_evaluation*': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'TFNodes/yolo_evaluation.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'conv2d_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'leaky_re_lu_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'conv2d_59': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
diff --git a/examples/onnxrt/object_detection/onnx_model_zoo/yolov4/quantization/ptq/yolov4.yaml b/examples/onnxrt/object_detection/onnx_model_zoo/yolov4/quantization/ptq/yolov4.yaml
@@ -24,12 +24,12 @@ model:                                               # mandatory. used to specif
 quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
   approach: post_training_static_quant
   op_wise: {
-    'StatefulPartitionedCall/model/lambda*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
-    'StatefulPartitionedCall/model/tf_op_layer_add*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
-    'StatefulPartitionedCall/model/tf_op_layer_Sigmoid*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
-    'StatefulPartitionedCall/model/tf_op_layer_split*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
-    'StatefulPartitionedCall/model/tf_op_layer_Reshape*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
-    'Transpose*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'StatefulPartitionedCall/model/lambda.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'StatefulPartitionedCall/model/tf_op_layer_add.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'StatefulPartitionedCall/model/tf_op_layer_Sigmoid.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'StatefulPartitionedCall/model/tf_op_layer_split.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'StatefulPartitionedCall/model/tf_op_layer_Reshape.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
+    'Transpose.*?': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'StatefulPartitionedCall/model/conv2d_101/BiasAdd': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
     'StatefulPartitionedCall/model/conv2d_109/BiasAdd': {'activation':  {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},
   }
diff --git a/examples/onnxrt/object_detection/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1.yaml b/examples/onnxrt/object_detection/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1.yaml
@@ -57,7 +57,7 @@ quantization:                                        # optional. tuning constrai
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
-     'Gather__*': {
+     'Gather__.*?': {
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
diff --git a/examples/onnxrt/object_detection/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1_qdq.yaml b/examples/onnxrt/object_detection/ssd_mobilenet_v1/quantization/ptq/ssd_mobilenet_v1_qdq.yaml
@@ -57,7 +57,7 @@ quantization:                                        # optional. tuning constrai
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
-     'Gather__*': {
+     'Gather__.*?': {
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
diff --git a/examples/onnxrt/object_detection/ssd_mobilenet_v2/quantization/ptq/ssd_mobilenet_v2.yaml b/examples/onnxrt/object_detection/ssd_mobilenet_v2/quantization/ptq/ssd_mobilenet_v2.yaml
@@ -61,7 +61,7 @@ quantization:                                        # optional. tuning constrai
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
-    'Gather__*': {
+    'Gather__.*?': {
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
diff --git a/examples/onnxrt/object_detection/ssd_mobilenet_v2/quantization/ptq/ssd_mobilenet_v2_qdq.yaml b/examples/onnxrt/object_detection/ssd_mobilenet_v2/quantization/ptq/ssd_mobilenet_v2_qdq.yaml
@@ -61,7 +61,7 @@ quantization:                                        # optional. tuning constrai
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
-    'Gather__*': {
+    'Gather__.*?': {
      'activation':  {'dtype': ['fp32']},
      'weight': {'dtype': ['fp32']}
      },
diff --git a/neural_compressor/adaptor/ox_utils/onnxrt_mid.py b/neural_compressor/adaptor/ox_utils/onnxrt_mid.py
@@ -172,9 +172,10 @@ def augment_graph(self, activation_only=False, weight_only=False):
                                 added_tensor.name = tensor
                                 added_outputs.append(added_tensor)
             else:
-                added_tensor = helper.ValueInfoProto()
-                added_tensor.name = tensor
-                added_outputs.append(added_tensor)
+                if tensor not in [t.name for t in model.graph.output]:
+                    added_tensor = helper.ValueInfoProto()
+                    added_tensor.name = tensor
+                    added_outputs.append(added_tensor)
 
         if self.augment_nodes:
             model.graph.node.extend(added_nodes) # pylint: disable=no-member
@@ -199,6 +200,12 @@ def get_intermediate_outputs(self):
         intermediate_outputs = []
         len_inputs = len(session.get_inputs())
         inputs_names = [session.get_inputs()[i].name for i in range(len_inputs)]
+        output_dicts = {}
+        
+        node_output_names = [output.name if output.name not in self.dequantized_output \
+                             else self.dequantized_output[output.name] \
+                             for output in session.get_outputs()]
+
         for idx, (inputs, labels) in enumerate(self.dataloader):
             ort_inputs = {}
             if len_inputs == 1:
@@ -220,26 +227,17 @@ def get_intermediate_outputs(self):
                 if idx > max(self.iterations):
                     break
                 if idx in self.iterations:
-                    intermediate_output= session.run(None, ort_inputs)
-                    min_maxs = []
-                    for output in intermediate_output:
-                        min_maxs.append([output.min(), output.max()])
-                    intermediate_outputs.append(min_maxs)
+                    for output_idx, output in enumerate(session.run(None, ort_inputs)): 
+                        if output.size != 0:
+                            output_dicts.setdefault(node_output_names[output_idx], \
+                                []).append([output.min(), output.max()])
             else:
-                intermediate_output= session.run(None, ort_inputs)
-                min_maxs = []
-                for output in intermediate_output:
-                    min_maxs.append([output.min(), output.max()])
-                intermediate_outputs.append(min_maxs)
-        node_output_names = [output.name if output.name not in self.dequantized_output \
-                             else self.dequantized_output[output.name] \
-                             for output in session.get_outputs()]
-        output_dicts_list = [
-            dict(zip(node_output_names, intermediate_output)) for intermediate_output in
-            intermediate_outputs
-        ]
+                for output_idx, output in enumerate(session.run(None, ort_inputs)): 
+                    if output.size != 0:
+                        output_dicts.setdefault(node_output_names[output_idx], \
+                            []).append([output.min(), output.max()])
 
-        return node_output_names, output_dicts_list
+        return list(output_dicts.keys()), output_dicts
 
     def _dequantize(self, tensor, scale_tensor, zo_tensor):
         ''' helper function to dequantize tensor
@@ -320,16 +318,12 @@ def _add_dequantize_transpose_node(self, tensor_name, scale_tensor, zo_tensor, d
         added_nodes = [pre_transpose_node, dequantize_node, post_transpose_node]
         return added_nodes, tensor_name + '_output'
 
-    def _map_calibration(self, node_output_names, output_dicts_list, calib_mode='naive'):
-        model = self.model
-        num_model_outputs = len(model.graph.output)
+    def _map_calibration(self, node_output_names, output_dicts, calib_mode='naive'):
         merged_dict = {}
-        for d in output_dicts_list:
-            for k, v in d.items():
-                merged_dict.setdefault(k + '_Min', []).append(v[0])
-                merged_dict.setdefault(k + '_Max', []).append(v[1])
-        added_node_output_names = node_output_names[num_model_outputs:]
-        node_names = added_node_output_names  # output names
+        for name, minmaxs in output_dicts.items():
+            for minmax in minmaxs:
+                merged_dict.setdefault(name + '_Min', []).append(minmax[0])
+                merged_dict.setdefault(name + '_Max', []).append(minmax[1])
 
         # Characterizing distribution of a node's values across test data sets
         clean_merged_dict = dict((i, merged_dict[i]) for i in merged_dict)
@@ -338,20 +332,20 @@ def _map_calibration(self, node_output_names, output_dicts_list, calib_mode='nai
                 tuple([
                     float(min(clean_merged_dict[name + '_Min'])),
                     float(max(clean_merged_dict[name + '_Max']))
-                ]) for name in added_node_output_names
+                ]) for name in node_output_names
             ]
         else:
             raise ValueError('Unknown value for calib_mode. \
                              Currently only naive mode is supported.')
 
-        final_dict = dict(zip(node_names, pairs))
+        final_dict = dict(zip(node_output_names, pairs))
 
         return final_dict
 
     def dump_minmax(self, calib_mode='naive'):
         self.augment_graph()
-        node_output_names, output_dicts_list = self.get_intermediate_outputs()
-        return self._map_calibration(node_output_names, output_dicts_list,
+        node_output_names, output_dicts = self.get_intermediate_outputs()
+        return self._map_calibration(node_output_names, output_dicts,
                                      calib_mode=calib_mode)
 
     def dump_calibration(self, calib_mode='naive'):
@@ -422,14 +416,8 @@ def dump_tensor(self, activation=True, weight=False):
             self.dynamically_quantized = \
                 "DynamicQuantizeLinear" in [node.op_type for node in self.model.graph.node]
         self.augment_graph(activation_only=not weight, weight_only=not activation)
-        _, output_dicts_list = self.get_intermediate_outputs()
-        output_dicts = {}
-        for output_dicts_iter in output_dicts_list:
-            for output_name in output_dicts_iter:
-                if output_name not in output_dicts:
-                    output_dicts[output_name] = []
-                output_dicts[output_name].append(output_dicts_iter[output_name])
-        iters = len(output_dicts_list)
+        _, output_dicts = self.get_intermediate_outputs()
+        iters = len(list(output_dicts.values())[-1])
         map_node_activation = [{} for _ in range(iters)]
         map_node_weight = {}
         self.white_nodes = [node.replace('_quant', '') for node in self.white_nodes]
diff --git a/test/adaptor/onnxrt_adaptor/test_onnxrt_augment.py b/test/adaptor/onnxrt_adaptor/test_onnxrt_augment.py
@@ -190,8 +190,8 @@ def test_augment_graph(self):
         added_outputs = ['A', 'B', 'C', 'D', 'E', 'F']
         # Original 3 nodes (exlude graph input/output)
         self.assertEqual(len(augmented_model_node_names), 3)
-        # Original 1 graph output + 6 intermediate outputs
-        self.assertEqual(len(augmented_model_outputs), 7)
+        # Original 1 graph output + 5 intermediate outputs
+        self.assertEqual(len(augmented_model_outputs), 6)
         for name in added_node_names:
             self.assertTrue(name in augmented_model_node_names)
         for output in added_outputs:
@@ -228,8 +228,8 @@ def test_augment_graph(self):
         added_outputs = ['I', 'J', 'H', 'G', 'K']
         # Original 2 nodes
         self.assertEqual(len(augmented_model_node_names), 2)
-        # Original 1 graph output + 5 intermediate outputs
-        self.assertEqual(len(augmented_model_outputs), 6)
+        # Original 1 graph output + 4 intermediate outputs
+        self.assertEqual(len(augmented_model_outputs), 5)
         for name in added_node_names:
             self.assertTrue(name in augmented_model_node_names)
         for output in added_outputs:
@@ -270,8 +270,8 @@ def test_augment_graph(self):
         added_outputs =  ['P', 'M', 'N', 'O', 'Q']
         # Original 4 nodes
         self.assertEqual(len(augmented_model_node_names), 4)
-        # Original 1 graph output + 5 intermediate outputs
-        self.assertEqual(len(augmented_model_outputs), 6)
+        # Original 1 graph output + 4 intermediate outputs
+        self.assertEqual(len(augmented_model_outputs), 5)
         for name in added_node_names:
             self.assertTrue(name in augmented_model_node_names)
         for output in added_outputs:
@@ -308,8 +308,8 @@ def test_augment_graph(self):
         added_outputs = ['R', 'Attention_mask', 'S', 'T', 'Attention_bias', 'Attention_weight']
         # Original 2 nodes
         self.assertEqual(len(augmented_model_node_names), 2)
-        # Original 1 graph output + 6 intermediate outputs
-        self.assertEqual(len(augmented_model_outputs), 7)
+        # Original 1 graph output + 5 intermediate outputs
+        self.assertEqual(len(augmented_model_outputs), 6)
         for name in added_node_names:
             self.assertTrue(name in augmented_model_node_names)
         for output in added_outputs:
@@ -385,9 +385,12 @@ def test_augment_graph(self):
 
         augmented_model_node_names = [node.name for node in augmented_model.graph.node]
         augmented_model_outputs = [output.name for output in augmented_model.graph.output]
-        added_outputs = ['attn_output']
+        added_node_names = ['attention_quant', 'attn_output_QuantizeLinear']
+        added_outputs = ['attn_output', 'output']
         self.assertEqual(len(augmented_model_node_names), 2)
         self.assertEqual(len(augmented_model_outputs), 2)
+        for name in added_node_names:
+            self.assertTrue(name in augmented_model_node_names)
         for output in added_outputs:
             self.assertTrue(output in augmented_model_outputs)
 
@@ -446,8 +449,8 @@ def test_augment_graph(self):
 
         augmented_model_node_names = [node.name for node in augmented_model.graph.node]
         augmented_model_outputs = [output.name for output in augmented_model.graph.output]
-        added_node_names = ['D_quantized_DequantizeLinear']
-        added_outputs = ['D_quantized_output']
+        added_node_names = ['A_QuantizeLinear', 'conv_quant', 'D_DequantizeLinear', 'D_quantized_DequantizeLinear']
+        added_outputs = ['D', 'D_quantized_output']
         self.assertEqual(len(augmented_model_node_names), 4)
         self.assertEqual(len(augmented_model_outputs), 2)
         for name in added_node_names:
@@ -503,7 +506,7 @@ def test_quant_param_calculation(self):
         node_output_names, output_dicts_list = augment.get_intermediate_outputs()
         dict_for_quantization = augment._map_calibration(node_output_names, output_dicts_list)
         #check the size of the quantization dictionary
-        self.assertEqual(len(quantization_params_dict), 11)
+        self.assertEqual(len(quantization_params_dict), 12)
         
         #check the computation of zp and scale
         for key, value in quantization_params_dict.items():

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ model: # mandatory. used to specif`
`24`	`24`	`quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space.`
`25`	`25`	`approach: post_training_static_quant`
`26`	`26`	`op_wise: {`
`27`		`- 'TFNodes/yolo_evaluation*?': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},`
	`27`	`+ 'TFNodes/yolo_evaluation.*?': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},`
`28`	`28`	`'conv2d_10': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},`
`29`	`29`	`'conv2d_13': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}},`
`30`	`30`	`}`