[fix] fix wide and deep and add onnx new mse higer is better

Author: ClarkChin08

lpot/adaptor/onnxrt.py

@@ -52,6 +52,9 @@ def __init__(self, framework_specific_info):
         self.quantizable_op_types = self._query_quantizable_op_types()
         self.evaluate_nums = 0
 
+        self.fp32_results = []
+        self.fp32_preds_as_label = False
+
     @dump_elapsed_time("Pass quantize model")
     def quantize(self, tune_cfg, model, dataLoader, q_func=None):
         """The function is used to do calibration and quanitization in post-training
@@ -286,12 +289,13 @@ def evaluate(self, input_graph, dataloader, postprocess=None,
         """
         session = ort.InferenceSession(input_graph.model.SerializeToString(), None)
         len_outputs = len(session.get_outputs())
+
         if metric:
-            if hasattr(metric, "compare_label"):
-                if not metric.compare_label:
-                    results = [[] for _ in range(len_outputs)]
-                    if not os.path.exists(os.path.join(self.work_space, "output_tensors")):
-                        os.makedirs(os.path.join(self.work_space, "output_tensors"))
+            metric.reset()
+            if hasattr(metric, "compare_label") and not metric.compare_label:
+                self.fp32_preds_as_label = True
+                results = []
+
         ort_inputs = {}
         len_inputs = len(session.get_inputs())
         inputs_names = [session.get_inputs()[i].name for i in range(len_inputs)]
@@ -311,37 +315,28 @@ def evaluate(self, input_graph, dataloader, postprocess=None,
                 for i in range(len_inputs):
                     ort_inputs.update({inputs_names[i]: batch[i]})
                 predictions = session.run(None, ort_inputs)
-            if metric:
-                if hasattr(metric, "compare_label"):
-                    if not metric.compare_label:
-                        for i in range(len_outputs):
-                            results[i].append(predictions[i])
+
+            if self.fp32_preds_as_label:
+                self.fp32_results.append(predictions) if fp32_baseline else \
+                    results.append(predictions)
 
             if postprocess is not None:
                 predictions, labels = postprocess((predictions, labels))
-            if metric is not None:
-                if not hasattr(metric, "compare_label"):
-                    metric.update(predictions, labels)
-                elif hasattr(metric, "compare_label") and metric.compare_label:
-                    metric.update(predictions, labels)
+            if metric is not None and not self.fp32_preds_as_label:
+                metric.update(predictions, labels)
             if idx + 1 == iteration:
                 break
-        if metric:
-            if hasattr(metric, "compare_label"):
-                if not metric.compare_label:
-                    metric.reset()
-                    results = [np.array(result) for result in results]
-                    if fp32_baseline:
-                        np.savez(os.path.join(self.work_space,"output_tensors", "fp32.npz"),
-                            *results)
-                        metric.update(results, results)
-                    else:
-                        np.savez(os.path.join(self.work_space,"output_tensors", "int8.npz"),
-                            *results)
-                        reference_file = np.load(os.path.join(self.work_space, "output_tensors", \
-                                     "fp32.npz"), allow_pickle=True)
-                        reference = [reference_file[key] for key in reference_file]
-                        metric.update(reference, results)
+
+        if self.fp32_preds_as_label:
+            from .ox_utils.util import collate_preds
+            if fp32_baseline:
+                results = collate_preds(self.fp32_results)
+                metric.update(results, results)
+            else:
+                reference = collate_preds(self.fp32_results)
+                results = collate_preds(results)
+                metric.update(results, reference)
+
         acc = metric.result() if metric is not None else 0
         return acc
 

lpot/adaptor/ox_utils/util.py

@@ -0,0 +1,31 @@
+#
+#  -*- coding: utf-8 -*-
+#
+#  Copyright (c) 2021 Intel Corporation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+import os
+import numpy as np
+
+def collate_preds(results):
+    batch = results[0]
+    if isinstance(batch, list):
+        results = zip(*results)
+        collate_results = []
+        for output in results:
+           collate_results.append(np.concatenate(output))
+    elif isinstance(batch, np.ndarray):
+        collate_results = np.concatenate(results)
+    return collate_results

lpot/adaptor/tensorflow.py

@@ -179,9 +179,11 @@ def evaluate(self, model, dataloader, postprocess=None,
                 output_postfix = "_int8.output"
                 outputs.extend(int8_inspect_node_name)
 
-        if metric and hasattr(metric, "compare_label") and not metric.compare_label:
-            self.fp32_preds_as_label = True
-            results = []
+        if metric: 
+            metric.reset()
+            if hasattr(metric, "compare_label") and not metric.compare_label:
+                self.fp32_preds_as_label = True
+                results = []
 
         origin_output_tensor_names = model.output_tensor_names
         model.output_tensor_names = outputs
@@ -235,7 +237,6 @@ def evaluate(self, model, dataloader, postprocess=None,
 
         if self.fp32_preds_as_label:
             from .tf_utils.util import collate_tf_preds
-            metric.reset()
             if fp32_baseline:
                 results = collate_tf_preds(self.fp32_results)
                 metric.update(results, results)

lpot/model/model.py

@@ -329,7 +329,7 @@ def __init__(self, model, framework_specific_info={}, **kwargs):
         self.workspace_path = deep_get(framework_specific_info, 'workspace_path', './')
         self.kwargs = copy.deepcopy(kwargs)
         kwargs.update({'name': deep_get(framework_specific_info, 'name')})
-        self.model = model
+        # self.model = model
         self.sess, self._input_tensor_names, self._output_tensor_names = \
             create_session_with_input_output(
                 model, input_tensor_names, output_tensor_names, **kwargs)
@@ -342,6 +342,10 @@ def __init__(self, model, framework_specific_info={}, **kwargs):
         if 'MakeIterator' in [node.op for node in self.sess.graph.as_graph_def().node]:
             self.iter_op = self.sess.graph.get_operation_by_name('MakeIterator')
 
+    @property
+    def model(self):
+        return self.sess.graph
+
     @property
     def graph_def(self):
         return self.sess.graph.as_graph_def()

lpot/quantization.py

@@ -155,9 +155,9 @@ def eval_func(model):
         elif eval_dataloader is not None:
             self.exp_quantizer.eval_dataloader = eval_dataloader
 
-        if self.exp_quantizer.framework == 'tensorflow':
-            return self.exp_quantizer().graph
         lpot_model = self.exp_quantizer()
+        if self.exp_quantizer.framework == 'tensorflow':
+            return lpot_model.graph if lpot_model else None
         if self.exp_quantizer.framework == 'pytorch':
             saved_path = os.path.abspath(os.path.join(os.path.expanduser(
                 self.exp_quantizer.conf.usr_cfg.tuning.workspace.path), 'checkpoint'))