[new] add mse higher is better to 1.2.1

Author: ClarkChin08

lpot/adaptor/tensorflow.py

@@ -62,6 +62,9 @@ def __init__(self, framework_specific_info):
             os.path.dirname(__file__), "tensorflow.yaml"))
         self.op_wise_sequences = self.query_handler.get_eightbit_patterns()
 
+        self.fp32_results = []
+        self.fp32_preds_as_label = False
+
     def log_histogram(self, writer, tag, values, step=0, bins=1000):
         import tensorflow as tf
         # Convert to a numpy array
@@ -177,9 +180,8 @@ def evaluate(self, model, dataloader, postprocess=None,
                 outputs.extend(int8_inspect_node_name)
 
         if metric and hasattr(metric, "compare_label") and not metric.compare_label:
-            results = [[] for _ in range(len(outputs))]
-            if not os.path.exists(os.path.join(self.work_dir, "output_tensors")):
-                os.makedirs(os.path.join(self.work_dir, "output_tensors"))
+            self.fp32_preds_as_label = True
+            results = []
 
         origin_output_tensor_names = model.output_tensor_names
         model.output_tensor_names = outputs
@@ -206,13 +208,10 @@ def evaluate(self, model, dataloader, postprocess=None,
             else:
                 predictions = model.sess.run(output_tensor, feed_dict)
 
-            if metric and hasattr(metric, "compare_label") and not metric.compare_label:
-                if isinstance(predictions, list):
-                    result = [np.array(value) for value in predictions.values()]
-                else:
-                    result = [predictions]
-                for i in range(len(outputs)):
-                    results[i].append(result[i])
+            if self.fp32_preds_as_label:
+                self.fp32_results.append(predictions) if fp32_baseline else \
+                    results.append(predictions)
+
             # Inspect node output, just get 1st iteration output tensors for now
             if idx == 0 and tensorboard:
                 for index, node_name in enumerate(outputs):
@@ -229,28 +228,22 @@ def evaluate(self, model, dataloader, postprocess=None,
                     predictions = predictions[:len(origin_output_tensor_names)]
             if postprocess is not None:
                 predictions, labels = postprocess((predictions, labels))
-            if metric is not None:
-                if not hasattr(metric, "compare_label"):
-                    metric.update(predictions, labels)
-                elif hasattr(metric, "compare_label") and metric.compare_label:
-                    metric.update(predictions, labels)
+            if metric is not None and not self.fp32_preds_as_label:
+                metric.update(predictions, labels)
             if idx + 1 == iteration:
                 break
-        if metric:
-            if hasattr(metric, "compare_label") and not metric.compare_label:
-                results = [np.array(result) for result in results]
-                metric.reset()
-                if fp32_baseline:
-                    np.savez(os.path.join(self.work_dir, "output_tensors", "fp32.npz"),
-                        *results)
-                    metric.update(results, results)
-                else:
-                    np.savez(os.path.join(self.work_dir, "output_tensors", "int8.npz"),
-                        *results)
-                    reference_file = np.load(os.path.join(self.work_dir, "output_tensors", \
-                                    "fp32.npz"), allow_pickle=True)
-                    reference = [reference_file[key] for key in reference_file]
-                    metric.update(reference, results)
+
+        if self.fp32_preds_as_label:
+            from .tf_utils.util import collate_tf_preds
+            metric.reset()
+            if fp32_baseline:
+                results = collate_tf_preds(self.fp32_results)
+                metric.update(results, results)
+            else:
+                reference = collate_tf_preds(self.fp32_results)
+                results = collate_tf_preds(results)
+                metric.update(results, reference)
+
         acc = metric.result() if metric is not None else 0
         if tensorboard:
             new_dir = temp_dir + "_acc_" + str(acc)

lpot/adaptor/tf_utils/util.py

@@ -226,15 +226,19 @@ def iterator_sess_run(sess, iter_op, feed_dict, output_tensor, iteration=-1, mea
         except tf.errors.OutOfRangeError:
             break
 
-    def collate_fn(results):
+    preds = collate_tf_preds(preds)
+    return preds
+
+def collate_tf_preds(results):
+    batch = results[0]
+    if isinstance(batch, list):
         results = zip(*results)
         collate_results = []
         for output in results:
            collate_results.append(np.concatenate(output))
-        return collate_results
-
-    preds = collate_fn(preds)
-    return preds
+    elif isinstance(batch, np.ndarray):
+        collate_results = np.concatenate(results)
+    return collate_results
 
 def get_input_node_names(graph_def):
     g = GraphAnalyzer()

lpot/conf/config.py

@@ -446,7 +446,7 @@ def percent_to_float(data):
     },
     Optional('tuning', default={
         'strategy': {'name': 'basic'},
-        'accuracy_criterion': {'relative': 0.01},
+        'accuracy_criterion': {'relative': 0.01, 'higher_is_better': True},
         'objective': 'performance',
         'exit_policy': {'timeout': 0, 'max_trials': 100, 'performance_only': False},
         'random_seed': 1978, 'tensorboard': False,
@@ -460,6 +460,7 @@ def percent_to_float(data):
         Optional('accuracy_criterion', default={'relative': 0.01}): {
             Optional('relative'): And(Or(str, float), Use(percent_to_float)),
             Optional('absolute'): And(Or(str, float), Use(percent_to_float)),
+            Optional('higher_is_better', default=True): bool,
         },
         Optional('objective', default='performance'): And(str, lambda s: s in OBJECTIVES),
         Optional('exit_policy', default={'timeout': 0,

lpot/experimental/metric/metric.py

@@ -115,7 +115,6 @@ def register(self, name, metric_cls):
         assert name not in self.metrics.keys(), 'registered metric name already exists.'
         self.metrics.update({name: metric_cls})
 
-
 def metric_registry(metric_type, framework):
     """The class decorator used to register all Metric subclasses.
        cross framework metric is supported by add param as framework='tensorflow, \
@@ -263,8 +262,8 @@ def _topk_shape_validate(preds, labels):
     return preds, labels
 
 def _shape_validate(preds, labels):
-    assert type(preds) in [int, list], 'preds must be in int or list'
-    assert type(labels) in [int, list], 'labels must be in int or list'
+    assert type(preds) in [int, list, np.ndarray], 'preds must be in int or list, ndarray'
+    assert type(labels) in [int, list, np.ndarray], 'labels must be in int or list, ndarray'
     if isinstance(preds, int):
         preds = [np.array([preds])]
     elif isinstance(preds[0], int):
@@ -471,10 +470,7 @@ def result(self):
         aes_sum = sum([np.sum(ae) for ae in aes])
         aes_size = sum([ae.size for ae in aes])
         assert aes_size, "predictions shouldn't be none"
-        if self.compare_label:
-            return aes_sum / aes_size
-        else:
-            return 1 / (aes_sum / aes_size + 0.001)
+        return aes_sum / aes_size
 
 @metric_registry('RMSE', 'tensorflow, mxnet, onnxrt_qlinearops, onnxrt_integerops')
 class RMSE(BaseMetric):
@@ -511,10 +507,7 @@ def result(self):
         squares_sum = sum([np.sum(square) for square in squares])
         squares_size = sum([square.size for square in squares])
         assert squares_size, "predictions should't be None"
-        if self.compare_label:
-            return squares_sum / squares_size
-        else:
-            return 1 / (squares_sum / squares_size + 0.001)
+        return squares_sum / squares_size
 
 @metric_registry('topk', 'tensorflow')
 class TensorflowTopK(BaseMetric):

lpot/objective.py

@@ -138,15 +138,19 @@ class Objective(object):
     """
 
     def __init__(self, accuracy_criterion, is_measure=False):
-        assert isinstance(
-            accuracy_criterion,
-            dict) and len(accuracy_criterion) == 1
-        k, v = list(accuracy_criterion.items())[0]
-        assert k in ['relative', 'absolute']
-        assert float(v) < 1 and float(v) > -1
-
-        self.acc_goal = float(v)
-        self.relative = True if k == 'relative' else False
+
+        assert isinstance(accuracy_criterion, dict), 'accuracy criterian should be dict'
+        assert 'relative' in accuracy_criterion or 'absolute' in accuracy_criterion, \
+            'accuracy criterion should set relative or absolute'
+        self.higher_is_better = True
+        for k, v in accuracy_criterion.items(): 
+            if k in ['relative', 'absolute']:
+                assert float(v) < 1 and float(v) > -1
+                self.relative = True if k == 'relative' else False
+                self.acc_goal = float(v)
+            elif k == 'higher_is_better':
+                self.higher_is_better = bool(v)
+
         self.baseline = None
         self.val = None
         self.is_measure = is_measure
@@ -158,7 +162,6 @@ def compare(self, last, baseline):
 
         Args:
             last (tuple): The tuple of last metric.
-            accuracy_criterion (float): The allowed accuracy absolute loss.
             baseline (tuple): The tuple saving FP32 baseline.
         """
         acc, perf = self.val
@@ -169,11 +172,16 @@ def compare(self, last, baseline):
             last_measure = 0
 
         base_acc, _ = baseline
+        
+        if self.relative:
+            acc_target = base_acc * (1 - float(self.acc_goal)) if self.higher_is_better \
+                else base_acc * (1 + float(self.acc_goal))
+        else:
+            acc_target =  base_acc - float(self.acc_goal) if self.higher_is_better \
+                else base_acc + float(self.acc_goal)
 
-        acc_target = base_acc - float(self.acc_goal) if not self.relative \
-            else base_acc * (1 - float(self.acc_goal))
-        if acc >= acc_target and (last_measure == 0 or perf < last_measure):
-            return True
+        if last_measure == 0 or perf < last_measure:
+            return acc < acc_target if self.higher_is_better else acc >= acc_target
         else:
             return False