Use subset metrics in all types of evaluators (#3979)

* Update base_evaluator.py

* Evaluators get metrics with get_config_metrics method only

* Update model evaluator unit tests

* Remove trailing-whitespace

* Add missing final new line

Author: pwolnows

tools/accuracy_checker/accuracy_checker/evaluators/base_evaluator.py

@@ -71,3 +71,19 @@ def write_results_to_csv(self, csv_file, ignore_results_formatting, metric_inter
             write_csv_result(
                 csv_file, processing_info, metrics_results, metric_interval, metrics_meta
             )
+
+    # get right metrics from config
+    @staticmethod
+    def get_config_metrics(config):
+        metrics = None
+        sub_evaluation = config.get('sub_evaluation', False)
+        if sub_evaluation:
+            size = config.get('subsample_size')
+            subset_metrics = config.get('subset_metrics', [])
+            for item in subset_metrics:
+                subset_size = item.get('subset_size')
+                if size is None or subset_size == size:
+                    # first subset_metrics or matching subsample_size
+                    metrics = item.get('metrics')
+                    break
+        return config.get('metrics', []) if (metrics is None) else metrics

tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py

@@ -106,7 +106,7 @@ def from_configs(cls, model_config, delayed_annotation_loading=False):
         postprocessor = PostprocessingExecutor(dataset_config.get('postprocessing'), dataset_name, dataset_metadata)
         metric_dispatcher = None
         if not delayed_annotation_loading:
-            metric_dispatcher = MetricsExecutor(get_config_metrics(dataset_config), dataset)
+            metric_dispatcher = MetricsExecutor(cls.get_config_metrics(dataset_config), dataset)
             if metric_dispatcher.profile_metrics:
                 metric_dispatcher.set_processing_info(ModelEvaluator.get_processing_info(model_config))
 
@@ -159,7 +159,7 @@ def validate_config(cls, model_config, delayed_annotation_loading=False):
                     )
                     config_errors.extend(
                         MetricsExecutor.validate_config(
-                            get_config_metrics(dataset_config), fetch_only=True,
+                            cls.get_config_metrics(dataset_config), fetch_only=True,
                             uri_prefix='{}.metrics'.format(current_dataset_uri))
                     )
 
@@ -227,7 +227,7 @@ def send_processing_info(self, sender):
         adapter_type = None
         if adapter:
             adapter_type = adapter if isinstance(adapter, str) else adapter.get('type')
-        metrics = get_config_metrics(dataset_config)
+        metrics = self.get_config_metrics(dataset_config)
         metric_info = [metric['type'] for metric in metrics]
         details.update({
             'metrics': metric_info,
@@ -773,10 +773,10 @@ def release(self):
     def provide_metric_references(cls, conf, return_header=True):
         processing_info = cls.get_processing_info(conf)
         dataset_config = conf['datasets'][0]
-        metric_dispatcher = MetricsExecutor(get_config_metrics(dataset_config), postpone_metrics=True)
+        metric_dispatcher = MetricsExecutor(cls.get_config_metrics(dataset_config), postpone_metrics=True)
         extracted_results, extracted_meta = [], []
         for result_presenter, metric_result in metric_dispatcher.get_metric_result_template(
-            get_config_metrics(dataset_config), False):
+            cls.get_config_metrics(dataset_config), False):
             result, metadata = result_presenter.extract_result(metric_result, names_from_refs=True)
             if isinstance(result, list):
                 extracted_results.extend(result)
@@ -788,18 +788,3 @@ def provide_metric_references(cls, conf, return_header=True):
         if not return_header:
             return report
         return header, report
-
-
-def get_config_metrics(config):
-    metrics = None
-    sub_evaluation = config.get('sub_evaluation', False)
-    if sub_evaluation:
-        size = config.get('subsample_size')
-        subset_metrics = config.get('subset_metrics', [])
-        for item in subset_metrics:
-            subset_size = item.get('subset_size')
-            if size is None or subset_size == size:
-                # first subset_metrics or matching subsample_size
-                metrics = item.get('metrics')
-                break
-    return config.get('metrics', []) if (metrics is None) else metrics

tools/accuracy_checker/accuracy_checker/evaluators/module_evaluator.py

@@ -147,10 +147,10 @@ def dataset_size(self):
     def provide_metric_references(cls, conf, return_header=True):
         processing_info = cls.get_processing_info(conf)
         dataset_config = conf['module_config']['datasets'][0]
-        metric_dispatcher = MetricsExecutor(dataset_config.get('metrics', []), postpone_metrics=True)
+        metric_dispatcher = MetricsExecutor(cls.get_config_metrics(dataset_config), postpone_metrics=True)
         extracted_results, extracted_meta = [], []
         for result_presenter, metric_result in metric_dispatcher.get_metric_result_template(
-            dataset_config.get('metrics', []), False):
+            cls.get_config_metrics(dataset_config), False):
             result, metadata = result_presenter.extract_result(metric_result, names_from_refs=True)
             if isinstance(result, list):
                 extracted_results.extend(result)

tools/accuracy_checker/accuracy_checker/evaluators/quantization_model_evaluator.py

@@ -609,6 +609,6 @@ def create_dataset_attributes(config, tag, dumped_annotations=None):
     )
     postprocessor = PostprocessingExecutor(dataset_config.get('postprocessing'), dataset_name, dataset_meta)
     if 'metrics' in dataset_config:
-        metric_dispatcher = MetricsExecutor(dataset_config.get('metrics', []), annotation_reader)
+        metric_dispatcher = MetricsExecutor(ModuleEvaluator.get_config_metrics(dataset_config), annotation_reader)
 
     return dataset, metric_dispatcher, preprocessor, postprocessor

tools/accuracy_checker/tests/test_model_evaluator.py

@@ -17,7 +17,6 @@
 from unittest.mock import Mock, MagicMock
 
 from accuracy_checker.evaluators import ModelEvaluator
-from accuracy_checker.evaluators.model_evaluator import get_config_metrics
 
 class TestModelEvaluator:
     def setup_method(self):
@@ -150,7 +149,7 @@ def test_model_evaluator_get_config_metrics(self, mocker):
                 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.65}]}]
         }
         metric = {'type': 'accuracy', 'top_k': 1, 'reference': 0.78}
-        selected_metric = get_config_metrics(dataset_config)[0]
+        selected_metric = ModelEvaluator.get_config_metrics(dataset_config)[0]
 
         assert metric['reference'] == selected_metric['reference']
         assert metric['top_k'] == selected_metric['top_k']
@@ -163,7 +162,7 @@ def test_model_evaluator_get_config_metrics_is_first_subset_metrics(self, mocker
                 {'subset_size': '20%', 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.72}]}]
         }
         subset_metric = {'type': 'accuracy', 'top_k': 5, 'reference': 0.65}
-        selected_metric = get_config_metrics(dataset_config_sub_evaluation)[0]
+        selected_metric = ModelEvaluator.get_config_metrics(dataset_config_sub_evaluation)[0]
 
         assert subset_metric['reference'] == selected_metric['reference']
         assert subset_metric['top_k'] == selected_metric['top_k']
@@ -176,7 +175,7 @@ def test_model_evaluator_get_config_metrics_with_subsample_size_from_subset_metr
                 {'subset_size': '20%', 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.72}]}]
         }
         subset_metric = {'type': 'accuracy', 'top_k': 5, 'reference': 0.72}
-        selected_metric = get_config_metrics(dataset_config_sub_evaluation)[0]
+        selected_metric = ModelEvaluator.get_config_metrics(dataset_config_sub_evaluation)[0]
 
         assert subset_metric['reference'] == selected_metric['reference']
         assert subset_metric['top_k'] == selected_metric['top_k']
@@ -189,7 +188,7 @@ def test_model_evaluator_get_config_metrics_from_subset_metrics(self, mocker):
                 'metrics': [{'type': 'accuracy', 'top_k': 5, 'reference': 0.65}]}]
         }
         subset_metric = {'type': 'accuracy', 'top_k': 5, 'reference': 0.65}
-        selected_metric = get_config_metrics(dataset_config_sub_evaluation)[0]
+        selected_metric = ModelEvaluator.get_config_metrics(dataset_config_sub_evaluation)[0]
 
         assert subset_metric['reference'] == selected_metric['reference']
         assert subset_metric['top_k'] == selected_metric['top_k']