[fix] fix pruning bug, fix mxnet dataloader bug, fix pytorch save bug, fix benchmark optimization bug, fix get graph def bug

ClarkChin08 · ftian1 · commit 05be6056f8ec · 2021-04-01T23:16:28.000+08:00
diff --git a/examples/helloworld/tf_example1/README.md b/examples/helloworld/tf_example1/README.md
@@ -47,8 +47,8 @@ evaluation:                                          # optional. required if use
 3. Run quantizaiton
 We only need to add the following lines for quantization to create an int8 model.
 ```python
-    import lpot
-    quantizer = lpot.Quantization('./conf.yaml')
+    from lpot.experimental import Quantization, common
+    quantizer = Quantization('./conf.yaml')
     quantizer.model = common.Model("./mobilenet_v1_1.0_224_frozen.pb")
     quantized_model = quantizer()
 ```
diff --git a/examples/helloworld/tf_example1/test.py b/examples/helloworld/tf_example1/test.py
@@ -4,9 +4,10 @@
 import numpy as np
 def main():
 
-    import lpot
-    quantizer = lpot.Quantization('./conf.yaml')
-    quantized_model = quantizer("./mobilenet_v1_1.0_224_frozen.pb")
+    from lpot.experimental import Quantization, common
+    quantizer = Quantization('./conf.yaml')
+    quantizer.model = common.Model("./mobilenet_v1_1.0_224_frozen.pb")
+    quantized_model = quantizer()
       
 if __name__ == "__main__":
 
diff --git a/examples/helloworld/tf_example2/README.md b/examples/helloworld/tf_example2/README.md
@@ -55,7 +55,6 @@ class Dataset(object):
 ### 3.Define a customized metric  
 This customized metric will caculate accuracy.
 ```python
-import lpot
 from lpot.metric import Metric
 class MyMetric(Metric):
   def __init__(self, *args):
@@ -84,8 +83,7 @@ class MyMetric(Metric):
 ```
 ### 4.Use the customized data loader and metric for quantization 
 ```python
-import lpot
-quantizer = lpot.Quantization('./conf.yaml')
+quantizer = Quantization('./conf.yaml')
 dataset = Dataset()
 quantizer.metric = common.Metric(MyMetric, 'hello_metric')
 quantizer.calib_dataloader = common.DataLoader(dataset, batch_size=1)
diff --git a/examples/helloworld/tf_example2/test.py b/examples/helloworld/tf_example2/test.py
@@ -16,8 +16,7 @@ def __len__(self):
       return len(self.test_images)
 
 # Define a customized Metric function 
-import lpot
-from lpot.experimental import common
+from lpot.experimental import Quantization,  common
 from lpot.metric import BaseMetric
 class MyMetric(BaseMetric):
   def __init__(self, *args):
@@ -45,12 +44,13 @@ def result(self):
 
 
 # Quantize with customized dataloader and metric
-quantizer = lpot.Quantization('./conf.yaml')
+quantizer = Quantization('./conf.yaml')
 dataset = Dataset()
-quantizer.metric('helll_metric', MyMetric)
-dataloader = quantizer.dataloader(dataset, batch_size=1)
-q_model = quantizer('../models/simple_model', \
-    q_dataloader=dataloader, eval_dataloader=dataloader)
+quantizer.metric = common.Metric(MyMetric, 'hello_metric')
+quantizer.calib_dataloader = common.DataLoader(dataset, batch_size=1)
+quantizer.eval_dataloader = common.DataLoader(dataset, batch_size=1)
+quantizer.model = common.Model('../models/saved_model')
+q_model = quantizer()
 
 # Optional, run quantized model
 import tensorflow as tf
diff --git a/examples/helloworld/tf_example3/README.md b/examples/helloworld/tf_example3/README.md
@@ -30,8 +30,8 @@ quantization:                                        # optional. tuning constrai
           height: 224
           width: 224
 ......
-evaluation:                                          # optional. required if user doesn't provide eval_func in lpot.Quantization.
-  accuracy:                                          # optional. required if user doesn't provide eval_func in lpot.Quantization.
+evaluation:                                          # optional. required if user doesn't provide eval_func in Quantization.
+  accuracy:                                          # optional. required if user doesn't provide eval_func in Quantization.
     metric:
       topk: 1                                        # built-in metrics are topk, map, f1, allow user to register new metric.
     dataloader:
@@ -51,9 +51,8 @@ evaluation:                                          # optional. required if use
 3. Run quantizaiton
 * In order to do quanzation for slim models, we need to get graph from slim .ckpt first. 
 ```python
-    import lpot
-    from lpot import common
-    quantizer = lpot.Quantization('./conf.yaml')
+    from lpot.experimental import Quantization,  common
+    quantizer = Quantization('./conf.yaml')
 
     # Get graph from slim checkpoint
     from tf_slim.nets import inception
diff --git a/examples/helloworld/tf_example3/test.py b/examples/helloworld/tf_example3/test.py
@@ -6,9 +6,12 @@
 
 def main():
 
-    import lpot
-    quantizer = lpot.Quantization('./conf.yaml')
-    quantized_model = quantizer('./inception_v1.ckpt')
+    from lpot.experimental import Quantization,  common
+    quantizer = Quantization('./conf.yaml')
+
+    # Do quantization
+    quantizer.model = common.Model('./inception_v1.ckpt')
+    quantized_model = quantizer()
   
      
 if __name__ == "__main__":
diff --git a/examples/helloworld/tf_example4/README.md b/examples/helloworld/tf_example4/README.md
@@ -11,8 +11,7 @@ This example is used to demonstrate how to quantize a TensorFlow checkpoint and
 2. Run quantizaiton
 We will create a dummy dataloader and only need to add the following lines for quantization to create an int8 model.
     ```python
-    import lpot
-    quantizer = lpot.Quantization('./conf.yaml')
+    quantizer = Quantization('./conf.yaml')
     dataset = quantizer.dataset('dummy', shape=(100, 100, 100, 3), label=True)
     quantizer.model = common.Model('./model/public/rfcn-resnet101-coco-tf/rfcn_resnet101_coco_2018_01_28/')
     quantizer.calib_dataloader = common.DataLoader(dataset)
diff --git a/examples/helloworld/tf_example5/README.md b/examples/helloworld/tf_example5/README.md
@@ -8,8 +8,8 @@ wget https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_6/mobil
 2. Update the root of dataset in conf.ymal
 The configuration will will create a TopK metric function for evaluation and configure the batch size, instance number and core number for performacne measurement.    
 ```yaml
-evaluation:                                          # optional. required if user doesn't provide eval_func in lpot.Quantization.
- accuracy:                                          # optional. required if user doesn't provide eval_func in lpot.Quantization.
+evaluation:                                          # optional. required if user doesn't provide eval_func in Quantization.
+ accuracy:                                          # optional. required if user doesn't provide eval_func in Quantization.
     metric:
       topk: 1                                        # built-in metrics are topk, map, f1, allow user to register new metric.
     dataloader:
@@ -45,8 +45,7 @@ evaluation:                                          # optional. required if use
 3. Run quantizaiton
 We only need to add the following lines for quantization to create an int8 model.
 ```python
-    import lpot
-    quantizer = lpot.Quantization('./conf.yaml')
+    quantizer = Quantization('./conf.yaml')
     quantized_model = quantizer('./mobilenet_v1_1.0_224_frozen.pb')
 ```
 * Run quantization and evaluation:
@@ -57,8 +56,9 @@ We only need to add the following lines for quantization to create an int8 model
 4. Run benchmark accoridng to config
 ```python
      # Optional, run benchmark 
-    from lpot import Benchmark
+    from lpot.experimental import Quantization,  Benchmark, common
     evaluator = Benchmark('./conf.yaml')
-    results = evaluator(quantized_model)
+    evaluator.model = common.Model(quantized_model)
+    results = evaluator()
  
 ```
diff --git a/examples/helloworld/tf_example5/test.py b/examples/helloworld/tf_example5/test.py
@@ -4,15 +4,16 @@
 import numpy as np
 def main():
 
-    import lpot
-    quantizer = lpot.Quantization('./conf.yaml')
-    model_path = "./mobilenet_v1_1.0_224_frozen.pb"
-    quantized_model = quantizer(model_path)
+    from lpot.experimental import Quantization,  common
+    quantizer = Quantization('./conf.yaml')
+    quantizer.model = common.Model("./mobilenet_v1_1.0_224_frozen.pb")
+    quantized_model = quantizer()
 
      # Optional, run benchmark 
-    from lpot import Benchmark
+    from lpot.experimental import Benchmark
     evaluator = Benchmark('./conf.yaml')
-    results = evaluator(quantized_model)
+    evaluator.model = common.Model(quantized_model.graph_def)
+    results = evaluator()
     batch_size = 1
     for mode, result in results.items():
        acc, batch_size, result_list = result
diff --git a/examples/helloworld/tf_example6/README.md b/examples/helloworld/tf_example6/README.md
@@ -0,0 +1,64 @@
+tf_example5 example
+=====================
+This example is used to demonstrate how to config benchmark in yaml for performance measuremnt.
+
+1. Download the FP32 model
+wget https://storage.googleapis.com/intel-optimized-tensorflow/models/v1_6/mobilenet_v1_1.0_224_frozen.pb
+
+2. Update the root of dataset in conf.ymal
+The configuration will will create a TopK metric function for evaluation and configure the batch size, instance number and core number for performacne measurement.    
+```yaml
+evaluation:                                          # optional. required if user doesn't provide eval_func in Quantization.
+ accuracy:                                          # optional. required if user doesn't provide eval_func in Quantization.
+    metric:
+      topk: 1                                        # built-in metrics are topk, map, f1, allow user to register new metric.
+    dataloader:
+      batch_size: 32 
+      dataset:
+        ImageRecord:
+          root: /path/to/imagenet/          # NOTE: modify to evaluation dataset location if needed
+      transform:
+        ParseDecodeImagenet:
+        BilinearImagenet: 
+          height: 224
+          width: 224
+
+ performance:                                       # optional. used to benchmark performance of passing model.
+    configs:
+      cores_per_instance: 4
+      num_of_instance: 7
+    dataloader:
+      batch_size: 1 
+      last_batch: discard 
+      dataset:
+        ImageRecord:
+          root: /path/to/imagenet/          # NOTE: modify to evaluation dataset location if needed
+      transform:
+        ParseDecodeImagenet:
+        ResizeCropImagenet: 
+          height: 224
+          width: 224
+          mean_value: [123.68, 116.78, 103.94]
+
+```
+
+3. Run quantizaiton
+We only need to add the following lines for quantization to create an int8 model.
+```python
+    from lpot import Quantization
+    quantizer = Quantization('./conf.yaml')
+    quantized_model = quantizer('./mobilenet_v1_1.0_224_frozen.pb')
+```
+* Run quantization and evaluation:
+```shell
+    python test.py
+``` 
+
+4. Run benchmark accoridng to config
+```python
+     # Optional, run benchmark 
+    from lpot import Benchmark
+    evaluator = Benchmark('./conf.yaml')
+    results = evaluator(quantized_model)
+ 
+```
diff --git a/examples/helloworld/tf_example6/conf.yaml b/examples/helloworld/tf_example6/conf.yaml
@@ -0,0 +1,65 @@
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+model:                                               # mandatory. lpot uses this model name and framework name to decide where to save tuning history and deploy yaml.
+  name: mobilenet_v1 
+  framework: tensorflow                              # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension.
+
+quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
+  calibration:
+    sampling_size: 20                                # optional. default value is the size of whole dataset. used to set how many portions of calibration dataset is used. exclusive with iterations field.
+    dataloader:
+      batch_size: 1
+      dataset:
+        ImageRecord:
+          root: small_dataset
+      transform:
+        ParseDecodeImagenet: {}
+        BilinearImagenet: 
+          height: 224
+          width: 224
+
+evaluation:                                          # optional. required if user doesn't provide eval_func in lpot.Quantization.
+  accuracy:                                          # optional. required if user doesn't provide eval_func in lpot.Quantization.
+    metric:
+      topk: 1                                        # built-in metrics are topk, map, f1, allow user to register new metric.
+    dataloader:
+      batch_size: 32 
+      dataset:
+        ImageRecord:
+          root: small_dataset
+      transform:
+        ParseDecodeImagenet: {}
+        BilinearImagenet: 
+          height: 224
+          width: 224
+  performance:                                       # optional. used to benchmark performance of passing model.
+    configs:
+      cores_per_instance: 4
+      num_of_instance: 7
+    dataloader:
+      batch_size: 1 
+      last_batch: discard 
+      dataset:
+        ImageRecord:
+          root: small_dataset
+      transform:
+        ParseDecodeImagenet: {}
+        ResizeCropImagenet: 
+          height: 224
+          width: 224
+          mean_value: [123.68, 116.78, 103.94]
+
+  
diff --git a/examples/helloworld/tf_example6/dataset b/examples/helloworld/tf_example6/dataset
@@ -0,0 +1 @@
+/home/chen13/dataset
diff --git a/examples/helloworld/tf_example6/lpot_workspace/tensorflow/mobilenet_v1/deploy.yaml b/examples/helloworld/tf_example6/lpot_workspace/tensorflow/mobilenet_v1/deploy.yaml
@@ -0,0 +1,42 @@
+model:
+  name: mobilenet_v1
+  framework: tensorflow
+  inputs: []
+  outputs: []
+device: cpu
+evaluation:
+  accuracy:
+    metric:
+      topk: 1
+    dataloader:
+      batch_size: 32
+      dataset:
+        ImageRecord:
+          root: small_dataset
+      transform:
+        ParseDecodeImagenet: {}
+        BilinearImagenet:
+          height: 224
+          width: 224
+      last_batch: rollover
+  performance:
+    configs:
+      cores_per_instance: 4
+      num_of_instance: 7
+    dataloader:
+      batch_size: 1
+      last_batch: discard
+      dataset:
+        ImageRecord:
+          root: small_dataset
+      transform:
+        ParseDecodeImagenet: {}
+        ResizeCropImagenet:
+          height: 224
+          width: 224
+          mean_value:
+          - 123.68
+          - 116.78
+          - 103.94
+    warmup: 5
+    iteration: -1
diff --git a/examples/helloworld/tf_example6/small_dataset b/examples/helloworld/tf_example6/small_dataset
@@ -0,0 +1 @@
+/home/chen13/dataset/small_dataset
diff --git a/examples/helloworld/tf_example6/test.py b/examples/helloworld/tf_example6/test.py
@@ -0,0 +1,26 @@
+import tensorflow as tf
+tf.compat.v1.disable_eager_execution()
+
+import numpy as np
+def main():
+
+    from lpot import Quantization
+    quantizer = Quantization('./conf.yaml')
+    quantized_model = quantizer("./mobilenet_v1_1.0_224_frozen.pb")
+
+     # Optional, run benchmark 
+    from lpot import Benchmark
+    import pdb; pdb.set_trace()
+    evaluator = Benchmark('./conf.yaml')
+    results = evaluator(quantized_model)
+    batch_size = 1
+    for mode, result in results.items():
+       acc, batch_size, result_list = result
+       latency = np.array(result_list).mean() / batch_size
+
+       print('Accuracy is {:.3f}'.format(acc))
+       print('Latency: {:.3f} ms'.format(latency * 1000))
+      
+if __name__ == "__main__":
+
+    main()
diff --git a/examples/pytorch/image_recognition/imagenet/cpu/prune/main.py b/examples/pytorch/image_recognition/imagenet/cpu/prune/main.py
@@ -205,7 +205,6 @@ def training_func_for_lpot(model):
 
             return
         prune.model = common.Model(model)
-        prune.eval_dataloader = val_loader
         prune.q_func = training_func_for_lpot
         q_model = prune()
         return
diff --git a/lpot/adaptor/tensorflow.py b/lpot/adaptor/tensorflow.py
@@ -48,6 +48,7 @@ def __init__(self, framework_specific_info):
         self.device = self.framework_specific_info['device']
         self.work_dir = os.path.abspath(self.framework_specific_info['workspace_path'])
         self.recipes = self.framework_specific_info['recipes']
+        self.optimization = deep_get(self.framework_specific_info, 'optimization', {})
         if not os.path.exists(self.work_dir):
             os.makedirs(self.work_dir)
         
diff --git a/lpot/adaptor/tf_utils/util.py b/lpot/adaptor/tf_utils/util.py
diff --git a/lpot/data/__init__.py b/lpot/data/__init__.py
diff --git a/lpot/data/dataloaders/__init__.py b/lpot/data/dataloaders/__init__.py
diff --git a/lpot/data/dataloaders/dataloader.py b/lpot/data/dataloaders/dataloader.py
diff --git a/lpot/quantization.py b/lpot/quantization.py
