Skip to content

Commit 1c371d0

Browse files
alalekalalek
committed
dnn(test): adjust tests for OpenVINO 2020.4
1 parent 55e8549 commit 1c371d0

File tree

7 files changed

+80
-7
lines changed

7 files changed

+80
-7
lines changed

‎modules/dnn/perf/perf_net.cpp

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,13 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv3)
196196
{
197197
if (backend == DNN_BACKEND_HALIDE)
198198
throw SkipTestException("");
199+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
200+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
201+
throw SkipTestException("Test is disabled in OpenVINO 2020.4");
202+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
203+
throw SkipTestException("Test is disabled in OpenVINO 2020.4");
204+
#endif
205+
199206
Mat sample = imread(findDataFile("dnn/dog416.png"));
200207
cvtColor(sample, sample, COLOR_BGR2RGB);
201208
Mat inp;
@@ -209,6 +216,12 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv4)
209216
throw SkipTestException("");
210217
if (target == DNN_TARGET_MYRIAD)
211218
throw SkipTestException("");
219+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
220+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
221+
throw SkipTestException("Test is disabled in OpenVINO 2020.4");
222+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
223+
throw SkipTestException("Test is disabled in OpenVINO 2020.4");
224+
#endif
212225
Mat sample = imread(findDataFile("dnn/dog416.png"));
213226
cvtColor(sample, sample, COLOR_BGR2RGB);
214227
Mat inp;
@@ -220,8 +233,6 @@ PERF_TEST_P_(DNNTestNetwork, YOLOv4_tiny)
220233
{
221234
if (backend == DNN_BACKEND_HALIDE)
222235
throw SkipTestException("");
223-
if (target == DNN_TARGET_MYRIAD)
224-
throw SkipTestException("");
225236
Mat sample = imread(findDataFile("dnn/dog416.png"));
226237
cvtColor(sample, sample, COLOR_BGR2RGB);
227238
Mat inp;

‎modules/dnn/test/test_backends.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -390,6 +390,8 @@ TEST_P(DNNTestNetwork, DenseNet_121)
390390
if (target == DNN_TARGET_OPENCL_FP16)
391391
{
392392
l1 = 2e-2; lInf = 9e-2;
393+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)
394+
lInf = 0.1f;
393395
}
394396
else if (target == DNN_TARGET_MYRIAD)
395397
{

‎modules/dnn/test/test_caffe_importer.cpp

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -489,6 +489,11 @@ TEST_P(Test_Caffe_nets, Colorization)
489489
{
490490
l1 = 0.5; lInf = 11;
491491
}
492+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
493+
{
494+
l1 = 0.26; lInf = 6.5;
495+
}
496+
492497
normAssert(out, ref, "", l1, lInf);
493498
expectNoFallbacksFromIE(net);
494499
}
@@ -515,8 +520,8 @@ TEST_P(Test_Caffe_nets, DenseNet_121)
515520
float l1 = default_l1, lInf = default_lInf;
516521
if (target == DNN_TARGET_OPENCL_FP16)
517522
{
518-
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2019020000)
519-
l1 = 0.04; lInf = 0.21;
523+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019020000)
524+
l1 = 0.045; lInf = 0.21;
520525
#else
521526
l1 = 0.017; lInf = 0.0795;
522527
#endif

‎modules/dnn/test/test_darknet_importer.cpp

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,12 @@ TEST_P(Test_Darknet_nets, YoloVoc)
323323
CV_TEST_TAG_LONG
324324
);
325325

326+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
327+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
328+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
329+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
330+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
331+
#endif
326332
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_GE(2019010000)
327333
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL_FP16)
328334
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16);
@@ -363,6 +369,12 @@ TEST_P(Test_Darknet_nets, TinyYoloVoc)
363369
{
364370
applyTestTag(CV_TEST_TAG_MEMORY_512MB);
365371

372+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
373+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
374+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
375+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
376+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
377+
#endif
366378
#if defined(INF_ENGINE_RELEASE)
367379
if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 || backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) &&
368380
target == DNN_TARGET_MYRIAD && getInferenceEngineVPUType() == CV_DNN_INFERENCE_ENGINE_VPU_TYPE_MYRIAD_X)
@@ -466,6 +478,13 @@ TEST_P(Test_Darknet_nets, YOLOv3)
466478
{
467479
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
468480

481+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
482+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
483+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
484+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
485+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
486+
#endif
487+
469488
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_MYRIAD)
470489
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH);
471490

@@ -530,6 +549,12 @@ TEST_P(Test_Darknet_nets, YOLOv4)
530549
{
531550
applyTestTag(CV_TEST_TAG_LONG, (target == DNN_TARGET_CPU ? CV_TEST_TAG_MEMORY_1GB : CV_TEST_TAG_MEMORY_2GB));
532551

552+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000) // nGraph compilation failure
553+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
554+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
555+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
556+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
557+
#endif
533558
#if defined(INF_ENGINE_RELEASE)
534559
if (target == DNN_TARGET_MYRIAD) // NC_OUT_OF_MEMORY
535560
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
@@ -626,7 +651,7 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny)
626651
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL)
627652
iouDiff = std::numeric_limits<double>::quiet_NaN();
628653
if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
629-
backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && DNN_TARGET_OPENCL_FP16)
654+
backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16)
630655
iouDiff = std::numeric_limits<double>::quiet_NaN();
631656
#endif
632657

@@ -646,7 +671,7 @@ TEST_P(Test_Darknet_nets, YOLOv4_tiny)
646671
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && target == DNN_TARGET_OPENCL)
647672
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
648673
if ((backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 ||
649-
backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && DNN_TARGET_OPENCL_FP16)
674+
backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH) && target == DNN_TARGET_OPENCL_FP16)
650675
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_OPENCL_FP16, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
651676
#endif
652677
}

‎modules/dnn/test/test_ie_models.cpp

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,8 @@ struct OpenVINOModelTestCaseInfo
7373
static const std::map<std::string, OpenVINOModelTestCaseInfo>& getOpenVINOTestModels()
7474
{
7575
static std::map<std::string, OpenVINOModelTestCaseInfo> g_models {
76-
#if INF_ENGINE_RELEASE >= 2018050000
76+
#if INF_ENGINE_RELEASE >= 2018050000 && \
77+
INF_ENGINE_RELEASE <= 2020999999 // don't use IRv5 models with 2020.1+
7778
// layout is defined by open_model_zoo/model_downloader
7879
// Downloaded using these parameters for Open Model Zoo downloader (2019R1):
7980
// ./downloader.py -o ${OPENCV_DNN_TEST_DATA_PATH}/omz_intel_models --cache_dir ${OPENCV_DNN_TEST_DATA_PATH}/.omz_cache/ \
@@ -295,6 +296,11 @@ TEST_P(DNNTestOpenVINO, models)
295296
}
296297
#endif
297298

299+
#if INF_ENGINE_VER_MAJOR_EQ(2020040000)
300+
if (targetId == DNN_TARGET_MYRIAD && modelName == "person-detection-retail-0002") // IRv5, OpenVINO 2020.4 regression
301+
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_MYRIAD, CV_TEST_TAG_DNN_SKIP_IE_NGRAPH, CV_TEST_TAG_DNN_SKIP_IE_VERSION);
302+
#endif
303+
298304
if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
299305
setInferenceEngineBackendType(CV_DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_API);
300306
else if (backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH)

‎modules/dnn/test/test_layers.cpp

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -364,6 +364,16 @@ TEST_P(Test_Caffe_layers, layer_prelu_fc)
364364
// Reference output values are in range [-0.0001, 10.3906]
365365
double l1 = (target == DNN_TARGET_MYRIAD) ? 0.005 : 0.0;
366366
double lInf = (target == DNN_TARGET_MYRIAD) ? 0.021 : 0.0;
367+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)
368+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL)
369+
{
370+
l1 = 0.006f; lInf = 0.05f;
371+
}
372+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
373+
{
374+
l1 = 0.01f; lInf = 0.05f;
375+
}
376+
#endif
367377
testLayerUsingCaffeModels("layer_prelu_fc", true, false, l1, lInf);
368378
}
369379

‎modules/dnn/test/test_onnx_importer.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -704,6 +704,13 @@ TEST_P(Test_ONNX_nets, TinyYolov2)
704704
// output range: [-11; 8]
705705
double l1 = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.017 : default_l1;
706706
double lInf = (target == DNN_TARGET_OPENCL_FP16 || target == DNN_TARGET_MYRIAD) ? 0.14 : default_lInf;
707+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)
708+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
709+
{
710+
l1 = 0.018f; lInf = 0.16f;
711+
}
712+
#endif
713+
707714
testONNXModels("tiny_yolo2", pb, l1, lInf);
708715
}
709716

@@ -781,6 +788,13 @@ TEST_P(Test_ONNX_nets, Emotion_ferplus)
781788
l1 = 2.4e-4;
782789
lInf = 6e-4;
783790
}
791+
#if defined(INF_ENGINE_RELEASE) && INF_ENGINE_VER_MAJOR_EQ(2020040000)
792+
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && target == DNN_TARGET_OPENCL_FP16)
793+
{
794+
l1 = 0.012f; lInf = 0.035f;
795+
}
796+
#endif
797+
784798
testONNXModels("emotion_ferplus", pb, l1, lInf);
785799
}
786800

0 commit comments

Comments
 (0)