Skip to content

Commit d991c22

Browse files
l-batl-bat
authored
Merge pull request opencv#16575 from l-bat:flownet2
Support FlowNet2 model * Support DataAugmentation layer * Fix warnings * Fix comments * Support Correlation layer * TEST * Support Correlation layer * Supported Accum and FlowWarp layers * Supported ChannelNorm layer * Supported Resample with inputs.size() > 1 * Fixed comments * Refactoring * Added tests * Add resample test * Added asserts in resize layer * Updated DataAugmentation layer * Update convolution layer * Refactoring * Fix data augmentation layer * Fix caffe importer * Fix resize * Switch to Mat ptr * Remove useless resize type * Used ResizeLayer in Accum * Split ChannelNormLayer * Delete duplicate assert * Add sample * Fix sample * Added colormap
1 parent 322960f commit d991c22

File tree

10 files changed

+829
-13
lines changed

10 files changed

+829
-13
lines changed

‎modules/dnn/include/opencv2/dnn/all_layers.hpp

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -556,6 +556,30 @@ CV__DNN_EXPERIMENTAL_NS_BEGIN
556556
static Ptr<Layer> create(const LayerParams& params);
557557
};
558558

559+
class CV_EXPORTS DataAugmentationLayer : public Layer
560+
{
561+
public:
562+
static Ptr<DataAugmentationLayer> create(const LayerParams& params);
563+
};
564+
565+
class CV_EXPORTS CorrelationLayer : public Layer
566+
{
567+
public:
568+
static Ptr<CorrelationLayer> create(const LayerParams& params);
569+
};
570+
571+
class CV_EXPORTS AccumLayer : public Layer
572+
{
573+
public:
574+
static Ptr<AccumLayer> create(const LayerParams& params);
575+
};
576+
577+
class CV_EXPORTS FlowWarpLayer : public Layer
578+
{
579+
public:
580+
static Ptr<FlowWarpLayer> create(const LayerParams& params);
581+
};
582+
559583
class CV_EXPORTS PriorBoxLayer : public Layer
560584
{
561585
public:

‎modules/dnn/src/caffe/caffe_importer.cpp

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,35 @@ class CaffeImporter
465465
net.mutable_layer(li)->mutable_bottom()->RemoveLast();
466466
type = "Eltwise";
467467
}
468+
else if (type == "Resample")
469+
{
470+
CV_Assert(layer.bottom_size() == 1 || layer.bottom_size() == 2);
471+
type = "Resize";
472+
String interp = layerParams.get<String>("type").toLowerCase();
473+
layerParams.set("interpolation", interp == "linear" ? "bilinear" : interp);
474+
475+
if (layerParams.has("factor"))
476+
{
477+
float factor = layerParams.get<float>("factor");
478+
CV_Assert(layer.bottom_size() != 2 || factor == 1.0);
479+
layerParams.set("zoom_factor", factor);
480+
481+
if ((interp == "linear" && factor != 1.0) ||
482+
(interp == "nearest" && factor < 1.0))
483+
CV_Error(Error::StsNotImplemented, "Unsupported Resample mode");
484+
}
485+
}
486+
else if ("Convolution" == type)
487+
{
488+
CV_Assert(layer.bottom_size() == layer.top_size());
489+
for (int i = 0; i < layer.bottom_size(); i++)
490+
{
491+
int conv_id = dstNet.addLayer(layer.top(i), type, layerParams);
492+
addInput(layer.bottom(i), conv_id, 0, dstNet);
493+
addedBlobs.push_back(BlobNote(layer.top(i), conv_id, 0));
494+
}
495+
continue;
496+
}
468497
else if ("ConvolutionDepthwise" == type)
469498
{
470499
type = "Convolution";

‎modules/dnn/src/init.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,10 @@ void initializeLayerFactory()
132132
CV_DNN_REGISTER_LAYER_CLASS(Padding, PaddingLayer);
133133
CV_DNN_REGISTER_LAYER_CLASS(Proposal, ProposalLayer);
134134
CV_DNN_REGISTER_LAYER_CLASS(Scale, ScaleLayer);
135+
CV_DNN_REGISTER_LAYER_CLASS(DataAugmentation, DataAugmentationLayer);
136+
CV_DNN_REGISTER_LAYER_CLASS(Correlation, CorrelationLayer);
137+
CV_DNN_REGISTER_LAYER_CLASS(Accum, AccumLayer);
138+
CV_DNN_REGISTER_LAYER_CLASS(FlowWarp, FlowWarpLayer);
135139

136140
CV_DNN_REGISTER_LAYER_CLASS(LSTM, LSTMLayer);
137141
}

‎modules/dnn/src/layers/accum_layer.cpp

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// This file is part of OpenCV project.
2+
// It is subject to the license terms in the LICENSE file found in the top-level directory
3+
// of this distribution and at http://opencv.org/license.html.
4+
5+
// Copyright (C) 2020, Intel Corporation, all rights reserved.
6+
// Third party copyrights are property of their respective owners.
7+
8+
#include "../precomp.hpp"
9+
#include "layers_common.hpp"
10+
11+
12+
namespace cv { namespace dnn {
13+
14+
class AccumLayerImpl CV_FINAL : public AccumLayer
15+
{
16+
public:
17+
AccumLayerImpl(const LayerParams& params)
18+
{
19+
setParamsFrom(params);
20+
top_height = params.get<int>("top_height", 0);
21+
top_width = params.get<int>("top_width", 0);
22+
divisor = params.get<int>("size_divisible_by", 0);
23+
have_reference = params.get<String>("have_reference", "false") == "true";
24+
}
25+
26+
virtual bool getMemoryShapes(const std::vector<MatShape> &inputs,
27+
const int requiredOutputs,
28+
std::vector<MatShape> &outputs,
29+
std::vector<MatShape> &internals) const CV_OVERRIDE
30+
{
31+
std::vector<int> outShape;
32+
int batch = inputs[0][0];
33+
outShape.push_back(batch);
34+
35+
if (have_reference)
36+
{
37+
CV_Assert(inputs.size() >= 2);
38+
int totalchannels = 0;
39+
for (int i = 0; i < inputs.size() - 1; i++) {
40+
CV_Assert(inputs[i][0] == batch);
41+
totalchannels += inputs[i][1];
42+
}
43+
outShape.push_back(totalchannels);
44+
45+
int height = inputs.back()[2];
46+
int width = inputs.back()[3];
47+
48+
outShape.push_back(height);
49+
outShape.push_back(width);
50+
}
51+
else
52+
{
53+
int maxwidth = -1;
54+
int maxheight = -1;
55+
int totalchannels = 0;
56+
57+
// Find largest blob size and count total channels
58+
for (int i = 0; i < inputs.size(); ++i)
59+
{
60+
totalchannels += inputs[i][1];
61+
maxheight = std::max(maxheight, inputs[i][2]);
62+
maxwidth = std::max(maxwidth, inputs[i][3]);
63+
CV_Assert(inputs[i][0] == batch);
64+
}
65+
outShape.push_back(totalchannels);
66+
67+
int out_h = divisor ? static_cast<int>(ceil(maxheight / divisor) * divisor) : top_height;
68+
int out_w = divisor ? static_cast<int>(ceil(maxwidth / divisor) * divisor) : top_width;
69+
70+
// Layer can specify custom top size which is larger than default
71+
if (out_h <= maxheight || out_w <= maxwidth)
72+
{
73+
out_h = maxheight;
74+
out_w = maxwidth;
75+
}
76+
77+
outShape.push_back(out_h);
78+
outShape.push_back(out_w);
79+
}
80+
81+
outputs.assign(1, outShape);
82+
return false;
83+
}
84+
85+
virtual void finalize(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr) CV_OVERRIDE
86+
{
87+
LayerParams resizeParams;
88+
resizeParams.set("interpolation", "bilinear");
89+
resizeParams.set("align_corners", true);
90+
resize = ResizeLayer::create(resizeParams);
91+
}
92+
93+
void forward(InputArrayOfArrays inputs_arr, OutputArrayOfArrays outputs_arr, OutputArrayOfArrays internals_arr) CV_OVERRIDE
94+
{
95+
CV_TRACE_FUNCTION();
96+
CV_TRACE_ARG_VALUE(name, "name", name.c_str());
97+
98+
std::vector<Mat> inputs, outputs;
99+
inputs_arr.getMatVector(inputs);
100+
outputs_arr.getMatVector(outputs);
101+
102+
const int out_h = outputs[0].size[2];
103+
const int out_w = outputs[0].size[3];
104+
float* out_data = outputs[0].ptr<float>();
105+
std::vector<int> sizes(&outputs[0].size[0], &outputs[0].size[0] + outputs[0].size.dims());
106+
for (int i = 0; i < inputs.size() - have_reference; i++)
107+
{
108+
sizes[1] = inputs[i].size[1];
109+
Mat outSlice(sizes, CV_32F, out_data);
110+
111+
if (out_h == inputs[i].size[2] && out_w == inputs[i].size[3])
112+
{
113+
inputs[i].copyTo(outSlice);
114+
}
115+
else
116+
{
117+
std::vector<Mat> inp_slices, out_slices;
118+
inp_slices.push_back(inputs[i]);
119+
out_slices.push_back(outSlice);
120+
121+
resize->finalize(inp_slices, out_slices);
122+
resize->forward(inp_slices, out_slices, internals_arr);
123+
}
124+
out_data += outSlice.total(1);
125+
}
126+
}
127+
128+
private:
129+
int top_height;
130+
int top_width;
131+
int divisor;
132+
bool have_reference;
133+
Ptr<ResizeLayer> resize;
134+
};
135+
136+
Ptr<AccumLayer> AccumLayer::create(const LayerParams& params)
137+
{
138+
return Ptr<AccumLayer>(new AccumLayerImpl(params));
139+
}
140+
141+
}} // namespace cv::dnn

0 commit comments

Comments
 (0)