Skip to content

Commit 94718f2

Browse files
TristonKFTristonKF
authored
Fix UVCDObjectDetectionLoop.ino (Ameba-AIoT#283)
* Fix UVCDObjectDetectionLoop.ino Fix video streaming occasionally displaying pink and green colors. * Update UVCDObjectDetectionLoop.ino * Update nnbegin function Remove nn_channel param in nnbegin function.
1 parent 61365e5 commit 94718f2

File tree

3 files changed

+423
-424
lines changed

3 files changed

+423
-424
lines changed

‎Arduino_package/hardware/libraries/NeuralNetwork/examples/UVCDObjectDetectionLoop/UVCDObjectDetectionLoop.ino

Lines changed: 164 additions & 165 deletions
Original file line numberDiff line numberDiff line change
@@ -1,165 +1,164 @@
1-
/*
2-
This example describes how to use USB UVCD api with OBJECT_DETECTION.
3-
In this example, the device is setup to function as a USB camera.
4-
Connect to PC and use the device as a USB camera.
5-
Perform objects prediction when the device is connect to PC and PotPlayer.
6-
7-
Example guide: https://ameba-doc-arduino-sdk.readthedocs-hosted.com/en/latest/amebapro2/Example_Guides/Neural%20Network/UVCD%20Object%20Detection.html
8-
9-
10-
NN Model Selection
11-
Select Neural Network(NN) task and models using modelSelect(nntask, objdetmodel, facedetmodel, facerecogmodel).
12-
Replace with NA_MODEL if they are not necessary for your selected NN Task.
13-
14-
NN task
15-
=======
16-
OBJECT_DETECTION/ FACE_DETECTION/ FACE_RECOGNITION
17-
18-
Models
19-
=======
20-
YOLOv3 model DEFAULT_YOLOV3TINY / CUSTOMIZED_YOLOV3TINY
21-
YOLOv4 model DEFAULT_YOLOV4TINY / CUSTOMIZED_YOLOV4TINY
22-
YOLOv7 model DEFAULT_YOLOV7TINY / CUSTOMIZED_YOLOV7TINY
23-
SCRFD model DEFAULT_SCRFD / CUSTOMIZED_SCRFD
24-
MobileFaceNet model DEFAULT_MOBILEFACENET/ CUSTOMIZED_MOBILEFACENET
25-
No model NA_MODEL
26-
*/
27-
28-
#include "StreamIO.h"
29-
#include "VideoStream.h"
30-
#include "UVCD.h"
31-
#include "NNObjectDetection.h"
32-
#include "VideoStreamOverlay.h"
33-
#include "ObjectClassList.h"
34-
35-
36-
37-
#define STREAM_CHANNEL 0
38-
#define CHANNELNN 3
39-
40-
int reconnect = 0;
41-
int disconnect = 0;
42-
43-
Video camera_uvcd;
44-
NNObjectDetection ObjDet;
45-
UVCD usb_uvcd;
46-
47-
// Lower resolution for NN processing
48-
#define NNWIDTH 576
49-
#define NNHEIGHT 320
50-
51-
StreamIO videoStreamer(1, 1); // 1 Input Video -> 1 Output USB_CAM
52-
StreamIO videoStreamerNN(1, 1);
53-
VideoSetting stream_config(USB_UVCD_STREAM_PRESET);
54-
VideoSetting configNN(NNWIDTH, NNHEIGHT, 10, VIDEO_RGB, 0);
55-
VideoSetting config(VIDEO_FHD, 30, VIDEO_H264, 0);
56-
void setup()
57-
{
58-
Serial.begin(115200);
59-
// Configure camera video channel with video format information
60-
camera_uvcd.configVideoChannel(STREAM_CHANNEL, stream_config);
61-
62-
camera_uvcd.configVideoChannel(CHANNELNN, configNN);
63-
// Configure usb_uvcd with identical video format information
64-
usb_uvcd.configVideo(stream_config);
65-
camera_uvcd.videoInit();
66-
67-
// Configure object detection with corresponding video format information
68-
// Select Neural Network(NN) task and models
69-
ObjDet.configVideo(configNN);
70-
ObjDet.modelSelect(OBJECT_DETECTION, DEFAULT_YOLOV4TINY, NA_MODEL, NA_MODEL);
71-
ObjDet.begin();
72-
73-
// Configure StreamIO object to stream data from camera video channel to usb_uvcd
74-
videoStreamer.registerInput(camera_uvcd.getStream(STREAM_CHANNEL));
75-
videoStreamer.registerOutput(usb_uvcd);
76-
if (videoStreamer.begin() != 0) {
77-
Serial.println("StreamIO link start failed");
78-
}
79-
// Start data stream from video channel
80-
camera_uvcd.channelBegin(STREAM_CHANNEL);
81-
82-
// Configure StreamIO object to stream data from RGB video channel to object detection
83-
videoStreamerNN.registerInput(camera_uvcd.getStream(CHANNELNN));
84-
videoStreamerNN.setStackSize();
85-
videoStreamerNN.setTaskPriority();
86-
videoStreamerNN.registerOutput(ObjDet);
87-
if (videoStreamerNN.begin() != 0) {
88-
Serial.println("StreamIO link start failed");
89-
}
90-
91-
// Start video channel for NN
92-
camera_uvcd.channelBegin(CHANNELNN);
93-
94-
// Start usb uvcd for NN
95-
usb_uvcd.nnbegin(camera_uvcd.getStream(STREAM_CHANNEL), videoStreamer.linker, STREAM_CHANNEL, CHANNELNN, camera_uvcd.videostream_status(STREAM_CHANNEL));
96-
97-
OSD.configVideo(STREAM_CHANNEL, config);
98-
OSD.begin();
99-
}
100-
101-
void loop()
102-
{
103-
if (!usb_uvcd.isUsbUvcConnected(camera_uvcd.videostream_status(STREAM_CHANNEL))) {
104-
Serial.println("USB UVC device disconnected");
105-
// Handle disconnection processes
106-
if (disconnect == 0) {
107-
disconnectNN();
108-
}
109-
return;
110-
} else {
111-
// Handle reconnection or restart processes
112-
if (reconnect == 1) {
113-
reconnectNN();
114-
}
115-
std::vector<ObjectDetectionResult> results = ObjDet.getResult();
116-
uint16_t im_h = stream_config.height();
117-
uint16_t im_w = stream_config.width();
118-
119-
printf("Total number of objects detected = %d\r\n", ObjDet.getResultCount());
120-
OSD.createBitmap(STREAM_CHANNEL);
121-
if (ObjDet.getResultCount() > 0) {
122-
for (int i = 0; i < ObjDet.getResultCount(); i++) {
123-
int obj_type = results[i].type();
124-
if (itemList[obj_type].filter) { // check if item should be ignored
125-
126-
ObjectDetectionResult item = results[i];
127-
// Result coordinates are floats ranging from 0.00 to 1.00
128-
// Multiply with RTSP resolution to get coordinates in pixels
129-
int xmin = (int)(item.xMin() * im_w);
130-
int xmax = (int)(item.xMax() * im_w);
131-
int ymin = (int)(item.yMin() * im_h);
132-
int ymax = (int)(item.yMax() * im_h);
133-
// Draw boundary box
134-
printf("Item %d %s:\t%d %d %d %d\n\r", i, itemList[obj_type].objectName, xmin, xmax, ymin, ymax);
135-
OSD.drawRect(STREAM_CHANNEL, xmin, ymin, xmax, ymax, 3, OSD_COLOR_WHITE);
136-
137-
// Print identification text
138-
char text_str[20];
139-
snprintf(text_str, sizeof(text_str), "%s %d", itemList[obj_type].objectName, item.score());
140-
OSD.drawText(STREAM_CHANNEL, xmin, ymin - OSD.getTextHeight(STREAM_CHANNEL), text_str, OSD_COLOR_CYAN);
141-
}
142-
}
143-
}
144-
}
145-
OSD.update(STREAM_CHANNEL);
146-
147-
// delay to wait for new results
148-
delay(100);
149-
}
150-
151-
void disconnectNN()
152-
{
153-
camera_uvcd.channelEnd(CHANNELNN); // Stop video channel from NN
154-
camera_uvcd.channelBegin(STREAM_CHANNEL); // Stop video channel from video channel
155-
reconnect = 1;
156-
disconnect = 1;
157-
}
158-
159-
void reconnectNN()
160-
{
161-
camera_uvcd.channelBegin(CHANNELNN); // Start video channel for NN
162-
camera_uvcd.channelBegin(STREAM_CHANNEL); // Start video channel for video channel
163-
reconnect = 0;
164-
disconnect = 0;
165-
}
1+
/*
2+
This example describes how to use USB UVCD api with OBJECT_DETECTION.
3+
In this example, the device is setup to function as a USB camera.
4+
Connect to PC and use the device as a USB camera.
5+
Perform objects prediction when the device is connect to PC and PotPlayer.
6+
7+
Example guide: https://ameba-doc-arduino-sdk.readthedocs-hosted.com/en/latest/amebapro2/Example_Guides/Neural%20Network/UVCD%20Object%20Detection.html
8+
9+
10+
NN Model Selection
11+
Select Neural Network(NN) task and models using modelSelect(nntask, objdetmodel, facedetmodel, facerecogmodel).
12+
Replace with NA_MODEL if they are not necessary for your selected NN Task.
13+
14+
NN task
15+
=======
16+
OBJECT_DETECTION/ FACE_DETECTION/ FACE_RECOGNITION
17+
18+
Models
19+
=======
20+
YOLOv3 model DEFAULT_YOLOV3TINY / CUSTOMIZED_YOLOV3TINY
21+
YOLOv4 model DEFAULT_YOLOV4TINY / CUSTOMIZED_YOLOV4TINY
22+
YOLOv7 model DEFAULT_YOLOV7TINY / CUSTOMIZED_YOLOV7TINY
23+
SCRFD model DEFAULT_SCRFD / CUSTOMIZED_SCRFD
24+
MobileFaceNet model DEFAULT_MOBILEFACENET/ CUSTOMIZED_MOBILEFACENET
25+
No model NA_MODEL
26+
*/
27+
28+
#include "StreamIO.h"
29+
#include "VideoStream.h"
30+
#include "UVCD.h"
31+
#include "NNObjectDetection.h"
32+
#include "VideoStreamOverlay.h"
33+
#include "ObjectClassList.h"
34+
35+
36+
37+
#define STREAM_CHANNEL 0
38+
#define CHANNELNN 3
39+
40+
int reconnect = 0;
41+
int disconnect = 0;
42+
43+
Video camera_uvcd;
44+
NNObjectDetection ObjDet;
45+
UVCD usb_uvcd;
46+
47+
// Lower resolution for NN processing
48+
#define NNWIDTH 576
49+
#define NNHEIGHT 320
50+
51+
StreamIO videoStreamer(1, 1); // 1 Input Video -> 1 Output USB_CAM
52+
StreamIO videoStreamerNN(1, 1);
53+
VideoSetting stream_config(VIDEO_FHD, 24, VIDEO_H264, 0);
54+
VideoSetting configNN(NNWIDTH, NNHEIGHT, 10, VIDEO_RGB, 0);
55+
void setup()
56+
{
57+
Serial.begin(115200);
58+
// Configure camera video channel with video format information
59+
camera_uvcd.configVideoChannel(STREAM_CHANNEL, stream_config);
60+
61+
camera_uvcd.configVideoChannel(CHANNELNN, configNN);
62+
// Configure usb_uvcd with identical video format information
63+
usb_uvcd.configVideo(stream_config);
64+
camera_uvcd.videoInit();
65+
66+
// Configure object detection with corresponding video format information
67+
// Select Neural Network(NN) task and models
68+
ObjDet.configVideo(configNN);
69+
ObjDet.modelSelect(OBJECT_DETECTION, DEFAULT_YOLOV4TINY, NA_MODEL, NA_MODEL);
70+
ObjDet.begin();
71+
72+
// Configure StreamIO object to stream data from camera video channel to usb_uvcd
73+
videoStreamer.registerInput(camera_uvcd.getStream(STREAM_CHANNEL));
74+
videoStreamer.registerOutput(usb_uvcd);
75+
if (videoStreamer.begin() != 0) {
76+
Serial.println("StreamIO link start failed");
77+
}
78+
// Start data stream from video channel
79+
camera_uvcd.channelBegin(STREAM_CHANNEL);
80+
81+
// Configure StreamIO object to stream data from RGB video channel to object detection
82+
videoStreamerNN.registerInput(camera_uvcd.getStream(CHANNELNN));
83+
videoStreamerNN.setStackSize();
84+
videoStreamerNN.setTaskPriority();
85+
videoStreamerNN.registerOutput(ObjDet);
86+
if (videoStreamerNN.begin() != 0) {
87+
Serial.println("StreamIO link start failed");
88+
}
89+
90+
// Start video channel for NN
91+
camera_uvcd.channelBegin(CHANNELNN);
92+
93+
// Start usb uvcd for NN
94+
usb_uvcd.nnbegin(camera_uvcd.getStream(STREAM_CHANNEL), videoStreamer.linker, STREAM_CHANNEL, camera_uvcd.videostream_status(STREAM_CHANNEL));
95+
96+
OSD.configVideo(STREAM_CHANNEL, stream_config);
97+
OSD.begin();
98+
}
99+
100+
void loop()
101+
{
102+
if (!usb_uvcd.isUsbUvcConnected(camera_uvcd.videostream_status(STREAM_CHANNEL))) {
103+
Serial.println("USB UVC device disconnected");
104+
// Handle disconnection processes
105+
if (disconnect == 0) {
106+
disconnectNN();
107+
}
108+
return;
109+
} else {
110+
// Handle reconnection or restart processes
111+
if (reconnect == 1) {
112+
reconnectNN();
113+
}
114+
std::vector<ObjectDetectionResult> results = ObjDet.getResult();
115+
uint16_t im_h = stream_config.height();
116+
uint16_t im_w = stream_config.width();
117+
118+
printf("Total number of objects detected = %d\r\n", ObjDet.getResultCount());
119+
OSD.createBitmap(STREAM_CHANNEL);
120+
if (ObjDet.getResultCount() > 0) {
121+
for (int i = 0; i < ObjDet.getResultCount(); i++) {
122+
int obj_type = results[i].type();
123+
if (itemList[obj_type].filter) { // check if item should be ignored
124+
125+
ObjectDetectionResult item = results[i];
126+
// Result coordinates are floats ranging from 0.00 to 1.00
127+
// Multiply with RTSP resolution to get coordinates in pixels
128+
int xmin = (int)(item.xMin() * im_w);
129+
int xmax = (int)(item.xMax() * im_w);
130+
int ymin = (int)(item.yMin() * im_h);
131+
int ymax = (int)(item.yMax() * im_h);
132+
// Draw boundary box
133+
printf("Item %d %s:\t%d %d %d %d\n\r", i, itemList[obj_type].objectName, xmin, xmax, ymin, ymax);
134+
OSD.drawRect(STREAM_CHANNEL, xmin, ymin, xmax, ymax, 3, OSD_COLOR_WHITE);
135+
136+
// Print identification text
137+
char text_str[20];
138+
snprintf(text_str, sizeof(text_str), "%s %d", itemList[obj_type].objectName, item.score());
139+
OSD.drawText(STREAM_CHANNEL, xmin, ymin - OSD.getTextHeight(STREAM_CHANNEL), text_str, OSD_COLOR_CYAN);
140+
}
141+
}
142+
}
143+
}
144+
OSD.update(STREAM_CHANNEL);
145+
146+
// delay to wait for new results
147+
delay(100);
148+
}
149+
150+
void disconnectNN()
151+
{
152+
camera_uvcd.channelEnd(CHANNELNN); // Stop video channel from NN
153+
camera_uvcd.channelBegin(STREAM_CHANNEL); // Stop video channel from video channel
154+
reconnect = 1;
155+
disconnect = 1;
156+
}
157+
158+
void reconnectNN()
159+
{
160+
camera_uvcd.channelBegin(CHANNELNN); // Start video channel for NN
161+
camera_uvcd.channelBegin(STREAM_CHANNEL); // Start video channel for video channel
162+
reconnect = 0;
163+
disconnect = 0;
164+
}

0 commit comments

Comments
 (0)