From 0592e286f11bd78a9f99a415758afd9b7cf0a2e0 Mon Sep 17 00:00:00 2001 From: Alex c Date: Sat, 14 Aug 2021 15:50:14 -0700 Subject: [PATCH 1/2] new --- examples/openpose/openpose.cpp | 2 +- .../03_keypoints_from_image.cpp | 2 +- .../05_keypoints_from_images_multi_gpu.cpp | 2 +- .../tutorial_api_cpp/06_face_from_image.cpp | 2 +- .../tutorial_api_cpp/07_hand_from_image.cpp | 2 +- .../08_heatmaps_from_image.cpp | 2 +- .../09_keypoints_from_heatmaps.cpp | 2 +- .../10_asynchronous_custom_input.cpp | 2 +- ...asynchronous_custom_input_multi_camera.cpp | 2 +- .../12_asynchronous_custom_output.cpp | 2 +- ...chronous_custom_input_output_and_datum.cpp | 2 +- .../14_synchronous_custom_input.cpp | 2 +- .../15_synchronous_custom_preprocessing.cpp | 2 +- .../16_synchronous_custom_postprocessing.cpp | 2 +- .../17_synchronous_custom_output.cpp | 2 +- .../18_synchronous_custom_all_and_datum.cpp | 2 +- include/openpose/filestream/headers.hpp | 2 + include/openpose/filestream/udpJsonSender.hpp | 29 +++ .../openpose/filestream/wUdpJsonSender.hpp | 107 +++++++++++ include/openpose/flags.hpp | 1 + include/openpose/wrapper/wrapperAuxiliary.hpp | 8 + .../openpose/wrapper/wrapperStructOutput.hpp | 7 +- src/openpose/filestream/CMakeLists.txt | 1 + src/openpose/filestream/defineTemplates.cpp | 1 + src/openpose/filestream/udpJsonSender.cpp | 176 ++++++++++++++++++ src/openpose/wrapper/wrapperAuxiliary.cpp | 1 + src/openpose/wrapper/wrapperStructOutput.cpp | 5 +- 27 files changed, 351 insertions(+), 19 deletions(-) create mode 100644 include/openpose/filestream/udpJsonSender.hpp create mode 100644 include/openpose/filestream/wUdpJsonSender.hpp create mode 100644 src/openpose/filestream/udpJsonSender.cpp diff --git a/examples/openpose/openpose.cpp b/examples/openpose/openpose.cpp index 45e45bf9a..158929b02 100755 --- a/examples/openpose/openpose.cpp +++ b/examples/openpose/openpose.cpp @@ -106,7 +106,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port), FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/03_keypoints_from_image.cpp b/examples/tutorial_api_cpp/03_keypoints_from_image.cpp index 466e22bb7..6b8e9bacb 100644 --- a/examples/tutorial_api_cpp/03_keypoints_from_image.cpp +++ b/examples/tutorial_api_cpp/03_keypoints_from_image.cpp @@ -148,7 +148,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output}; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/05_keypoints_from_images_multi_gpu.cpp b/examples/tutorial_api_cpp/05_keypoints_from_images_multi_gpu.cpp index bbb4a463a..912ec39ec 100644 --- a/examples/tutorial_api_cpp/05_keypoints_from_images_multi_gpu.cpp +++ b/examples/tutorial_api_cpp/05_keypoints_from_images_multi_gpu.cpp @@ -154,7 +154,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/06_face_from_image.cpp b/examples/tutorial_api_cpp/06_face_from_image.cpp index c121060d0..1c083156c 100644 --- a/examples/tutorial_api_cpp/06_face_from_image.cpp +++ b/examples/tutorial_api_cpp/06_face_from_image.cpp @@ -151,7 +151,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/07_hand_from_image.cpp b/examples/tutorial_api_cpp/07_hand_from_image.cpp index 96187f961..3c58e1760 100644 --- a/examples/tutorial_api_cpp/07_hand_from_image.cpp +++ b/examples/tutorial_api_cpp/07_hand_from_image.cpp @@ -151,7 +151,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/08_heatmaps_from_image.cpp b/examples/tutorial_api_cpp/08_heatmaps_from_image.cpp index 1d13e9a72..72d2068be 100644 --- a/examples/tutorial_api_cpp/08_heatmaps_from_image.cpp +++ b/examples/tutorial_api_cpp/08_heatmaps_from_image.cpp @@ -177,7 +177,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port) }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/09_keypoints_from_heatmaps.cpp b/examples/tutorial_api_cpp/09_keypoints_from_heatmaps.cpp index 411fa5199..991000dbd 100644 --- a/examples/tutorial_api_cpp/09_keypoints_from_heatmaps.cpp +++ b/examples/tutorial_api_cpp/09_keypoints_from_heatmaps.cpp @@ -151,7 +151,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/10_asynchronous_custom_input.cpp b/examples/tutorial_api_cpp/10_asynchronous_custom_input.cpp index 64cd91dfc..965aa4be2 100644 --- a/examples/tutorial_api_cpp/10_asynchronous_custom_input.cpp +++ b/examples/tutorial_api_cpp/10_asynchronous_custom_input.cpp @@ -157,7 +157,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/11_asynchronous_custom_input_multi_camera.cpp b/examples/tutorial_api_cpp/11_asynchronous_custom_input_multi_camera.cpp index 10942b1e4..56f6d3982 100644 --- a/examples/tutorial_api_cpp/11_asynchronous_custom_input_multi_camera.cpp +++ b/examples/tutorial_api_cpp/11_asynchronous_custom_input_multi_camera.cpp @@ -156,7 +156,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/12_asynchronous_custom_output.cpp b/examples/tutorial_api_cpp/12_asynchronous_custom_output.cpp index 8a7e7ba1b..32583570f 100644 --- a/examples/tutorial_api_cpp/12_asynchronous_custom_output.cpp +++ b/examples/tutorial_api_cpp/12_asynchronous_custom_output.cpp @@ -186,7 +186,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/13_asynchronous_custom_input_output_and_datum.cpp b/examples/tutorial_api_cpp/13_asynchronous_custom_input_output_and_datum.cpp index 17894a0ed..8f6cf865f 100644 --- a/examples/tutorial_api_cpp/13_asynchronous_custom_input_output_and_datum.cpp +++ b/examples/tutorial_api_cpp/13_asynchronous_custom_input_output_and_datum.cpp @@ -258,7 +258,7 @@ void configureWrapper(op::WrapperT& opWrapperT) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/14_synchronous_custom_input.cpp b/examples/tutorial_api_cpp/14_synchronous_custom_input.cpp index 087c17004..043503d14 100644 --- a/examples/tutorial_api_cpp/14_synchronous_custom_input.cpp +++ b/examples/tutorial_api_cpp/14_synchronous_custom_input.cpp @@ -170,7 +170,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/15_synchronous_custom_preprocessing.cpp b/examples/tutorial_api_cpp/15_synchronous_custom_preprocessing.cpp index 3cadbe6ab..cbe76fca5 100644 --- a/examples/tutorial_api_cpp/15_synchronous_custom_preprocessing.cpp +++ b/examples/tutorial_api_cpp/15_synchronous_custom_preprocessing.cpp @@ -145,7 +145,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/16_synchronous_custom_postprocessing.cpp b/examples/tutorial_api_cpp/16_synchronous_custom_postprocessing.cpp index b760a3417..fdb78bf77 100644 --- a/examples/tutorial_api_cpp/16_synchronous_custom_postprocessing.cpp +++ b/examples/tutorial_api_cpp/16_synchronous_custom_postprocessing.cpp @@ -146,7 +146,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // GUI (comment or use default argument to disable any visual output) const op::WrapperStructGui wrapperStructGui{ diff --git a/examples/tutorial_api_cpp/17_synchronous_custom_output.cpp b/examples/tutorial_api_cpp/17_synchronous_custom_output.cpp index c5406bed5..718108f85 100644 --- a/examples/tutorial_api_cpp/17_synchronous_custom_output.cpp +++ b/examples/tutorial_api_cpp/17_synchronous_custom_output.cpp @@ -203,7 +203,7 @@ void configureWrapper(op::Wrapper& opWrapper) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapper.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/examples/tutorial_api_cpp/18_synchronous_custom_all_and_datum.cpp b/examples/tutorial_api_cpp/18_synchronous_custom_all_and_datum.cpp index 6956f63a4..36640a54b 100644 --- a/examples/tutorial_api_cpp/18_synchronous_custom_all_and_datum.cpp +++ b/examples/tutorial_api_cpp/18_synchronous_custom_all_and_datum.cpp @@ -322,7 +322,7 @@ void configureWrapper(op::WrapperT& opWrapperT) op::String(FLAGS_write_video), FLAGS_write_video_fps, FLAGS_write_video_with_audio, op::String(FLAGS_write_heatmaps), op::String(FLAGS_write_heatmaps_format), op::String(FLAGS_write_video_3d), op::String(FLAGS_write_video_adam), op::String(FLAGS_write_bvh), op::String(FLAGS_udp_host), - op::String(FLAGS_udp_port)}; + op::String(FLAGS_udp_port),FLAGS_udp_json_output }; opWrapperT.configure(wrapperStructOutput); // No GUI. Equivalent to: opWrapper.configure(op::WrapperStructGui{}); // Set to single-thread (for sequential processing and/or debugging and/or reducing latency) diff --git a/include/openpose/filestream/headers.hpp b/include/openpose/filestream/headers.hpp index 68db0bc56..21bb3629e 100644 --- a/include/openpose/filestream/headers.hpp +++ b/include/openpose/filestream/headers.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include diff --git a/include/openpose/filestream/udpJsonSender.hpp b/include/openpose/filestream/udpJsonSender.hpp new file mode 100644 index 000000000..54187f001 --- /dev/null +++ b/include/openpose/filestream/udpJsonSender.hpp @@ -0,0 +1,29 @@ +#ifndef OPENPOSE_FILESTREAM_UDP_JSON_SENDER_HPP +#define OPENPOSE_FILESTREAM_UDP_JSON_SENDER_HPP + +#include + +namespace op +{ + class OP_API UdpJsonSender + { + public: + UdpJsonSender(const std::string& udpHost, const std::string& udpPort); + + virtual ~UdpJsonSender(); + + void sendJson(const std::vector, std::string>>& keypointVector); + + private: + // PIMPL idiom + // http://www.cppsamples.com/common-tasks/pimpl.html + struct ImplJsonUdpSender; + std::shared_ptr spImpl; + + // PIMP requires DELETE_COPY & destructor, or extra code + // http://oliora.github.io/2015/12/29/pimpl-and-rule-of-zero.html + DELETE_COPY(UdpJsonSender); + }; +} + +#endif // OPENPOSE_FILESTREAM_UDP_JSON_SENDER_HPP diff --git a/include/openpose/filestream/wUdpJsonSender.hpp b/include/openpose/filestream/wUdpJsonSender.hpp new file mode 100644 index 000000000..880975340 --- /dev/null +++ b/include/openpose/filestream/wUdpJsonSender.hpp @@ -0,0 +1,107 @@ +#ifndef OPENPOSE_FILESTREAM_W_UDP_JSON_SENDER_HPP +#define OPENPOSE_FILESTREAM_W_UDP_JSON_SENDER_HPP + +#include +#include +#include + +namespace op +{ + template + class WUdpJsonSender : public WorkerConsumer + { + public: + explicit WUdpJsonSender(const std::shared_ptr& udpJsonSender); + + virtual ~WUdpJsonSender(); + + void initializationOnThread(); + + void workConsumer(const TDatums& tDatums); + + private: + const std::shared_ptr spUdpSender; + + DELETE_COPY(WUdpJsonSender); + }; +} + + + + + +// Implementation +#include +namespace op +{ + template + WUdpJsonSender::WUdpJsonSender(const std::shared_ptr& udpSender) : + spUdpSender{udpSender} + { + } + + template + WUdpJsonSender::~WUdpJsonSender() + { + } + + template + void WUdpJsonSender::initializationOnThread() + { + } + + template + void WUdpJsonSender::workConsumer(const TDatums& tDatums) + { + try + { + if (checkNoNullNorEmpty(tDatums)) + { + // Debugging log + opLogIfDebug("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + // Profiling speed + const auto profilerKey = Profiler::timerInit(__LINE__, __FUNCTION__, __FILE__); + + for (auto i = 0u; i < tDatums->size(); i++) + { + const auto& tDatumPtr = (*tDatums)[i]; + + // Pose IDs from long long to float + Array poseIds{ tDatumPtr->poseIds }; + + const std::vector, std::string>> keypointVector{ + // Pose IDs + std::make_pair(poseIds, "person_id"), + // 2D + std::make_pair(tDatumPtr->poseKeypoints, "pose_keypoints_2d"), + std::make_pair(tDatumPtr->faceKeypoints, "face_keypoints_2d"), + std::make_pair(tDatumPtr->handKeypoints[0], "hand_left_keypoints_2d"), + std::make_pair(tDatumPtr->handKeypoints[1], "hand_right_keypoints_2d"), + // 3D + std::make_pair(tDatumPtr->poseKeypoints3D, "pose_keypoints_3d"), + std::make_pair(tDatumPtr->faceKeypoints3D, "face_keypoints_3d"), + std::make_pair(tDatumPtr->handKeypoints3D[0], "hand_left_keypoints_3d"), + std::make_pair(tDatumPtr->handKeypoints3D[1], "hand_right_keypoints_3d") + }; + + spUdpSender->sendJson(keypointVector); + } + + // Profiling speed + Profiler::timerEnd(profilerKey); + Profiler::printAveragedTimeMsOnIterationX(profilerKey, __LINE__, __FUNCTION__, __FILE__); + // Debugging log + opLogIfDebug("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + } + catch (const std::exception& e) + { + this->stop(); + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + COMPILE_TEMPLATE_DATUM(WUdpJsonSender); +} + +#endif // OPENPOSE_FILESTREAM_W_UDP_SENDER_HPP diff --git a/include/openpose/flags.hpp b/include/openpose/flags.hpp index ba281c97c..c9614ee3b 100644 --- a/include/openpose/flags.hpp +++ b/include/openpose/flags.hpp @@ -273,6 +273,7 @@ DEFINE_string(write_keypoint_format, "yml", "(Deprecated, use `write // Result Saving - Extra Algorithms DEFINE_string(write_bvh, "", "Experimental, not available yet. E.g., `~/Desktop/mocapResult.bvh`."); // UDP Communication +DEFINE_bool(udp_json_output, false, "Will output JSON format data to the udp_host and udp_port values instead of to the fileoutput."); DEFINE_string(udp_host, "", "Experimental, not available yet. IP for UDP communication. E.g., `192.168.0.1`."); DEFINE_string(udp_port, "8051", "Experimental, not available yet. Port number for UDP communication."); #endif // OPENPOSE_FLAGS_DISABLE_POSE diff --git a/include/openpose/wrapper/wrapperAuxiliary.hpp b/include/openpose/wrapper/wrapperAuxiliary.hpp index c2820df17..a7cca38ec 100644 --- a/include/openpose/wrapper/wrapperAuxiliary.hpp +++ b/include/openpose/wrapper/wrapperAuxiliary.hpp @@ -778,6 +778,14 @@ namespace op const auto peopleJsonSaver = std::make_shared(writeJsonCleaned); outputWs.emplace_back(std::make_shared>(peopleJsonSaver)); } + + //Enable the json over UDP output + if (wrapperStructOutput.udpJsonOutput) { + const auto udpJsonSender = std::make_shared(wrapperStructOutput.udpHost.getStdString(), + wrapperStructOutput.udpPort.getStdString()); + outputWs.emplace_back(std::make_shared>(udpJsonSender)); + } + opLog("", Priority::Low, __LINE__, __FUNCTION__, __FILE__); // Write people pose/foot/face/hand/etc. data on disk (COCO validation JSON format) if (!wrapperStructOutput.writeCocoJson.empty()) diff --git a/include/openpose/wrapper/wrapperStructOutput.hpp b/include/openpose/wrapper/wrapperStructOutput.hpp index 257c2bc70..29578017d 100644 --- a/include/openpose/wrapper/wrapperStructOutput.hpp +++ b/include/openpose/wrapper/wrapperStructOutput.hpp @@ -142,6 +142,11 @@ namespace op */ String udpPort; + /** + * Whether to output json coordinates to a udp stream using udpHost and udpPort. + */ + bool udpJsonOutput; + /** * Constructor of the struct. * It has the recommended and default values we recommend for each element of the struct. @@ -157,7 +162,7 @@ namespace op const String& writeHeatMaps = "", const String& writeHeatMapsFormat = "png", const String& writeVideo3D = "", const String& writeVideoAdam = "", const String& writeBvh = "", const String& udpHost = "", - const String& udpPort = "8051"); + const String& udpPort = "8051", const bool udpJsonOutput = false); }; } diff --git a/src/openpose/filestream/CMakeLists.txt b/src/openpose/filestream/CMakeLists.txt index 59d930397..7482fc532 100644 --- a/src/openpose/filestream/CMakeLists.txt +++ b/src/openpose/filestream/CMakeLists.txt @@ -9,6 +9,7 @@ set(SOURCES_OP_FILESTREAM bvhSaver.cpp keypointSaver.cpp peopleJsonSaver.cpp udpSender.cpp + udpJsonSender.cpp videoSaver.cpp) include(${CMAKE_SOURCE_DIR}/cmake/Utils.cmake) diff --git a/src/openpose/filestream/defineTemplates.cpp b/src/openpose/filestream/defineTemplates.cpp index c67366eda..38c6aca99 100644 --- a/src/openpose/filestream/defineTemplates.cpp +++ b/src/openpose/filestream/defineTemplates.cpp @@ -13,6 +13,7 @@ namespace op DEFINE_TEMPLATE_DATUM(WPeopleJsonSaver); DEFINE_TEMPLATE_DATUM(WPoseSaver); DEFINE_TEMPLATE_DATUM(WUdpSender); + DEFINE_TEMPLATE_DATUM(WUdpJsonSender); DEFINE_TEMPLATE_DATUM(WVideoSaver); DEFINE_TEMPLATE_DATUM(WVideoSaver3D); } diff --git a/src/openpose/filestream/udpJsonSender.cpp b/src/openpose/filestream/udpJsonSender.cpp new file mode 100644 index 000000000..784f19544 --- /dev/null +++ b/src/openpose/filestream/udpJsonSender.cpp @@ -0,0 +1,176 @@ +#include +//#ifdef USE_ASIO + #include + #include +//#endif +#ifdef USE_EIGEN + #include +#endif +#include + +namespace op +{ + //#ifdef USE_ASIO + class UdpClient + { + public: + UdpClient(const std::string& host, const std::string& port) : + mIoService{}, + mUdpSocket{mIoService, asio::ip::udp::endpoint(asio::ip::udp::v4(), 0)} + { + try + { + asio::ip::udp::resolver resolver{mIoService}; + asio::ip::udp::resolver::query query{asio::ip::udp::v4(), host, port}; + asio::ip::udp::resolver::iterator iter = resolver.resolve(query); + mUdpEndpoint = *iter; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + ~UdpClient() + { + try + { + mUdpSocket.close(); + } + catch (const std::exception& e) + { + errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + void send(const std::string& msg) + { + try + { + mUdpSocket.send_to(asio::buffer(msg, msg.size()), mUdpEndpoint); + // std::cout << "sent data: " << msg << std::endl; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + private: + asio::io_service mIoService; + asio::ip::udp::socket mUdpSocket; + asio::ip::udp::endpoint mUdpEndpoint; + }; + + + std::string vectorToJson(const float x, const float y, const float z) + { + return std::string{"{"} + + "\"x\":" + std::to_string(x) + "," + + "\"y\":" + std::to_string(y) + "," + + "\"z\":" + std::to_string(z) + "}"; + } + //#endif + + struct UdpSender::ImplUdpSender + { + // #ifdef USE_ASIO + // Used when increasing spCaffeNets + UdpClient mUdpClient; + + ImplUdpSender(const std::string& udpHost, const std::string& udpPort) : + mUdpClient(udpHost, udpPort) + { + } + // #endif + }; + + UdpSender::UdpSender(const std::string& udpHost, const std::string& udpPort) + // #ifdef USE_ASIO + : spImpl{new ImplUdpSender{udpHost, udpPort}} + // #endif + { + try + { + // error("UdpSender (`--udp_host` and `--udp_port` flags) buggy and not working yet, but we are" + // "working on it! Coming soon!", __LINE__, __FUNCTION__, __FILE__); + #if !defined(USE_ASIO) || !defined(USE_EIGEN) + error("Both `WITH_ASIO` and `WITH_EIGEN` flags must be enabled in CMake for UDP sender.", + __LINE__, __FUNCTION__, __FILE__); + UNUSED(udpHost); + UNUSED(udpPort); + #endif + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + UdpSender::~UdpSender() + { + } + + void UdpSender::sendJointAngles(const double* const adamPosePtr, const int adamPoseRows, + const double* const adamTranslationPtr, + const double* const adamFaceCoeffsExpPtr, const int faceCoeffRows) + { + #if defined(USE_ASIO) && defined(USE_EIGEN) + try + { + if (adamPosePtr != nullptr && adamTranslationPtr != nullptr && adamFaceCoeffsExpPtr != nullptr) + { + const Eigen::Map adamTranslation(adamTranslationPtr); + const Eigen::Map> adamPose( + adamPosePtr, adamPoseRows, 3); + const Eigen::Map adamFaceCoeffsExp(adamFaceCoeffsExpPtr, faceCoeffRows); + + const std::string prefix = "AnimData:"; + const std::string totalPositionString = "\"totalPosition\":" + + vectorToJson(adamTranslation(0), adamTranslation(1), adamTranslation(2)); + std::string jointAnglesString = "\"jointAngles\":["; + for (int i = 0; i < adamPoseRows; i++) + { + jointAnglesString += vectorToJson(adamPose(i, 0), adamPose(i, 1), adamPose(i, 2)); + if (i != adamPoseRows - 1) + { + jointAnglesString += ","; + } + } + jointAnglesString += "]"; + + std::string facialParamsString = "\"facialParams\":["; + for (int i = 0; i < faceCoeffRows; i++) + { + facialParamsString += std::to_string(adamFaceCoeffsExp(i)); + if (i != faceCoeffRows - 1) + { + facialParamsString += ","; + } + } + facialParamsString += "]"; + + // facialParamsString + std::to_string(mouth_open) + "," + std::to_string(leye_open) + "," + std::to_string(reye_open) + "]"; + + // std::string rootHeightString = "\"rootHeight\":" + std::to_string(dist_root_foot); + + const std::string data = prefix + "{" + facialParamsString + + "," + totalPositionString + + "," + jointAnglesString + "}"; + + spImpl->mUdpClient.send(data); + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + #else + UNUSED(adamPosePtr); + UNUSED(adamPoseRows); + UNUSED(adamTranslationPtr); + UNUSED(adamFaceCoeffsExpPtr); + UNUSED(faceCoeffRows); + #endif + } +} diff --git a/src/openpose/wrapper/wrapperAuxiliary.cpp b/src/openpose/wrapper/wrapperAuxiliary.cpp index 2290663e0..1b0cd1eb3 100644 --- a/src/openpose/wrapper/wrapperAuxiliary.cpp +++ b/src/openpose/wrapper/wrapperAuxiliary.cpp @@ -67,6 +67,7 @@ namespace op !wrapperStructOutput.writeImages.empty() || !wrapperStructOutput.writeVideo.empty() || !wrapperStructOutput.writeKeypoint.empty() || !wrapperStructOutput.writeJson.empty() || !wrapperStructOutput.writeCocoJson.empty() || !wrapperStructOutput.writeHeatMaps.empty() + || wrapperStructOutput.udpJsonOutput ); const auto savingCvOutput = ( !wrapperStructOutput.writeImages.empty() || !wrapperStructOutput.writeVideo.empty() diff --git a/src/openpose/wrapper/wrapperStructOutput.cpp b/src/openpose/wrapper/wrapperStructOutput.cpp index 0a05d578d..c5f629ab4 100644 --- a/src/openpose/wrapper/wrapperStructOutput.cpp +++ b/src/openpose/wrapper/wrapperStructOutput.cpp @@ -9,7 +9,7 @@ namespace op const String& writeVideo_, const double writeVideoFps_, const bool writeVideoWithAudio_, const String& writeHeatMaps_, const String& writeHeatMapsFormat_, const String& writeVideo3D_, const String& writeVideoAdam_, const String& writeBvh_, const String& udpHost_, - const String& udpPort_) : + const String& udpPort_, const bool udpJsonOutput_) : verbose{verbose_}, writeKeypoint{writeKeypoint_}, writeKeypointFormat{writeKeypointFormat_}, @@ -28,7 +28,8 @@ namespace op writeVideoAdam{writeVideoAdam_}, writeBvh{writeBvh_}, udpHost{udpHost_}, - udpPort{udpPort_} + udpPort{udpPort_}, + udpJsonOutput{ udpJsonOutput_ } { try { From c2b61e7eb38858ac598dd6165f9fd77aa4ca7433 Mon Sep 17 00:00:00 2001 From: Alex c Date: Sat, 14 Aug 2021 18:07:46 -0700 Subject: [PATCH 2/2] Updated udpJsonSender.hpp --- include/openpose/filestream/udpJsonSender.hpp | 186 ++++++++++++++++-- 1 file changed, 166 insertions(+), 20 deletions(-) diff --git a/include/openpose/filestream/udpJsonSender.hpp b/include/openpose/filestream/udpJsonSender.hpp index 54187f001..15421aab8 100644 --- a/include/openpose/filestream/udpJsonSender.hpp +++ b/include/openpose/filestream/udpJsonSender.hpp @@ -1,29 +1,175 @@ -#ifndef OPENPOSE_FILESTREAM_UDP_JSON_SENDER_HPP -#define OPENPOSE_FILESTREAM_UDP_JSON_SENDER_HPP +#include -#include +#include +#include + +#include +#include +#include namespace op { - class OP_API UdpJsonSender - { - public: - UdpJsonSender(const std::string& udpHost, const std::string& udpPort); + class UdpJsonClient + { + public: + UdpJsonClient(const std::string& host, const std::string& port) : + mIoService{}, + mUdpSocket{ mIoService, asio::ip::udp::endpoint(asio::ip::udp::v4(), 0) } + { + try + { + asio::ip::udp::resolver resolver{ mIoService }; + asio::ip::udp::resolver::query query{ asio::ip::udp::v4(), host, port }; + asio::ip::udp::resolver::iterator iter = resolver.resolve(query); + mUdpEndpoint = *iter; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } - virtual ~UdpJsonSender(); + ~UdpJsonClient() + { + try + { + mUdpSocket.close(); + } + catch (const std::exception& e) + { + errorDestructor(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } - void sendJson(const std::vector, std::string>>& keypointVector); + void send(const std::string& msg) + { + try + { + mUdpSocket.send_to(asio::buffer(msg, msg.size()), mUdpEndpoint); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } - private: - // PIMPL idiom - // http://www.cppsamples.com/common-tasks/pimpl.html - struct ImplJsonUdpSender; - std::shared_ptr spImpl; + private: + asio::io_service mIoService; + asio::ip::udp::socket mUdpSocket; + asio::ip::udp::endpoint mUdpEndpoint; + }; - // PIMP requires DELETE_COPY & destructor, or extra code - // http://oliora.github.io/2015/12/29/pimpl-and-rule-of-zero.html - DELETE_COPY(UdpJsonSender); - }; -} + struct UdpJsonSender::ImplJsonUdpSender + { + // Used when increasing spCaffeNets + UdpJsonClient mUdpClient; + + ImplJsonUdpSender(const std::string& udpHost, const std::string& udpPort) : + mUdpClient(udpHost, udpPort) + { + } + }; + + UdpJsonSender::UdpJsonSender(const std::string& udpHost, const std::string& udpPort) + : spImpl{ new ImplJsonUdpSender{udpHost, udpPort} } + + { + // Debugging log + opLogIfDebug("Starting UDP JSON Sender", Priority::Low, __LINE__, __FUNCTION__, __FILE__); + } + + UdpJsonSender::~UdpJsonSender() + { + } -#endif // OPENPOSE_FILESTREAM_UDP_JSON_SENDER_HPP + void UdpJsonSender::sendJson(const std::vector, std::string>>& keypointVector) + { + try + { + // Sanity check + for (const auto& keypointPair : keypointVector) + if (!keypointPair.first.empty() && keypointPair.first.getNumberDimensions() != 3 + && keypointPair.first.getNumberDimensions() != 1) + error("keypointVector.getNumberDimensions() != 1 && != 3.", __LINE__, __FUNCTION__, __FILE__); + + //Instead of streaming to a file, use a similar means to stream into a string that will be sent over udp + std::stringbuf streamBuf; + std::ostream os(&streamBuf); + + //Build the json output - adapted from the file output model + os << "{"; + try + { + // Sanity check + for (const auto& keypointPair : keypointVector) + if (!keypointPair.first.empty() && keypointPair.first.getNumberDimensions() != 3 + && keypointPair.first.getNumberDimensions() != 1) + error("keypointVector.getNumberDimensions() != 1 && != 3.", __LINE__, __FUNCTION__, __FILE__); + // Add people keypoints + os << "\"people\" : "; + os << "["; + // Get max numberPeople + auto numberPeople = 0; + for (auto vectorIndex = 0u; vectorIndex < keypointVector.size(); vectorIndex++) + numberPeople = fastMax(numberPeople, keypointVector[vectorIndex].first.getSize(0)); + for (auto person = 0; person < numberPeople; person++) + { + os << "{"; + //jsonOfstream.objectOpen(); + for (auto vectorIndex = 0u; vectorIndex < keypointVector.size(); vectorIndex++) + { + const auto& keypoints = keypointVector[vectorIndex].first; + const auto& keypointName = keypointVector[vectorIndex].second; + const auto numberElementsPerRaw = keypoints.getSize(1) * keypoints.getSize(2); + os << "\"" + keypointName + "\" : "; + //Only the person_id node isn't an array (open) + if (keypointName != "person_id") { + os << "["; + } + // Body parts + if (numberElementsPerRaw > 0) + { + const auto finalIndex = person * numberElementsPerRaw; + for (auto element = 0; element < numberElementsPerRaw - 1; element++) + { + os << keypoints[finalIndex + element]; + os << ","; + } + // Last element (no comma) + os << keypoints[finalIndex + numberElementsPerRaw - 1]; + } + + //Only the person_id node isn't an array (close) + if (keypointName != "person_id") { + // Close array + os << "]"; + } + if (vectorIndex < keypointVector.size() - 1) { + os << ","; + } + } + os << "}"; + if (person < numberPeople - 1) + { + os << ","; + } + } + // Close bodies array + os << "]"; + os << "}"; + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + + //Send the constructed json over UDP + spImpl->mUdpClient.send(streamBuf.str()); + + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } +}