From 16c77c027a9f6fe5107c9a4f846f091c5c599991 Mon Sep 17 00:00:00 2001 From: fujialin <757524349@qq.com> Date: Thu, 10 Nov 2022 15:20:54 +0800 Subject: [PATCH 1/4] rockchip flowunit code --- .../video_out/ffmpeg_video_muxer.cc | 1 + .../common_src/video_out/ffmpeg_writer.cc | 2 +- .../video_out/video_out_flowunit.cc | 380 ++++++++++++++++++ .../common_src/video_out/video_out_flowunit.h | 124 ++++++ .../video_out/video_out_flowunit_test.cc | 186 +++++++++ .../local_camera/local_camera_flowunit.h | 1 + .../flowunit/local_camera/v4l2_camera.cc | 2 +- .../flowunit/to_cpuimg/mpp_to_cpu_flowunit.h | 1 + .../video_decoder/video_decoder_flowunit.h | 2 + .../video_out/ffmpeg_video_encoder.cc | 1 + 10 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc create mode 100644 src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h create mode 100644 src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc diff --git a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc index 4b9f4ceed..8c5e6830c 100644 --- a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc +++ b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc @@ -25,6 +25,7 @@ modelbox::Status FfmpegVideoMuxer::Init( const std::shared_ptr &writer) { destination_url_ = writer->GetDestinationURL(); format_ctx_ = writer->GetCtx(); + if (format_ctx_ == nullptr) { const auto *msg = "FfmpegVideoMuxer.Init fail format ctx is nullptr"; MBLOG_ERROR << msg; diff --git a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc index 20a646260..4ceae4c9e 100644 --- a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc +++ b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc @@ -73,4 +73,4 @@ modelbox::Status FfmpegWriter::Open(const std::string &format_name, std::string FfmpegWriter::GetFormatName() { return format_name_; } std::string FfmpegWriter::GetDestinationURL() { return destination_url_; } -std::shared_ptr FfmpegWriter::GetCtx() { return format_ctx_; } \ No newline at end of file +std::shared_ptr FfmpegWriter::GetCtx() { return format_ctx_; } diff --git a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc new file mode 100644 index 000000000..d55fa4bc0 --- /dev/null +++ b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc @@ -0,0 +1,380 @@ +/* + * Copyright 2022 The Modelbox Project Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "video_out_flowunit.h" + +#include + +#include +#include +#include + +#include "modelbox/flowunit.h" +#include "modelbox/flowunit_api_helper.h" + +modelbox::Status VideoOutFlowUnit::Open( + const std::shared_ptr &opts) { + default_dest_url_ = opts->GetString("default_dest_url", ""); + + return modelbox::STATUS_OK; +} + +modelbox::Status VideoOutFlowUnit::Close() { return modelbox::STATUS_OK; } + +modelbox::Status VideoOutFlowUnit::Process( + std::shared_ptr ctx) { + auto image_queue = std::static_pointer_cast< + modelbox::BlockingQueue>>( + ctx->GetPrivate(SHOW_QUEUE_CTX)); + if (image_queue != nullptr) { + auto input_buffer_list = ctx->Input(FRAME_INFO_INPUT); + for (size_t i = 0; i < input_buffer_list->Size(); ++i) { + image_queue->Push(input_buffer_list->At(i), 50); + } + return modelbox::STATUS_SUCCESS; + } + + // others do video encoding + auto muxer = + std::static_pointer_cast(ctx->GetPrivate(MUXER_CTX)); + auto encoder = std::static_pointer_cast( + ctx->GetPrivate(ENCODER_CTX)); + if (muxer == nullptr || encoder == nullptr) { + MBLOG_ERROR << "Stream not inited"; + return {modelbox::STATUS_FAULT, "Stream not inited"}; + } + + std::vector> av_frame_list; + auto ret = ReadFrames(ctx, av_frame_list); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Read input frame failed"; + return {modelbox::STATUS_FAULT, "Read input frame failed"}; + } + + std::vector> av_packet_list; + ret = EncodeFrame(encoder, av_frame_list, av_packet_list); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Encode frame failed"; + return {modelbox::STATUS_FAULT, "Encode frame failed"}; + } + + ret = MuxPacket(muxer, encoder->GetCtx()->time_base, av_packet_list); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Mux packet failed"; + return {modelbox::STATUS_FAULT, "Mux packet failed"}; + } + + return modelbox::STATUS_SUCCESS; +} + +modelbox::Status VideoOutFlowUnit::ReadFrames( + const std::shared_ptr &ctx, + std::vector> &av_frame_list) { + auto frame_buffer_list = ctx->Input(FRAME_INFO_INPUT); + if (frame_buffer_list == nullptr || frame_buffer_list->Size() == 0) { + MBLOG_ERROR << "Input frame list is empty"; + return {modelbox::STATUS_FAULT, "Input frame list is empty"}; + } + + auto frame_index_ptr = + std::static_pointer_cast(ctx->GetPrivate(FRAME_INDEX_CTX)); + for (auto frame_buffer : *frame_buffer_list) { + std::shared_ptr av_frame; + auto ret = ReadFrameFromBuffer(frame_buffer, av_frame); + av_frame->pts = *frame_index_ptr; + ++(*frame_index_ptr); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Read frame from buffer failed"; + return ret; + } + + av_frame_list.push_back(av_frame); + } + + return modelbox::STATUS_SUCCESS; +} + +modelbox::Status VideoOutFlowUnit::ReadFrameFromBuffer( + std::shared_ptr &frame_buffer, + std::shared_ptr &av_frame) { + auto *frame_ptr = av_frame_alloc(); + if (frame_ptr == nullptr) { + MBLOG_ERROR << "Alloca frame failed"; + return {modelbox::STATUS_FAULT, "Alloca frame failed"}; + } + + av_frame.reset(frame_ptr, [](AVFrame *ptr) { av_frame_free(&ptr); }); + frame_buffer->Get("width", av_frame->width); + frame_buffer->Get("height", av_frame->height); + std::string pix_fmt; + frame_buffer->Get("pix_fmt", pix_fmt); + auto iter = videodecode::g_av_pix_fmt_map.find(pix_fmt); + if (iter == videodecode::g_av_pix_fmt_map.end()) { + MBLOG_ERROR << "Encoder not support pix fmt " << pix_fmt; + return {modelbox::STATUS_NOTSUPPORT, + "Encoder not support pix fmt " + pix_fmt}; + } + av_frame->format = iter->second; + auto ret = + av_image_fill_arrays(av_frame->data, av_frame->linesize, + (const uint8_t *)frame_buffer->ConstData(), + iter->second, av_frame->width, av_frame->height, 1); + if (ret < 0) { + GET_FFMPEG_ERR(ret, ffmpeg_err); + MBLOG_ERROR << "avpicture_fill failed, err " << ffmpeg_err; + return {modelbox::STATUS_FAULT, "avpicture_fill failed, err "}; + } + + return modelbox::STATUS_SUCCESS; +} + +modelbox::Status VideoOutFlowUnit::EncodeFrame( + const std::shared_ptr &encoder, + const std::vector> &av_frame_list, + std::vector> &av_packet_list) { + for (const auto &frame : av_frame_list) { + auto ret = encoder->Encode(GetBindDevice(), frame, av_packet_list); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Encoder encode frame failed reason: " + ret.Errormsg(); + return ret; + } + } + + return modelbox::STATUS_SUCCESS; +} + +modelbox::Status VideoOutFlowUnit::MuxPacket( + const std::shared_ptr &muxer, const AVRational &time_base, + std::vector> &av_packet_list) { + for (const auto &packet : av_packet_list) { + auto ret = muxer->Mux(time_base, packet); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Muxer mux packet failed"; + return ret; + } + } + + return modelbox::STATUS_SUCCESS; +} + +void VideoOutFlowUnit::ProcessShow( + const std::string &dest_url, + const std::shared_ptr< + modelbox::BlockingQueue>> + &image_queue) { + std::string win_name = "modelbox_show"; + if (dest_url.length() > 2) { + win_name = dest_url.substr(2); + } + + cv::namedWindow(win_name, cv::WINDOW_AUTOSIZE); + std::shared_ptr buf; + std::shared_ptr back_buf; + while (image_queue->Pop(&buf)) { + if (buf == nullptr) { + break; + } + + // at least 1, even not set widht, height + int32_t width = 1; + int32_t height = 1; + std::string pix_fmt = "bgr"; + buf->Get("width", width); + buf->Get("height", height); + buf->Get("pix_fmt", pix_fmt); + void *input_data = const_cast(buf->ConstData()); + bool isnv12 = (pix_fmt == "nv12"); + cv::Mat img_data(cv::Size(width, isnv12 ? height * 3 / 2 : height), + isnv12 ? CV_8UC1 : CV_8UC3, input_data); + cv::Mat show_img = img_data; + // todo color change + if (pix_fmt == "rgb") { + cv::cvtColor(img_data, show_img, cv::COLOR_RGB2BGR); + } else if (pix_fmt == "nv12") { + cv::cvtColor(img_data, show_img, cv::COLOR_YUV2BGR_NV12); + } + + cv::imshow(win_name, show_img); + cv::waitKey(10); + back_buf = buf; + } + + cv::destroyWindow(win_name); +} + +modelbox::Status VideoOutFlowUnit::PrepareVideoOut( + const std::shared_ptr &data_ctx, + const std::string &dest_url, const std::string &format_name) { + auto frame_buffer_list = data_ctx->Input(FRAME_INFO_INPUT); + if (frame_buffer_list == nullptr || frame_buffer_list->Size() == 0) { + MBLOG_ERROR << "Input [frame_info] is empty"; + return {modelbox::STATUS_FAULT, "Input [frame_info] is empty"}; + } + + auto frame_buffer = frame_buffer_list->At(0); + int32_t width = 0; + int32_t height = 0; + int32_t rate_num = 25; + int32_t rate_den = 1; + frame_buffer->Get("width", width); + frame_buffer->Get("height", height); + frame_buffer->Get("rate_num", rate_num); + frame_buffer->Get("rate_den", rate_den); + + if (width == 0 || height == 0) { + MBLOG_ERROR << "buffer meta is invalid"; + return {modelbox::STATUS_INVALID, "buffer meta is invalid"}; + } + + auto encoder = std::make_shared(); + auto ret = + encoder->Init(GetBindDevice(), width, height, {rate_num, rate_den}); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Init encoder failed"; + return {modelbox::STATUS_FAULT, "Init encoder failed"}; + } + + auto writer = std::make_shared(); + ret = writer->Open(format_name, dest_url); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Open ffmepg writer failed, format " << format_name + << ", url " << dest_url; + return {modelbox::STATUS_FAULT, "Open ffmepg writer failed, format " + + format_name + ", url " + dest_url}; + } + + auto muxer = std::make_shared(); + ret = muxer->Init(encoder->GetCtx(), writer); + if (ret != modelbox::STATUS_SUCCESS) { + MBLOG_ERROR << "Init muxer failed"; + return {modelbox::STATUS_FAULT, "Init muxer failed"}; + } + + auto color_cvt = std::make_shared(); + + data_ctx->SetPrivate(MUXER_CTX, muxer); + data_ctx->SetPrivate(ENCODER_CTX, encoder); + data_ctx->SetPrivate(COLOR_CVT_CTX, color_cvt); + return modelbox::STATUS_OK; +} + +modelbox::Status VideoOutFlowUnit::DataPre( + std::shared_ptr data_ctx) { + std::string dest_url; + auto ret = GetDestUrl(data_ctx, dest_url); + if (ret != modelbox::STATUS_SUCCESS || dest_url.empty()) { + MBLOG_ERROR << "dest_url in config is empty, no dest url available"; + return {modelbox::STATUS_FAULT, + "dest_url in config is empty, no dest url available"}; + } + + MBLOG_INFO << "videoout url=" << dest_url; + + auto frame_index_ptr = std::make_shared(0); + data_ctx->SetPrivate(FRAME_INDEX_CTX, frame_index_ptr); + + if (dest_url[0] >= '0' && dest_url[0] <= '9') { + // 视频输出， 类似0:windows_name配置 + std::shared_ptr show_thread; + auto image_queue = std::make_shared< + modelbox::BlockingQueue>>(2); + show_thread.reset(new std::thread(&VideoOutFlowUnit::ProcessShow, this, + dest_url, image_queue), + [image_queue](std::thread *p) { + image_queue->Shutdown(); + if (p && p->joinable()) { + p->join(); + } + delete p; + }); + data_ctx->SetPrivate(SHOW_CTX, show_thread); + data_ctx->SetPrivate(SHOW_QUEUE_CTX, image_queue); + return modelbox::STATUS_OK; + } + + std::string format_name = "mp4"; + if (dest_url.substr(0, 4) == "rtsp") { + format_name = "rtsp"; + } + + return PrepareVideoOut(data_ctx, dest_url, format_name); +} + +modelbox::Status VideoOutFlowUnit::GetDestUrl( + const std::shared_ptr &data_ctx, + std::string &dest_url) { + dest_url = default_dest_url_; + + Defer { + std::regex url_auth_pattern("://[^ /]*?:[^ /]*?@"); + auto result = std::regex_replace(dest_url, url_auth_pattern, "://*:*@"); + MBLOG_INFO << "video_out url is " << result; + }; + + // 3种方式获取 + auto stream_meta = data_ctx->GetInputMeta(FRAME_INFO_INPUT); + if (stream_meta != nullptr) { + auto dest_url_ptr = + std::static_pointer_cast(stream_meta->GetMeta(DEST_URL)); + if (dest_url_ptr != nullptr && !(*dest_url_ptr).empty()) { + dest_url = *dest_url_ptr; + return modelbox::STATUS_SUCCESS; + } + } + + auto config = data_ctx->GetSessionConfig(); + auto cfg_str = config->GetString("iva_task_output"); + if (cfg_str.empty()) { + return modelbox::STATUS_SUCCESS; + } + + nlohmann::json url_json = nlohmann::json::parse(cfg_str); + if (url_json.contains("data") && url_json["data"].contains("url")) { + dest_url = url_json["data"]["url"].get(); + } + + return modelbox::STATUS_SUCCESS; +} + +modelbox::Status VideoOutFlowUnit::DataPost( + std::shared_ptr data_ctx) { + data_ctx->SetPrivate(MUXER_CTX, nullptr); + data_ctx->SetPrivate(ENCODER_CTX, nullptr); + data_ctx->SetPrivate(SHOW_CTX, nullptr); + data_ctx->SetPrivate(SHOW_QUEUE_CTX, nullptr); + return modelbox::STATUS_OK; +} + +MODELBOX_FLOWUNIT(VideoOutFlowUnit, desc) { + desc.SetFlowUnitName(FLOWUNIT_NAME); + desc.SetFlowUnitGroupType("Video"); + desc.AddFlowUnitInput({FRAME_INFO_INPUT, "cpu"}); + desc.SetFlowType(modelbox::STREAM); + desc.SetInputContiguous(false); + // 禁止异步执行，编码必须一帧帧的编码 + desc.SetResourceNice(false); + desc.SetDescription(FLOWUNIT_DESC); + desc.AddFlowUnitOption(modelbox::FlowUnitOption( + "default_dest_url", "string", true, "", "the encoder dest url")); +} + +MODELBOX_DRIVER_FLOWUNIT(desc) { + desc.Desc.SetName(FLOWUNIT_NAME); + desc.Desc.SetClass(modelbox::DRIVER_CLASS_FLOWUNIT); + desc.Desc.SetType(FLOWUNIT_TYPE); + desc.Desc.SetDescription(FLOWUNIT_DESC); + desc.Desc.SetVersion("1.0.0"); +} diff --git a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h new file mode 100644 index 000000000..a3122c62c --- /dev/null +++ b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h @@ -0,0 +1,124 @@ +/* + * Copyright 2022 The Modelbox Project Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MODELBOX_FLOWUNIT_VIDEO_OUT_CPU_H_ +#define MODELBOX_FLOWUNIT_VIDEO_OUT_CPU_H_ + +#include +#include +#include + +#include + +#include "ffmpeg_video_encoder.h" +#include "ffmpeg_video_muxer.h" +#include "ffmpeg_writer.h" +#include "modelbox/flowunit.h" +#include "video_decode_common.h" + +constexpr const char *FLOWUNIT_TYPE = "rockchip"; +constexpr const char *FLOWUNIT_NAME = "video_out"; +constexpr const char *FLOWUNIT_DESC = + "\n\t@Brief: A video out flowunit. \n" + "\t@Port parameter: The input port buffer meta type is image \n" + "\t The image type buffer contains the following meta fields:\n" + "\t\tField Name: width, Type: int32_t\n" + "\t\tField Name: height, Type: int32_t\n" + "\t\tField Name: width_stride, Type: int32_t\n" + "\t\tField Name: height_stride, Type: int32_t\n" + "\t\tField Name: channel, Type: int32_t\n" + "\t\tField Name: pix_fmt, Type: string\n" + "\t\tField Name: layout, Type: int32_t\n" + "\t\tField Name: shape, Type: vector\n" + "\t\tField Name: type, Type: ModelBoxDataType::MODELBOX_UINT8\n" + "\t@Constraint: The field value range of this flowunit supports: " + "'pix_fmt': " + "[rgb, bgr, nv12], 'layout': [hwc]. "; +constexpr const char *DEST_URL = "dest_url"; +constexpr const char *COLOR_CVT_CTX = "color_cvt_ctx"; +constexpr const char *FRAME_INDEX_CTX = "frame_index_ctx"; +constexpr const char *ENCODER_CTX = "encoder_ctx"; +constexpr const char *MUXER_CTX = "muxer_ctx"; +constexpr const char *SHOW_CTX = "show_ctx"; +constexpr const char *SHOW_QUEUE_CTX = "show_queue_ctx"; +constexpr const char *FORMAT_NAME = "format_name"; +constexpr const char *CODEC_NAME = "codec_name"; +constexpr const char *DESTINATION_URL = "destination_url"; +constexpr const char *FRAME_INFO_INPUT = "in_video_frame"; + +class VideoOutFlowUnit : public modelbox::FlowUnit { + public: + VideoOutFlowUnit() = default; + ~VideoOutFlowUnit() override = default; + + modelbox::Status Open( + const std::shared_ptr &opts) override; + + modelbox::Status Close() override; + + modelbox::Status Process( + std::shared_ptr data_ctx) override; + + modelbox::Status DataPre( + std::shared_ptr data_ctx) override; + + modelbox::Status DataPost( + std::shared_ptr data_ctx) override; + + modelbox::Status DataGroupPre( + std::shared_ptr data_ctx) override { + return modelbox::STATUS_OK; + }; + + modelbox::Status DataGroupPost( + std::shared_ptr data_ctx) override { + return modelbox::STATUS_OK; + }; + + private: + modelbox::Status GetDestUrl( + const std::shared_ptr &data_ctx, + std::string &dest_url); + + modelbox::Status ReadFrames( + const std::shared_ptr &ctx, + std::vector> &av_frame_list); + + modelbox::Status ReadFrameFromBuffer( + std::shared_ptr &frame_buffer, + std::shared_ptr &av_frame); + + modelbox::Status EncodeFrame( + const std::shared_ptr &encoder, + const std::vector> &av_frame_list, + std::vector> &av_packet_list); + + modelbox::Status MuxPacket( + const std::shared_ptr &muxer, + const AVRational &time_base, + std::vector> &av_packet_list); + + void ProcessShow(const std::string &dest_url, + const std::shared_ptr>> &image_queue); + modelbox::Status PrepareVideoOut( + const std::shared_ptr &data_ctx, + const std::string &dest_url, const std::string &format_name); + + std::string default_dest_url_; +}; + +#endif // MODELBOX_FLOWUNIT_VIDEO_OUT_CPU_H_ diff --git a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc new file mode 100644 index 000000000..814f1c423 --- /dev/null +++ b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc @@ -0,0 +1,186 @@ +/* + * Copyright 2022 The Modelbox Project Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "driver_flow_test.h" +#include "flowunit_mockflowunit/flowunit_mockflowunit.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "modelbox/base/log.h" +#include "modelbox/buffer.h" +#include "test/mock/minimodelbox/mockflow.h" + +namespace modelbox { +class RockchipVideoEncoderFlowUnitTest : public testing::Test { + public: + RockchipVideoEncoderFlowUnitTest() : flow_(std::make_shared()){}; + + protected: + void SetUp() override{}; + + void TearDown() override{}; + + public: + std::shared_ptr flow_; + + void StartFlow(const std::string& toml_content, uint64_t millisecond); + + private: + Status AddMockFlowUnit(); +}; + +void RockchipVideoEncoderFlowUnitTest::StartFlow( + const std::string& toml_content, uint64_t millisecond) { + flow_ = std::make_shared(); + auto ret = AddMockFlowUnit(); + EXPECT_EQ(ret, STATUS_SUCCESS); + + ret = flow_->BuildAndRun("VideoEncoder", toml_content, millisecond); + EXPECT_EQ(ret, STATUS_STOP); +} + +Status RockchipVideoEncoderFlowUnitTest::AddMockFlowUnit() { + { + auto mock_desc = + GenerateFlowunitDesc("encoder_start_unit", {}, {"stream_meta"}); + auto open_func = + [=](const std::shared_ptr& flow_option, + const std::shared_ptr& mock_flowunit) { + auto ext_data = mock_flowunit->CreateExternalData(); + EXPECT_NE(ext_data, nullptr); + auto buffer_list = ext_data->CreateBufferList(); + buffer_list->Build({1}); + auto status = ext_data->Send(buffer_list); + EXPECT_EQ(status, STATUS_SUCCESS); + status = ext_data->Close(); + EXPECT_EQ(status, STATUS_SUCCESS); + return modelbox::STATUS_OK; + }; + auto data_pre_func = + [&](const std::shared_ptr& data_ctx, + const std::shared_ptr& mock_flowunit) { + MBLOG_INFO << "stream_meta " + << "DataPre"; + auto test_meta = std::make_shared("test"); + auto data_meta = std::make_shared(); + data_meta->SetMeta("test", test_meta); + data_ctx->SetOutputMeta("stream_meta", data_meta); + return modelbox::STATUS_OK; + }; + auto process_func = + [=](const std::shared_ptr& data_ctx, + const std::shared_ptr& mock_flowunit) { + auto output_buf = data_ctx->Output("stream_meta"); + std::vector shape(1, 1); + output_buf->Build(shape); + + return modelbox::STATUS_OK; + }; + auto mock_functions = std::make_shared(); + mock_functions->RegisterOpenFunc(open_func); + mock_functions->RegisterDataPreFunc(data_pre_func); + mock_functions->RegisterProcessFunc(process_func); + flow_->AddFlowUnitDesc(mock_desc, mock_functions->GenerateCreateFunc(), + TEST_DRIVER_DIR); + } + { + auto mock_desc = GenerateFlowunitDesc("encoder_image_produce", + {"stream_meta"}, {"frame_info"}); + mock_desc->SetOutputType(EXPAND); + auto process_func = + [=](const std::shared_ptr& data_ctx, + const std::shared_ptr& mock_flowunit) { + std::string img_path; + static int64_t frame_index = 0; + if ((frame_index / 24) % 2 == 0) { + img_path = + std::string(TEST_ASSETS) + "/video/rgb_460800_480x320_a.data"; + } else { + img_path = + std::string(TEST_ASSETS) + "/video/rgb_460800_480x320_b.data"; + } + + std::ifstream img_file(img_path); + if (!img_file.is_open()) { + MBLOG_ERROR << "Open failed, path " << img_path; + return STATUS_FAULT; + } + + size_t file_size = 460800; + auto output_buff_list = data_ctx->Output("frame_info"); + std::vector shape(1, file_size); + output_buff_list->Build(shape); + auto output_buff = output_buff_list->At(0); + auto* ptr = (char*)output_buff->MutableData(); + img_file.read(ptr, file_size); + output_buff->Set("width", 480); + output_buff->Set("height", 320); + output_buff->Set("rate_num", 24); + output_buff->Set("rate_den", 1); + output_buff->Set("pix_fmt", std::string("rgb")); + output_buff->Set("index", frame_index); + + if (frame_index == 1339) { // 60S + return modelbox::STATUS_STOP; + } + + ++frame_index; + auto event = std::make_shared(); + data_ctx->SendEvent(event); + return modelbox::STATUS_CONTINUE; + }; + auto mock_functions = std::make_shared(); + mock_functions->RegisterProcessFunc(process_func); + flow_->AddFlowUnitDesc(mock_desc, mock_functions->GenerateCreateFunc(), + TEST_DRIVER_DIR); + } + + return STATUS_SUCCESS; +} + +TEST_F(RockchipVideoEncoderFlowUnitTest, InitUnit) { + const std::string test_lib_dir = TEST_DRIVER_DIR; + const std::string test_data_dir = TEST_DATA_DIR; + auto ret = system("nc localhost 554 -z"); + if (errno != 0 || ret != 0) { + GTEST_SKIP(); + } + + std::string dest_url = "rtsp://localhost/test_" + std::to_string(rand()); + std::string toml_content = R"( + [driver] + skip-default = true + dir=[")" + test_lib_dir + + "\",\"" + test_data_dir + "\"]\n " + + R"([graph] + graphconf = '''digraph demo { + encoder_start_unit[type=flowunit, flowunit=encoder_start_unit, device=cpu, deviceid=0, label=""] + encoder_image_produce[type=flowunit, flowunit=encoder_image_produce, device=cpu, deviceid=0, label=" | "] + videoencoder[type=flowunit, flowunit=video_encoder, device=cpu, deviceid=0, label="", queue_size_frame_info=16, default_dest_url=")" + + dest_url + R"("] + encoder_start_unit:stream_meta -> encoder_image_produce:stream_meta + encoder_image_produce:frame_info -> videoencoder:in_video_frame + }''' + format = "graphviz" + )"; + StartFlow(toml_content, 1000 * 1000); +} + +} // namespace modelbox \ No newline at end of file diff --git a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h index d78d3763c..d111b276a 100644 --- a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h @@ -65,6 +65,7 @@ class RockChipLocalCameraFlowUnit : public modelbox::FlowUnit { std::shared_ptr &img_buf); MppFrame ProcessYVY2(const uint8_t *buf, size_t size, size_t w, size_t h, std::shared_ptr &img_buf); + MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt, MppBuffer mpp_buf); diff --git a/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc b/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc index 1791ee66f..57e59fb3e 100644 --- a/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc +++ b/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc @@ -111,7 +111,7 @@ int32_t V4L2Camera::GetCamfd(int32_t id, const std::string &bus_info) { } Defer { globfree(&glob_result); }; - + // detect it is a camera device if (modelbox::STATUS_SUCCESS != CamIoCtl(fd, VIDIOC_QUERYCAP, &cap)) { MBLOG_DEBUG << "Not v4l2 device:" << glob_result.gl_pathv[i]; diff --git a/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h b/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h index 7a348cd8a..f09e95228 100644 --- a/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h @@ -42,6 +42,7 @@ constexpr const char *FLOWUNIT_DESC = "\t@Constraint: The field value range of this flowunit supports: " "'pix_fmt': " "[rgb_packed,bgr_packed], 'layout': [hwc]. "; + constexpr const char *IN_IMG = "in_image"; constexpr const char *OUT_IMG = "out_image"; diff --git a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h index b4104fc47..e59f5d8ee 100644 --- a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h @@ -34,6 +34,7 @@ #include "rga.h" constexpr const char *FLOWUNIT_NAME = "video_decoder"; + constexpr const char *FLOWUNIT_DESC = "\n\t@Brief: A video decoder flowunit on rockchip. \n" "\t@Port parameter: The input port buffer type is video_packet, the output " @@ -64,6 +65,7 @@ constexpr const char *FLOWUNIT_DESC = "\t@Constraint: The flowuint 'video_decoder' must be used pair " "with 'video_demuxer. the output buffer meta fields 'pix_fmt' is " "'brg_packed' or 'rgb_packed', 'layout' is 'hcw'."; + constexpr const char *VIDEO_PACKET_INPUT = "in_video_packet"; constexpr const char *FRAME_INFO_OUTPUT = "out_video_frame"; constexpr const char *CODEC_META = "codec_meta"; diff --git a/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc b/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc index e5883a24f..482ede4f8 100644 --- a/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc +++ b/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc @@ -360,6 +360,7 @@ std::shared_ptr FfmpegVideoEncoder::NewPacket(MppPacket &packet) { size_t len = mpp_packet_get_length(packet); auto *av_packet_ptr = av_packet_alloc(); + Defer { mpp_packet_deinit(&packet); }; if (av_packet_ptr == nullptr) { MBLOG_ERROR << "av packet alloc failed"; From 1d50296b5650dcc8ffa29a5d9050488a2893dbe6 Mon Sep 17 00:00:00 2001 From: fujialin <757524349@qq.com> Date: Mon, 21 Nov 2022 16:03:48 +0800 Subject: [PATCH 2/4] modified review comments --- .../video_out/video_out_flowunit.cc | 380 ------------------ .../common_src/video_out/video_out_flowunit.h | 124 ------ .../video_out/video_out_flowunit_test.cc | 186 --------- .../local_camera/local_camera_flowunit.h | 2 + .../flowunit/local_camera/v4l2_camera.cc | 2 +- .../flowunit/to_cpuimg/mpp_to_cpu_flowunit.h | 2 +- 6 files changed, 4 insertions(+), 692 deletions(-) delete mode 100644 src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc delete mode 100644 src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h delete mode 100644 src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc diff --git a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc deleted file mode 100644 index d55fa4bc0..000000000 --- a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.cc +++ /dev/null @@ -1,380 +0,0 @@ -/* - * Copyright 2022 The Modelbox Project Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "video_out_flowunit.h" - -#include - -#include -#include -#include - -#include "modelbox/flowunit.h" -#include "modelbox/flowunit_api_helper.h" - -modelbox::Status VideoOutFlowUnit::Open( - const std::shared_ptr &opts) { - default_dest_url_ = opts->GetString("default_dest_url", ""); - - return modelbox::STATUS_OK; -} - -modelbox::Status VideoOutFlowUnit::Close() { return modelbox::STATUS_OK; } - -modelbox::Status VideoOutFlowUnit::Process( - std::shared_ptr ctx) { - auto image_queue = std::static_pointer_cast< - modelbox::BlockingQueue>>( - ctx->GetPrivate(SHOW_QUEUE_CTX)); - if (image_queue != nullptr) { - auto input_buffer_list = ctx->Input(FRAME_INFO_INPUT); - for (size_t i = 0; i < input_buffer_list->Size(); ++i) { - image_queue->Push(input_buffer_list->At(i), 50); - } - return modelbox::STATUS_SUCCESS; - } - - // others do video encoding - auto muxer = - std::static_pointer_cast(ctx->GetPrivate(MUXER_CTX)); - auto encoder = std::static_pointer_cast( - ctx->GetPrivate(ENCODER_CTX)); - if (muxer == nullptr || encoder == nullptr) { - MBLOG_ERROR << "Stream not inited"; - return {modelbox::STATUS_FAULT, "Stream not inited"}; - } - - std::vector> av_frame_list; - auto ret = ReadFrames(ctx, av_frame_list); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Read input frame failed"; - return {modelbox::STATUS_FAULT, "Read input frame failed"}; - } - - std::vector> av_packet_list; - ret = EncodeFrame(encoder, av_frame_list, av_packet_list); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Encode frame failed"; - return {modelbox::STATUS_FAULT, "Encode frame failed"}; - } - - ret = MuxPacket(muxer, encoder->GetCtx()->time_base, av_packet_list); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Mux packet failed"; - return {modelbox::STATUS_FAULT, "Mux packet failed"}; - } - - return modelbox::STATUS_SUCCESS; -} - -modelbox::Status VideoOutFlowUnit::ReadFrames( - const std::shared_ptr &ctx, - std::vector> &av_frame_list) { - auto frame_buffer_list = ctx->Input(FRAME_INFO_INPUT); - if (frame_buffer_list == nullptr || frame_buffer_list->Size() == 0) { - MBLOG_ERROR << "Input frame list is empty"; - return {modelbox::STATUS_FAULT, "Input frame list is empty"}; - } - - auto frame_index_ptr = - std::static_pointer_cast(ctx->GetPrivate(FRAME_INDEX_CTX)); - for (auto frame_buffer : *frame_buffer_list) { - std::shared_ptr av_frame; - auto ret = ReadFrameFromBuffer(frame_buffer, av_frame); - av_frame->pts = *frame_index_ptr; - ++(*frame_index_ptr); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Read frame from buffer failed"; - return ret; - } - - av_frame_list.push_back(av_frame); - } - - return modelbox::STATUS_SUCCESS; -} - -modelbox::Status VideoOutFlowUnit::ReadFrameFromBuffer( - std::shared_ptr &frame_buffer, - std::shared_ptr &av_frame) { - auto *frame_ptr = av_frame_alloc(); - if (frame_ptr == nullptr) { - MBLOG_ERROR << "Alloca frame failed"; - return {modelbox::STATUS_FAULT, "Alloca frame failed"}; - } - - av_frame.reset(frame_ptr, [](AVFrame *ptr) { av_frame_free(&ptr); }); - frame_buffer->Get("width", av_frame->width); - frame_buffer->Get("height", av_frame->height); - std::string pix_fmt; - frame_buffer->Get("pix_fmt", pix_fmt); - auto iter = videodecode::g_av_pix_fmt_map.find(pix_fmt); - if (iter == videodecode::g_av_pix_fmt_map.end()) { - MBLOG_ERROR << "Encoder not support pix fmt " << pix_fmt; - return {modelbox::STATUS_NOTSUPPORT, - "Encoder not support pix fmt " + pix_fmt}; - } - av_frame->format = iter->second; - auto ret = - av_image_fill_arrays(av_frame->data, av_frame->linesize, - (const uint8_t *)frame_buffer->ConstData(), - iter->second, av_frame->width, av_frame->height, 1); - if (ret < 0) { - GET_FFMPEG_ERR(ret, ffmpeg_err); - MBLOG_ERROR << "avpicture_fill failed, err " << ffmpeg_err; - return {modelbox::STATUS_FAULT, "avpicture_fill failed, err "}; - } - - return modelbox::STATUS_SUCCESS; -} - -modelbox::Status VideoOutFlowUnit::EncodeFrame( - const std::shared_ptr &encoder, - const std::vector> &av_frame_list, - std::vector> &av_packet_list) { - for (const auto &frame : av_frame_list) { - auto ret = encoder->Encode(GetBindDevice(), frame, av_packet_list); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Encoder encode frame failed reason: " + ret.Errormsg(); - return ret; - } - } - - return modelbox::STATUS_SUCCESS; -} - -modelbox::Status VideoOutFlowUnit::MuxPacket( - const std::shared_ptr &muxer, const AVRational &time_base, - std::vector> &av_packet_list) { - for (const auto &packet : av_packet_list) { - auto ret = muxer->Mux(time_base, packet); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Muxer mux packet failed"; - return ret; - } - } - - return modelbox::STATUS_SUCCESS; -} - -void VideoOutFlowUnit::ProcessShow( - const std::string &dest_url, - const std::shared_ptr< - modelbox::BlockingQueue>> - &image_queue) { - std::string win_name = "modelbox_show"; - if (dest_url.length() > 2) { - win_name = dest_url.substr(2); - } - - cv::namedWindow(win_name, cv::WINDOW_AUTOSIZE); - std::shared_ptr buf; - std::shared_ptr back_buf; - while (image_queue->Pop(&buf)) { - if (buf == nullptr) { - break; - } - - // at least 1, even not set widht, height - int32_t width = 1; - int32_t height = 1; - std::string pix_fmt = "bgr"; - buf->Get("width", width); - buf->Get("height", height); - buf->Get("pix_fmt", pix_fmt); - void *input_data = const_cast(buf->ConstData()); - bool isnv12 = (pix_fmt == "nv12"); - cv::Mat img_data(cv::Size(width, isnv12 ? height * 3 / 2 : height), - isnv12 ? CV_8UC1 : CV_8UC3, input_data); - cv::Mat show_img = img_data; - // todo color change - if (pix_fmt == "rgb") { - cv::cvtColor(img_data, show_img, cv::COLOR_RGB2BGR); - } else if (pix_fmt == "nv12") { - cv::cvtColor(img_data, show_img, cv::COLOR_YUV2BGR_NV12); - } - - cv::imshow(win_name, show_img); - cv::waitKey(10); - back_buf = buf; - } - - cv::destroyWindow(win_name); -} - -modelbox::Status VideoOutFlowUnit::PrepareVideoOut( - const std::shared_ptr &data_ctx, - const std::string &dest_url, const std::string &format_name) { - auto frame_buffer_list = data_ctx->Input(FRAME_INFO_INPUT); - if (frame_buffer_list == nullptr || frame_buffer_list->Size() == 0) { - MBLOG_ERROR << "Input [frame_info] is empty"; - return {modelbox::STATUS_FAULT, "Input [frame_info] is empty"}; - } - - auto frame_buffer = frame_buffer_list->At(0); - int32_t width = 0; - int32_t height = 0; - int32_t rate_num = 25; - int32_t rate_den = 1; - frame_buffer->Get("width", width); - frame_buffer->Get("height", height); - frame_buffer->Get("rate_num", rate_num); - frame_buffer->Get("rate_den", rate_den); - - if (width == 0 || height == 0) { - MBLOG_ERROR << "buffer meta is invalid"; - return {modelbox::STATUS_INVALID, "buffer meta is invalid"}; - } - - auto encoder = std::make_shared(); - auto ret = - encoder->Init(GetBindDevice(), width, height, {rate_num, rate_den}); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Init encoder failed"; - return {modelbox::STATUS_FAULT, "Init encoder failed"}; - } - - auto writer = std::make_shared(); - ret = writer->Open(format_name, dest_url); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Open ffmepg writer failed, format " << format_name - << ", url " << dest_url; - return {modelbox::STATUS_FAULT, "Open ffmepg writer failed, format " + - format_name + ", url " + dest_url}; - } - - auto muxer = std::make_shared(); - ret = muxer->Init(encoder->GetCtx(), writer); - if (ret != modelbox::STATUS_SUCCESS) { - MBLOG_ERROR << "Init muxer failed"; - return {modelbox::STATUS_FAULT, "Init muxer failed"}; - } - - auto color_cvt = std::make_shared(); - - data_ctx->SetPrivate(MUXER_CTX, muxer); - data_ctx->SetPrivate(ENCODER_CTX, encoder); - data_ctx->SetPrivate(COLOR_CVT_CTX, color_cvt); - return modelbox::STATUS_OK; -} - -modelbox::Status VideoOutFlowUnit::DataPre( - std::shared_ptr data_ctx) { - std::string dest_url; - auto ret = GetDestUrl(data_ctx, dest_url); - if (ret != modelbox::STATUS_SUCCESS || dest_url.empty()) { - MBLOG_ERROR << "dest_url in config is empty, no dest url available"; - return {modelbox::STATUS_FAULT, - "dest_url in config is empty, no dest url available"}; - } - - MBLOG_INFO << "videoout url=" << dest_url; - - auto frame_index_ptr = std::make_shared(0); - data_ctx->SetPrivate(FRAME_INDEX_CTX, frame_index_ptr); - - if (dest_url[0] >= '0' && dest_url[0] <= '9') { - // 视频输出， 类似0:windows_name配置 - std::shared_ptr show_thread; - auto image_queue = std::make_shared< - modelbox::BlockingQueue>>(2); - show_thread.reset(new std::thread(&VideoOutFlowUnit::ProcessShow, this, - dest_url, image_queue), - [image_queue](std::thread *p) { - image_queue->Shutdown(); - if (p && p->joinable()) { - p->join(); - } - delete p; - }); - data_ctx->SetPrivate(SHOW_CTX, show_thread); - data_ctx->SetPrivate(SHOW_QUEUE_CTX, image_queue); - return modelbox::STATUS_OK; - } - - std::string format_name = "mp4"; - if (dest_url.substr(0, 4) == "rtsp") { - format_name = "rtsp"; - } - - return PrepareVideoOut(data_ctx, dest_url, format_name); -} - -modelbox::Status VideoOutFlowUnit::GetDestUrl( - const std::shared_ptr &data_ctx, - std::string &dest_url) { - dest_url = default_dest_url_; - - Defer { - std::regex url_auth_pattern("://[^ /]*?:[^ /]*?@"); - auto result = std::regex_replace(dest_url, url_auth_pattern, "://*:*@"); - MBLOG_INFO << "video_out url is " << result; - }; - - // 3种方式获取 - auto stream_meta = data_ctx->GetInputMeta(FRAME_INFO_INPUT); - if (stream_meta != nullptr) { - auto dest_url_ptr = - std::static_pointer_cast(stream_meta->GetMeta(DEST_URL)); - if (dest_url_ptr != nullptr && !(*dest_url_ptr).empty()) { - dest_url = *dest_url_ptr; - return modelbox::STATUS_SUCCESS; - } - } - - auto config = data_ctx->GetSessionConfig(); - auto cfg_str = config->GetString("iva_task_output"); - if (cfg_str.empty()) { - return modelbox::STATUS_SUCCESS; - } - - nlohmann::json url_json = nlohmann::json::parse(cfg_str); - if (url_json.contains("data") && url_json["data"].contains("url")) { - dest_url = url_json["data"]["url"].get(); - } - - return modelbox::STATUS_SUCCESS; -} - -modelbox::Status VideoOutFlowUnit::DataPost( - std::shared_ptr data_ctx) { - data_ctx->SetPrivate(MUXER_CTX, nullptr); - data_ctx->SetPrivate(ENCODER_CTX, nullptr); - data_ctx->SetPrivate(SHOW_CTX, nullptr); - data_ctx->SetPrivate(SHOW_QUEUE_CTX, nullptr); - return modelbox::STATUS_OK; -} - -MODELBOX_FLOWUNIT(VideoOutFlowUnit, desc) { - desc.SetFlowUnitName(FLOWUNIT_NAME); - desc.SetFlowUnitGroupType("Video"); - desc.AddFlowUnitInput({FRAME_INFO_INPUT, "cpu"}); - desc.SetFlowType(modelbox::STREAM); - desc.SetInputContiguous(false); - // 禁止异步执行，编码必须一帧帧的编码 - desc.SetResourceNice(false); - desc.SetDescription(FLOWUNIT_DESC); - desc.AddFlowUnitOption(modelbox::FlowUnitOption( - "default_dest_url", "string", true, "", "the encoder dest url")); -} - -MODELBOX_DRIVER_FLOWUNIT(desc) { - desc.Desc.SetName(FLOWUNIT_NAME); - desc.Desc.SetClass(modelbox::DRIVER_CLASS_FLOWUNIT); - desc.Desc.SetType(FLOWUNIT_TYPE); - desc.Desc.SetDescription(FLOWUNIT_DESC); - desc.Desc.SetVersion("1.0.0"); -} diff --git a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h deleted file mode 100644 index a3122c62c..000000000 --- a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2022 The Modelbox Project Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MODELBOX_FLOWUNIT_VIDEO_OUT_CPU_H_ -#define MODELBOX_FLOWUNIT_VIDEO_OUT_CPU_H_ - -#include -#include -#include - -#include - -#include "ffmpeg_video_encoder.h" -#include "ffmpeg_video_muxer.h" -#include "ffmpeg_writer.h" -#include "modelbox/flowunit.h" -#include "video_decode_common.h" - -constexpr const char *FLOWUNIT_TYPE = "rockchip"; -constexpr const char *FLOWUNIT_NAME = "video_out"; -constexpr const char *FLOWUNIT_DESC = - "\n\t@Brief: A video out flowunit. \n" - "\t@Port parameter: The input port buffer meta type is image \n" - "\t The image type buffer contains the following meta fields:\n" - "\t\tField Name: width, Type: int32_t\n" - "\t\tField Name: height, Type: int32_t\n" - "\t\tField Name: width_stride, Type: int32_t\n" - "\t\tField Name: height_stride, Type: int32_t\n" - "\t\tField Name: channel, Type: int32_t\n" - "\t\tField Name: pix_fmt, Type: string\n" - "\t\tField Name: layout, Type: int32_t\n" - "\t\tField Name: shape, Type: vector\n" - "\t\tField Name: type, Type: ModelBoxDataType::MODELBOX_UINT8\n" - "\t@Constraint: The field value range of this flowunit supports: " - "'pix_fmt': " - "[rgb, bgr, nv12], 'layout': [hwc]. "; -constexpr const char *DEST_URL = "dest_url"; -constexpr const char *COLOR_CVT_CTX = "color_cvt_ctx"; -constexpr const char *FRAME_INDEX_CTX = "frame_index_ctx"; -constexpr const char *ENCODER_CTX = "encoder_ctx"; -constexpr const char *MUXER_CTX = "muxer_ctx"; -constexpr const char *SHOW_CTX = "show_ctx"; -constexpr const char *SHOW_QUEUE_CTX = "show_queue_ctx"; -constexpr const char *FORMAT_NAME = "format_name"; -constexpr const char *CODEC_NAME = "codec_name"; -constexpr const char *DESTINATION_URL = "destination_url"; -constexpr const char *FRAME_INFO_INPUT = "in_video_frame"; - -class VideoOutFlowUnit : public modelbox::FlowUnit { - public: - VideoOutFlowUnit() = default; - ~VideoOutFlowUnit() override = default; - - modelbox::Status Open( - const std::shared_ptr &opts) override; - - modelbox::Status Close() override; - - modelbox::Status Process( - std::shared_ptr data_ctx) override; - - modelbox::Status DataPre( - std::shared_ptr data_ctx) override; - - modelbox::Status DataPost( - std::shared_ptr data_ctx) override; - - modelbox::Status DataGroupPre( - std::shared_ptr data_ctx) override { - return modelbox::STATUS_OK; - }; - - modelbox::Status DataGroupPost( - std::shared_ptr data_ctx) override { - return modelbox::STATUS_OK; - }; - - private: - modelbox::Status GetDestUrl( - const std::shared_ptr &data_ctx, - std::string &dest_url); - - modelbox::Status ReadFrames( - const std::shared_ptr &ctx, - std::vector> &av_frame_list); - - modelbox::Status ReadFrameFromBuffer( - std::shared_ptr &frame_buffer, - std::shared_ptr &av_frame); - - modelbox::Status EncodeFrame( - const std::shared_ptr &encoder, - const std::vector> &av_frame_list, - std::vector> &av_packet_list); - - modelbox::Status MuxPacket( - const std::shared_ptr &muxer, - const AVRational &time_base, - std::vector> &av_packet_list); - - void ProcessShow(const std::string &dest_url, - const std::shared_ptr>> &image_queue); - modelbox::Status PrepareVideoOut( - const std::shared_ptr &data_ctx, - const std::string &dest_url, const std::string &format_name); - - std::string default_dest_url_; -}; - -#endif // MODELBOX_FLOWUNIT_VIDEO_OUT_CPU_H_ diff --git a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc b/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc deleted file mode 100644 index 814f1c423..000000000 --- a/src/drivers/devices/rockchip/common_src/video_out/video_out_flowunit_test.cc +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Copyright 2022 The Modelbox Project Authors. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "driver_flow_test.h" -#include "flowunit_mockflowunit/flowunit_mockflowunit.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "modelbox/base/log.h" -#include "modelbox/buffer.h" -#include "test/mock/minimodelbox/mockflow.h" - -namespace modelbox { -class RockchipVideoEncoderFlowUnitTest : public testing::Test { - public: - RockchipVideoEncoderFlowUnitTest() : flow_(std::make_shared()){}; - - protected: - void SetUp() override{}; - - void TearDown() override{}; - - public: - std::shared_ptr flow_; - - void StartFlow(const std::string& toml_content, uint64_t millisecond); - - private: - Status AddMockFlowUnit(); -}; - -void RockchipVideoEncoderFlowUnitTest::StartFlow( - const std::string& toml_content, uint64_t millisecond) { - flow_ = std::make_shared(); - auto ret = AddMockFlowUnit(); - EXPECT_EQ(ret, STATUS_SUCCESS); - - ret = flow_->BuildAndRun("VideoEncoder", toml_content, millisecond); - EXPECT_EQ(ret, STATUS_STOP); -} - -Status RockchipVideoEncoderFlowUnitTest::AddMockFlowUnit() { - { - auto mock_desc = - GenerateFlowunitDesc("encoder_start_unit", {}, {"stream_meta"}); - auto open_func = - [=](const std::shared_ptr& flow_option, - const std::shared_ptr& mock_flowunit) { - auto ext_data = mock_flowunit->CreateExternalData(); - EXPECT_NE(ext_data, nullptr); - auto buffer_list = ext_data->CreateBufferList(); - buffer_list->Build({1}); - auto status = ext_data->Send(buffer_list); - EXPECT_EQ(status, STATUS_SUCCESS); - status = ext_data->Close(); - EXPECT_EQ(status, STATUS_SUCCESS); - return modelbox::STATUS_OK; - }; - auto data_pre_func = - [&](const std::shared_ptr& data_ctx, - const std::shared_ptr& mock_flowunit) { - MBLOG_INFO << "stream_meta " - << "DataPre"; - auto test_meta = std::make_shared("test"); - auto data_meta = std::make_shared(); - data_meta->SetMeta("test", test_meta); - data_ctx->SetOutputMeta("stream_meta", data_meta); - return modelbox::STATUS_OK; - }; - auto process_func = - [=](const std::shared_ptr& data_ctx, - const std::shared_ptr& mock_flowunit) { - auto output_buf = data_ctx->Output("stream_meta"); - std::vector shape(1, 1); - output_buf->Build(shape); - - return modelbox::STATUS_OK; - }; - auto mock_functions = std::make_shared(); - mock_functions->RegisterOpenFunc(open_func); - mock_functions->RegisterDataPreFunc(data_pre_func); - mock_functions->RegisterProcessFunc(process_func); - flow_->AddFlowUnitDesc(mock_desc, mock_functions->GenerateCreateFunc(), - TEST_DRIVER_DIR); - } - { - auto mock_desc = GenerateFlowunitDesc("encoder_image_produce", - {"stream_meta"}, {"frame_info"}); - mock_desc->SetOutputType(EXPAND); - auto process_func = - [=](const std::shared_ptr& data_ctx, - const std::shared_ptr& mock_flowunit) { - std::string img_path; - static int64_t frame_index = 0; - if ((frame_index / 24) % 2 == 0) { - img_path = - std::string(TEST_ASSETS) + "/video/rgb_460800_480x320_a.data"; - } else { - img_path = - std::string(TEST_ASSETS) + "/video/rgb_460800_480x320_b.data"; - } - - std::ifstream img_file(img_path); - if (!img_file.is_open()) { - MBLOG_ERROR << "Open failed, path " << img_path; - return STATUS_FAULT; - } - - size_t file_size = 460800; - auto output_buff_list = data_ctx->Output("frame_info"); - std::vector shape(1, file_size); - output_buff_list->Build(shape); - auto output_buff = output_buff_list->At(0); - auto* ptr = (char*)output_buff->MutableData(); - img_file.read(ptr, file_size); - output_buff->Set("width", 480); - output_buff->Set("height", 320); - output_buff->Set("rate_num", 24); - output_buff->Set("rate_den", 1); - output_buff->Set("pix_fmt", std::string("rgb")); - output_buff->Set("index", frame_index); - - if (frame_index == 1339) { // 60S - return modelbox::STATUS_STOP; - } - - ++frame_index; - auto event = std::make_shared(); - data_ctx->SendEvent(event); - return modelbox::STATUS_CONTINUE; - }; - auto mock_functions = std::make_shared(); - mock_functions->RegisterProcessFunc(process_func); - flow_->AddFlowUnitDesc(mock_desc, mock_functions->GenerateCreateFunc(), - TEST_DRIVER_DIR); - } - - return STATUS_SUCCESS; -} - -TEST_F(RockchipVideoEncoderFlowUnitTest, InitUnit) { - const std::string test_lib_dir = TEST_DRIVER_DIR; - const std::string test_data_dir = TEST_DATA_DIR; - auto ret = system("nc localhost 554 -z"); - if (errno != 0 || ret != 0) { - GTEST_SKIP(); - } - - std::string dest_url = "rtsp://localhost/test_" + std::to_string(rand()); - std::string toml_content = R"( - [driver] - skip-default = true - dir=[")" + test_lib_dir + - "\",\"" + test_data_dir + "\"]\n " + - R"([graph] - graphconf = '''digraph demo { - encoder_start_unit[type=flowunit, flowunit=encoder_start_unit, device=cpu, deviceid=0, label=""] - encoder_image_produce[type=flowunit, flowunit=encoder_image_produce, device=cpu, deviceid=0, label=" | "] - videoencoder[type=flowunit, flowunit=video_encoder, device=cpu, deviceid=0, label="", queue_size_frame_info=16, default_dest_url=")" + - dest_url + R"("] - encoder_start_unit:stream_meta -> encoder_image_produce:stream_meta - encoder_image_produce:frame_info -> videoencoder:in_video_frame - }''' - format = "graphviz" - )"; - StartFlow(toml_content, 1000 * 1000); -} - -} // namespace modelbox \ No newline at end of file diff --git a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h index d111b276a..9fb6083e5 100644 --- a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h @@ -65,6 +65,8 @@ class RockChipLocalCameraFlowUnit : public modelbox::FlowUnit { std::shared_ptr &img_buf); MppFrame ProcessYVY2(const uint8_t *buf, size_t size, size_t w, size_t h, std::shared_ptr &img_buf); + MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt, + MppBuffer mpp_buf); MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt, MppBuffer mpp_buf); diff --git a/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc b/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc index 57e59fb3e..1791ee66f 100644 --- a/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc +++ b/src/drivers/devices/rockchip/flowunit/local_camera/v4l2_camera.cc @@ -111,7 +111,7 @@ int32_t V4L2Camera::GetCamfd(int32_t id, const std::string &bus_info) { } Defer { globfree(&glob_result); }; - + // detect it is a camera device if (modelbox::STATUS_SUCCESS != CamIoCtl(fd, VIDIOC_QUERYCAP, &cap)) { MBLOG_DEBUG << "Not v4l2 device:" << glob_result.gl_pathv[i]; diff --git a/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h b/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h index f09e95228..55bbd1ee4 100644 --- a/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h @@ -42,7 +42,7 @@ constexpr const char *FLOWUNIT_DESC = "\t@Constraint: The field value range of this flowunit supports: " "'pix_fmt': " "[rgb_packed,bgr_packed], 'layout': [hwc]. "; - + constexpr const char *IN_IMG = "in_image"; constexpr const char *OUT_IMG = "out_image"; From 46919f7038e7b637b68d17f0105f7af0fda9bfa3 Mon Sep 17 00:00:00 2001 From: fujialin <757524349@qq.com> Date: Wed, 7 Dec 2022 10:52:12 +0800 Subject: [PATCH 3/4] adaptation rockchip image --- CMake/FindROCKCHIP.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMake/FindROCKCHIP.cmake b/CMake/FindROCKCHIP.cmake index a5476b961..812e89933 100644 --- a/CMake/FindROCKCHIP.cmake +++ b/CMake/FindROCKCHIP.cmake @@ -3,9 +3,9 @@ if(DEFINED ENV{ROCKCHIP_PATH}) message(STATUS "DEFINED ${HINTS_ROCKCHIP_PATH}") else() - set(HINTS_ROCKCHIP_PATH "/opt/rockchip") + set(HINTS_ROCKCHIP_PATH "/usr/local/rockchip") - message(STATUS "set default search path: /opt/rockchip") + message(STATUS "set default search path: /usr/local/rockchip") endif() find_path(ROCKCHIP_RGA_INCLUDE NAMES im2d.h rga.h From 7b1e4a87763546e380b80eadc3c6e3c953d0afb0 Mon Sep 17 00:00:00 2001 From: fujialin <757524349@qq.com> Date: Mon, 19 Dec 2022 11:06:00 +0800 Subject: [PATCH 4/4] adaptation rockchip image --- CMake/FindROCKCHIP.cmake | 43 ++++++++++++------- docker/Dockerfile.rknnrt.build.ubuntu | 1 - .../devices/rockchip/core/CMakeLists.txt | 2 +- .../devices/rockchip/core/rockchip_api.cc | 10 +++++ .../rockchip/flowunit/crop/crop_flowunit.cc | 1 - .../flowunit/image_decoder/image_decoder.cc | 1 - .../inference_rknpu2/rknpu2_inference.cc | 7 ++- .../rknpu2_inference_flowunit.h | 2 +- .../local_camera/local_camera_flowunit.h | 3 -- .../flowunit/resize/resize_flowunit.cc | 1 - .../video_decoder/video_decoder_flowunit.cc | 1 - .../flowunit/video_out/video_out_flowunit.cc | 17 ++++++-- 12 files changed, 60 insertions(+), 29 deletions(-) diff --git a/CMake/FindROCKCHIP.cmake b/CMake/FindROCKCHIP.cmake index 812e89933..b21fab8e5 100644 --- a/CMake/FindROCKCHIP.cmake +++ b/CMake/FindROCKCHIP.cmake @@ -9,28 +9,41 @@ else() endif() find_path(ROCKCHIP_RGA_INCLUDE NAMES im2d.h rga.h - HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/include) + HINTS ${HINTS_ROCKCHIP_PATH}/rga/include) mark_as_advanced(ROCKCHIP_RGA_INCLUDE) find_path(ROCKCHIP_MPP_INCLUDE NAMES rk_mpi.h rk_type.h - HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include/rockchip) + HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include) mark_as_advanced(ROCKCHIP_MPP_INCLUDE) -find_path(RKNN_INCLUDE NAMES rknn_api.h - HINTS ${HINTS_ROCKCHIP_PATH}/rknn/include) -mark_as_advanced(RKNN_INCLUDE) +find_path(RKNPU_INCLUDE NAMES rknn_api.h + HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/include) +mark_as_advanced(RKNPU_INCLUDE) -find_path(RKNPU2_INCLUDE NAMES rknn_api.h - HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/include) -mark_as_advanced(RKNPU2_INCLUDE) +find_library(RKNPU_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/Linux/lib64/) +mark_as_advanced(RKNPU_LIBRARY) -find_library(RKNN_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknn/lib) -mark_as_advanced(RKNN_LIBRARY) +if(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588) + find_path(RKNPU2_INCLUDE NAMES rknn_api.h + HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/include) + mark_as_advanced(RKNPU2_INCLUDE) -find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/lib) -mark_as_advanced(RKNPU2_LIBRARY) + find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64) + mark_as_advanced(RKNPU2_LIBRARY) + message(STATUS "rk3588 platform") +elseif(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X) + find_path(RKNPU2_INCLUDE NAMES rknn_api.h + HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/include) + mark_as_advanced(RKNPU2_INCLUDE) -find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/lib) + find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64) + mark_as_advanced(RKNPU2_LIBRARY) + message(STATUS "rk356x platform") +else() + message(STATUS "other platform") +endif() + +find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rga/libs/Linux/gcc-aarch64) mark_as_advanced(RKRGA_LIBRARY) find_library(RKMPP_LIBRARY NAMES rockchip_mpp HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/lib) @@ -42,8 +55,8 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(ROCKCHIP VERSION_VAR ROCKCHIP_VERSION_STRING) if(ROCKCHIP_FOUND) - set(RKNN_LIBRARIES ${RKNN_LIBRARY}) - set(RKNN_INCLUDE_DIR ${RKNN_INCLUDE}) + set(RKNPU_LIBRARIES ${RKNPU_LIBRARY}) + set(RKNPU_INCLUDE_DIR ${RKNPU_INCLUDE}) set(RKNPU2_INCLUDE_DIR ${RKNPU2_INCLUDE}) set(RKNPU2_LIBRARIES ${RKNPU2_LIBRARY}) set(RKRGA_LIBRARIES ${RKRGA_LIBRARY}) diff --git a/docker/Dockerfile.rknnrt.build.ubuntu b/docker/Dockerfile.rknnrt.build.ubuntu index fb3313488..67936a58c 100644 --- a/docker/Dockerfile.rknnrt.build.ubuntu +++ b/docker/Dockerfile.rknnrt.build.ubuntu @@ -59,7 +59,6 @@ RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; \ systemctl enable ssh RUN echo "/usr/local/rockchip/rga/libs/Linux/gcc-aarch64" >> /etc/ld.so.conf.d/rockchip.conf && \ - echo "/usr/local/rockchip/rknpu/rknn/rknn_api/librknn_api/lib64" >> /etc/ld.so.conf.d/rockchip.conf && \ if [ -d /usr/local/rockchip/rknpu2/runtime/RK356X ]; then echo "/usr/local/rockchip/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64" >> /etc/ld.so.conf.d/rockchip.conf; elif [ -d /usr/local/rockchip/rknpu2/runtime/RK3588 ]; then echo "/usr/local/rockchip/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64" >> /etc/ld.so.conf.d/rockchip.conf; fi && \ echo "/usr/local/rockchip/rkmpp/lib" >> /etc/ld.so.conf.d/rockchip.conf diff --git a/src/drivers/devices/rockchip/core/CMakeLists.txt b/src/drivers/devices/rockchip/core/CMakeLists.txt index b24be7403..0769dbd07 100644 --- a/src/drivers/devices/rockchip/core/CMakeLists.txt +++ b/src/drivers/devices/rockchip/core/CMakeLists.txt @@ -29,7 +29,7 @@ include_directories(${LIBMODELBOX_INCLUDE}) include_directories(${LIBMODELBOX_BASE_INCLUDE}) include_directories(${LIBMODELBOX_DEVICE_ROCKCHIP_INCLUDE}) include_directories(${ROCKCHIP_MPP_INCLUDE}) -include_directories(${RKNN_INCLUDE_DIR}) +include_directories(${RKNPU_INCLUDE_DIR}) include_directories(${ROCKCHIP_RGA_INCLUDE}) set(HEADER diff --git a/src/drivers/devices/rockchip/core/rockchip_api.cc b/src/drivers/devices/rockchip/core/rockchip_api.cc index ae6b88ceb..4ef03ed65 100644 --- a/src/drivers/devices/rockchip/core/rockchip_api.cc +++ b/src/drivers/devices/rockchip/core/rockchip_api.cc @@ -119,6 +119,16 @@ Status CopyNVMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws, Status CopyRGBMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws, int hs) { + if (psrc == nullptr) { + MBLOG_ERROR << "psrc is nullptr"; + return {STATUS_FAULT, "psrc is nullptr"}; + } + + if (pdst == nullptr) { + MBLOG_ERROR << "pdst is nullptr"; + return {STATUS_FAULT, "pdst is nullptr"}; + } + uint8_t *rgbsrc = psrc; uint8_t *rgbdst = pdst; diff --git a/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc b/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc index 6f4bee99f..60e86ef66 100644 --- a/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc @@ -161,7 +161,6 @@ MODELBOX_FLOWUNIT(RockchipCropFlowUnit, rk_crop_desc) { rk_crop_desc.AddFlowUnitInput({IN_REGION, "cpu"}); rk_crop_desc.AddFlowUnitOutput({OUT_IMG, modelbox::DEVICE_TYPE}); rk_crop_desc.SetFlowType(modelbox::NORMAL); - rk_crop_desc.SetInputContiguous(false); rk_crop_desc.SetDescription(FLOWUNIT_DESC); } diff --git a/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc b/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc index d94e9d316..faca0bc5e 100644 --- a/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc +++ b/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc @@ -227,7 +227,6 @@ MODELBOX_FLOWUNIT(ImageDecoderFlowUnit, rk_imgdec_desc) { "the output pixel format")); rk_imgdec_desc.SetFlowType(modelbox::NORMAL); - rk_imgdec_desc.SetInputContiguous(false); rk_imgdec_desc.SetDescription(FLOWUNIT_DESC); } diff --git a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc index 76defd8e5..607e6696e 100644 --- a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc +++ b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc @@ -152,6 +152,9 @@ modelbox::Status modelbox::RKNPU2Inference::Init( return {modelbox::STATUS_FAULT, "query input_output error"}; } + MBLOG_INFO << "model input num: " << rknpu2_io_num.n_input + << ", output num: " << rknpu2_io_num.n_output << std::endl; + if (npu2model_input_list_.size() != rknpu2_io_num.n_input || npu2model_output_list_.size() != rknpu2_io_num.n_output) { MBLOG_ERROR << "model input output num mismatch: input num in graph is " @@ -285,9 +288,10 @@ size_t modelbox::RKNPU2Inference::GetInputBuffer( input_params->in_width_ = in_image->GetBytes(); } - if (input_buf_list->GetDevice()->GetType() == "rknpu") { + if (input_buf_list->GetDevice()->GetType() == "rockchip") { return CopyFromAlignMemory(input_buf_list, input_buf, input_params); } + input_buf.reset((uint8_t *)input_buf_list->ConstData(), [](uint8_t *p) {}); return input_buf_list->GetBytes(); } @@ -326,6 +330,7 @@ modelbox::Status modelbox::RKNPU2Inference::Infer( << " " << inputs_size_[i]; return modelbox::STATUS_FAULT; } + one_input.pass_through = false; one_input.type = (rknn_tensor_type)inputs_type_[i]; one_input.fmt = RKNN_TENSOR_NHWC; diff --git a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h index 772610294..a92351797 100644 --- a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h @@ -26,7 +26,7 @@ #include "rknpu2_inference.h" -constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rknpu2"; +constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rockchip"; constexpr const char *RKNPU2_INFERENCE_TYPE = "rknpu2"; class RKNPU2InferenceFlowUnit : public modelbox::FlowUnit { diff --git a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h index 9fb6083e5..d78d3763c 100644 --- a/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/local_camera/local_camera_flowunit.h @@ -68,9 +68,6 @@ class RockChipLocalCameraFlowUnit : public modelbox::FlowUnit { MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt, MppBuffer mpp_buf); - MppFrame SetMppFrameInfo(size_t w, size_t h, MppFrameFormat fmt, - MppBuffer mpp_buf); - uint32_t camWidth_{0}; uint32_t camHeight_{0}; uint32_t camera_id_{0}; diff --git a/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc b/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc index a2dcaa8ed..a49f489bd 100644 --- a/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc @@ -135,7 +135,6 @@ MODELBOX_FLOWUNIT(ResizeFlowUnit, rk_resize_desc) { rk_resize_desc.AddFlowUnitInput({IN_IMG}); rk_resize_desc.AddFlowUnitOutput({OUT_IMG}); rk_resize_desc.SetFlowType(modelbox::NORMAL); - rk_resize_desc.SetInputContiguous(false); rk_resize_desc.SetDescription(FLOWUNIT_DESC); rk_resize_desc.AddFlowUnitOption(modelbox::FlowUnitOption( "image_width", "int", true, "0", "the resize width")); diff --git a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc index 86e173ed1..4f76ed26a 100644 --- a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc @@ -174,7 +174,6 @@ MODELBOX_FLOWUNIT(VideoDecoderFlowUnit, desc) { desc.AddFlowUnitInput({VIDEO_PACKET_INPUT, "cpu"}); desc.AddFlowUnitOutput({FRAME_INFO_OUTPUT, modelbox::DEVICE_TYPE}); desc.SetFlowType(modelbox::STREAM); - desc.SetInputContiguous(false); desc.SetResourceNice(false); desc.SetDescription(FLOWUNIT_DESC); desc.AddFlowUnitOption(modelbox::FlowUnitOption( diff --git a/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc b/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc index 6e2ba78d9..8004e9ad6 100644 --- a/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc @@ -317,18 +317,25 @@ modelbox::Status VideoOutFlowUnit::GetDestUrl( const std::shared_ptr &data_ctx, std::string &dest_url) { dest_url = default_dest_url_; - Defer { std::regex url_auth_pattern("://[^ /]*?:[^ /]*?@"); auto result = std::regex_replace(dest_url, url_auth_pattern, "://*:*@"); MBLOG_INFO << "video_out url is " << result; }; + if (data_ctx == nullptr) { + MBLOG_ERROR << "data ctx is nullptr"; + return modelbox::STATUS_INVALID; + } + // 3种方式获取 auto stream_meta = data_ctx->GetInputMeta(FRAME_INFO_INPUT); if (stream_meta != nullptr) { + auto meta_dest_url = stream_meta->GetMeta(DEST_URL); auto dest_url_ptr = - std::static_pointer_cast(stream_meta->GetMeta(DEST_URL)); + meta_dest_url == nullptr + ? nullptr + : std::static_pointer_cast(meta_dest_url); if (dest_url_ptr != nullptr && !(*dest_url_ptr).empty()) { dest_url = *dest_url_ptr; return modelbox::STATUS_SUCCESS; @@ -336,6 +343,11 @@ modelbox::Status VideoOutFlowUnit::GetDestUrl( } auto config = data_ctx->GetSessionConfig(); + if (config == nullptr) { + MBLOG_ERROR << "data ctx session config is empty"; + return modelbox::STATUS_INVALID; + } + auto cfg_str = config->GetString("iva_task_output"); if (cfg_str.empty()) { return modelbox::STATUS_SUCCESS; @@ -368,7 +380,6 @@ MODELBOX_FLOWUNIT(VideoOutFlowUnit, desc) { desc.SetFlowUnitGroupType("Video"); desc.AddFlowUnitInput({FRAME_INFO_INPUT, "cpu"}); desc.SetFlowType(modelbox::STREAM); - desc.SetInputContiguous(false); // 禁止异步执行，编码必须一帧帧的编码 desc.SetResourceNice(false); desc.SetDescription(FLOWUNIT_DESC);