diff --git a/CMake/FindROCKCHIP.cmake b/CMake/FindROCKCHIP.cmake index a5476b961..b21fab8e5 100644 --- a/CMake/FindROCKCHIP.cmake +++ b/CMake/FindROCKCHIP.cmake @@ -3,34 +3,47 @@ if(DEFINED ENV{ROCKCHIP_PATH}) message(STATUS "DEFINED ${HINTS_ROCKCHIP_PATH}") else() - set(HINTS_ROCKCHIP_PATH "/opt/rockchip") + set(HINTS_ROCKCHIP_PATH "/usr/local/rockchip") - message(STATUS "set default search path: /opt/rockchip") + message(STATUS "set default search path: /usr/local/rockchip") endif() find_path(ROCKCHIP_RGA_INCLUDE NAMES im2d.h rga.h - HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/include) + HINTS ${HINTS_ROCKCHIP_PATH}/rga/include) mark_as_advanced(ROCKCHIP_RGA_INCLUDE) find_path(ROCKCHIP_MPP_INCLUDE NAMES rk_mpi.h rk_type.h - HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include/rockchip) + HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include) mark_as_advanced(ROCKCHIP_MPP_INCLUDE) -find_path(RKNN_INCLUDE NAMES rknn_api.h - HINTS ${HINTS_ROCKCHIP_PATH}/rknn/include) -mark_as_advanced(RKNN_INCLUDE) +find_path(RKNPU_INCLUDE NAMES rknn_api.h + HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/include) +mark_as_advanced(RKNPU_INCLUDE) -find_path(RKNPU2_INCLUDE NAMES rknn_api.h - HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/include) -mark_as_advanced(RKNPU2_INCLUDE) +find_library(RKNPU_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/Linux/lib64/) +mark_as_advanced(RKNPU_LIBRARY) -find_library(RKNN_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknn/lib) -mark_as_advanced(RKNN_LIBRARY) +if(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588) + find_path(RKNPU2_INCLUDE NAMES rknn_api.h + HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/include) + mark_as_advanced(RKNPU2_INCLUDE) -find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/lib) -mark_as_advanced(RKNPU2_LIBRARY) + find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64) + mark_as_advanced(RKNPU2_LIBRARY) + message(STATUS "rk3588 platform") +elseif(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X) + find_path(RKNPU2_INCLUDE NAMES rknn_api.h + HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/include) + mark_as_advanced(RKNPU2_INCLUDE) -find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/lib) + find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64) + mark_as_advanced(RKNPU2_LIBRARY) + message(STATUS "rk356x platform") +else() + message(STATUS "other platform") +endif() + +find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rga/libs/Linux/gcc-aarch64) mark_as_advanced(RKRGA_LIBRARY) find_library(RKMPP_LIBRARY NAMES rockchip_mpp HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/lib) @@ -42,8 +55,8 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(ROCKCHIP VERSION_VAR ROCKCHIP_VERSION_STRING) if(ROCKCHIP_FOUND) - set(RKNN_LIBRARIES ${RKNN_LIBRARY}) - set(RKNN_INCLUDE_DIR ${RKNN_INCLUDE}) + set(RKNPU_LIBRARIES ${RKNPU_LIBRARY}) + set(RKNPU_INCLUDE_DIR ${RKNPU_INCLUDE}) set(RKNPU2_INCLUDE_DIR ${RKNPU2_INCLUDE}) set(RKNPU2_LIBRARIES ${RKNPU2_LIBRARY}) set(RKRGA_LIBRARIES ${RKRGA_LIBRARY}) diff --git a/docker/Dockerfile.rknnrt.build.ubuntu b/docker/Dockerfile.rknnrt.build.ubuntu index fb3313488..67936a58c 100644 --- a/docker/Dockerfile.rknnrt.build.ubuntu +++ b/docker/Dockerfile.rknnrt.build.ubuntu @@ -59,7 +59,6 @@ RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; \ systemctl enable ssh RUN echo "/usr/local/rockchip/rga/libs/Linux/gcc-aarch64" >> /etc/ld.so.conf.d/rockchip.conf && \ - echo "/usr/local/rockchip/rknpu/rknn/rknn_api/librknn_api/lib64" >> /etc/ld.so.conf.d/rockchip.conf && \ if [ -d /usr/local/rockchip/rknpu2/runtime/RK356X ]; then echo "/usr/local/rockchip/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64" >> /etc/ld.so.conf.d/rockchip.conf; elif [ -d /usr/local/rockchip/rknpu2/runtime/RK3588 ]; then echo "/usr/local/rockchip/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64" >> /etc/ld.so.conf.d/rockchip.conf; fi && \ echo "/usr/local/rockchip/rkmpp/lib" >> /etc/ld.so.conf.d/rockchip.conf diff --git a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc index 4b9f4ceed..8c5e6830c 100644 --- a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc +++ b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc @@ -25,6 +25,7 @@ modelbox::Status FfmpegVideoMuxer::Init( const std::shared_ptr &writer) { destination_url_ = writer->GetDestinationURL(); format_ctx_ = writer->GetCtx(); + if (format_ctx_ == nullptr) { const auto *msg = "FfmpegVideoMuxer.Init fail format ctx is nullptr"; MBLOG_ERROR << msg; diff --git a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc index 20a646260..4ceae4c9e 100644 --- a/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc +++ b/src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc @@ -73,4 +73,4 @@ modelbox::Status FfmpegWriter::Open(const std::string &format_name, std::string FfmpegWriter::GetFormatName() { return format_name_; } std::string FfmpegWriter::GetDestinationURL() { return destination_url_; } -std::shared_ptr FfmpegWriter::GetCtx() { return format_ctx_; } \ No newline at end of file +std::shared_ptr FfmpegWriter::GetCtx() { return format_ctx_; } diff --git a/src/drivers/devices/rockchip/core/CMakeLists.txt b/src/drivers/devices/rockchip/core/CMakeLists.txt index b24be7403..0769dbd07 100644 --- a/src/drivers/devices/rockchip/core/CMakeLists.txt +++ b/src/drivers/devices/rockchip/core/CMakeLists.txt @@ -29,7 +29,7 @@ include_directories(${LIBMODELBOX_INCLUDE}) include_directories(${LIBMODELBOX_BASE_INCLUDE}) include_directories(${LIBMODELBOX_DEVICE_ROCKCHIP_INCLUDE}) include_directories(${ROCKCHIP_MPP_INCLUDE}) -include_directories(${RKNN_INCLUDE_DIR}) +include_directories(${RKNPU_INCLUDE_DIR}) include_directories(${ROCKCHIP_RGA_INCLUDE}) set(HEADER diff --git a/src/drivers/devices/rockchip/core/rockchip_api.cc b/src/drivers/devices/rockchip/core/rockchip_api.cc index ae6b88ceb..4ef03ed65 100644 --- a/src/drivers/devices/rockchip/core/rockchip_api.cc +++ b/src/drivers/devices/rockchip/core/rockchip_api.cc @@ -119,6 +119,16 @@ Status CopyNVMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws, Status CopyRGBMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws, int hs) { + if (psrc == nullptr) { + MBLOG_ERROR << "psrc is nullptr"; + return {STATUS_FAULT, "psrc is nullptr"}; + } + + if (pdst == nullptr) { + MBLOG_ERROR << "pdst is nullptr"; + return {STATUS_FAULT, "pdst is nullptr"}; + } + uint8_t *rgbsrc = psrc; uint8_t *rgbdst = pdst; diff --git a/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc b/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc index 6f4bee99f..60e86ef66 100644 --- a/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc @@ -161,7 +161,6 @@ MODELBOX_FLOWUNIT(RockchipCropFlowUnit, rk_crop_desc) { rk_crop_desc.AddFlowUnitInput({IN_REGION, "cpu"}); rk_crop_desc.AddFlowUnitOutput({OUT_IMG, modelbox::DEVICE_TYPE}); rk_crop_desc.SetFlowType(modelbox::NORMAL); - rk_crop_desc.SetInputContiguous(false); rk_crop_desc.SetDescription(FLOWUNIT_DESC); } diff --git a/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc b/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc index d94e9d316..faca0bc5e 100644 --- a/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc +++ b/src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc @@ -227,7 +227,6 @@ MODELBOX_FLOWUNIT(ImageDecoderFlowUnit, rk_imgdec_desc) { "the output pixel format")); rk_imgdec_desc.SetFlowType(modelbox::NORMAL); - rk_imgdec_desc.SetInputContiguous(false); rk_imgdec_desc.SetDescription(FLOWUNIT_DESC); } diff --git a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc index 76defd8e5..607e6696e 100644 --- a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc +++ b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc @@ -152,6 +152,9 @@ modelbox::Status modelbox::RKNPU2Inference::Init( return {modelbox::STATUS_FAULT, "query input_output error"}; } + MBLOG_INFO << "model input num: " << rknpu2_io_num.n_input + << ", output num: " << rknpu2_io_num.n_output << std::endl; + if (npu2model_input_list_.size() != rknpu2_io_num.n_input || npu2model_output_list_.size() != rknpu2_io_num.n_output) { MBLOG_ERROR << "model input output num mismatch: input num in graph is " @@ -285,9 +288,10 @@ size_t modelbox::RKNPU2Inference::GetInputBuffer( input_params->in_width_ = in_image->GetBytes(); } - if (input_buf_list->GetDevice()->GetType() == "rknpu") { + if (input_buf_list->GetDevice()->GetType() == "rockchip") { return CopyFromAlignMemory(input_buf_list, input_buf, input_params); } + input_buf.reset((uint8_t *)input_buf_list->ConstData(), [](uint8_t *p) {}); return input_buf_list->GetBytes(); } @@ -326,6 +330,7 @@ modelbox::Status modelbox::RKNPU2Inference::Infer( << " " << inputs_size_[i]; return modelbox::STATUS_FAULT; } + one_input.pass_through = false; one_input.type = (rknn_tensor_type)inputs_type_[i]; one_input.fmt = RKNN_TENSOR_NHWC; diff --git a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h index 772610294..a92351797 100644 --- a/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h @@ -26,7 +26,7 @@ #include "rknpu2_inference.h" -constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rknpu2"; +constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rockchip"; constexpr const char *RKNPU2_INFERENCE_TYPE = "rknpu2"; class RKNPU2InferenceFlowUnit : public modelbox::FlowUnit { diff --git a/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc b/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc index a2dcaa8ed..a49f489bd 100644 --- a/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc @@ -135,7 +135,6 @@ MODELBOX_FLOWUNIT(ResizeFlowUnit, rk_resize_desc) { rk_resize_desc.AddFlowUnitInput({IN_IMG}); rk_resize_desc.AddFlowUnitOutput({OUT_IMG}); rk_resize_desc.SetFlowType(modelbox::NORMAL); - rk_resize_desc.SetInputContiguous(false); rk_resize_desc.SetDescription(FLOWUNIT_DESC); rk_resize_desc.AddFlowUnitOption(modelbox::FlowUnitOption( "image_width", "int", true, "0", "the resize width")); diff --git a/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h b/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h index 7a348cd8a..55bbd1ee4 100644 --- a/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h @@ -42,6 +42,7 @@ constexpr const char *FLOWUNIT_DESC = "\t@Constraint: The field value range of this flowunit supports: " "'pix_fmt': " "[rgb_packed,bgr_packed], 'layout': [hwc]. "; + constexpr const char *IN_IMG = "in_image"; constexpr const char *OUT_IMG = "out_image"; diff --git a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc index 86e173ed1..4f76ed26a 100644 --- a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc @@ -174,7 +174,6 @@ MODELBOX_FLOWUNIT(VideoDecoderFlowUnit, desc) { desc.AddFlowUnitInput({VIDEO_PACKET_INPUT, "cpu"}); desc.AddFlowUnitOutput({FRAME_INFO_OUTPUT, modelbox::DEVICE_TYPE}); desc.SetFlowType(modelbox::STREAM); - desc.SetInputContiguous(false); desc.SetResourceNice(false); desc.SetDescription(FLOWUNIT_DESC); desc.AddFlowUnitOption(modelbox::FlowUnitOption( diff --git a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h index b4104fc47..e59f5d8ee 100644 --- a/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h +++ b/src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h @@ -34,6 +34,7 @@ #include "rga.h" constexpr const char *FLOWUNIT_NAME = "video_decoder"; + constexpr const char *FLOWUNIT_DESC = "\n\t@Brief: A video decoder flowunit on rockchip. \n" "\t@Port parameter: The input port buffer type is video_packet, the output " @@ -64,6 +65,7 @@ constexpr const char *FLOWUNIT_DESC = "\t@Constraint: The flowuint 'video_decoder' must be used pair " "with 'video_demuxer. the output buffer meta fields 'pix_fmt' is " "'brg_packed' or 'rgb_packed', 'layout' is 'hcw'."; + constexpr const char *VIDEO_PACKET_INPUT = "in_video_packet"; constexpr const char *FRAME_INFO_OUTPUT = "out_video_frame"; constexpr const char *CODEC_META = "codec_meta"; diff --git a/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc b/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc index e5883a24f..482ede4f8 100644 --- a/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc +++ b/src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc @@ -360,6 +360,7 @@ std::shared_ptr FfmpegVideoEncoder::NewPacket(MppPacket &packet) { size_t len = mpp_packet_get_length(packet); auto *av_packet_ptr = av_packet_alloc(); + Defer { mpp_packet_deinit(&packet); }; if (av_packet_ptr == nullptr) { MBLOG_ERROR << "av packet alloc failed"; diff --git a/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc b/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc index 6e2ba78d9..8004e9ad6 100644 --- a/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc +++ b/src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc @@ -317,18 +317,25 @@ modelbox::Status VideoOutFlowUnit::GetDestUrl( const std::shared_ptr &data_ctx, std::string &dest_url) { dest_url = default_dest_url_; - Defer { std::regex url_auth_pattern("://[^ /]*?:[^ /]*?@"); auto result = std::regex_replace(dest_url, url_auth_pattern, "://*:*@"); MBLOG_INFO << "video_out url is " << result; }; + if (data_ctx == nullptr) { + MBLOG_ERROR << "data ctx is nullptr"; + return modelbox::STATUS_INVALID; + } + // 3种方式获取 auto stream_meta = data_ctx->GetInputMeta(FRAME_INFO_INPUT); if (stream_meta != nullptr) { + auto meta_dest_url = stream_meta->GetMeta(DEST_URL); auto dest_url_ptr = - std::static_pointer_cast(stream_meta->GetMeta(DEST_URL)); + meta_dest_url == nullptr + ? nullptr + : std::static_pointer_cast(meta_dest_url); if (dest_url_ptr != nullptr && !(*dest_url_ptr).empty()) { dest_url = *dest_url_ptr; return modelbox::STATUS_SUCCESS; @@ -336,6 +343,11 @@ modelbox::Status VideoOutFlowUnit::GetDestUrl( } auto config = data_ctx->GetSessionConfig(); + if (config == nullptr) { + MBLOG_ERROR << "data ctx session config is empty"; + return modelbox::STATUS_INVALID; + } + auto cfg_str = config->GetString("iva_task_output"); if (cfg_str.empty()) { return modelbox::STATUS_SUCCESS; @@ -368,7 +380,6 @@ MODELBOX_FLOWUNIT(VideoOutFlowUnit, desc) { desc.SetFlowUnitGroupType("Video"); desc.AddFlowUnitInput({FRAME_INFO_INPUT, "cpu"}); desc.SetFlowType(modelbox::STREAM); - desc.SetInputContiguous(false); // 禁止异步执行,编码必须一帧帧的编码 desc.SetResourceNice(false); desc.SetDescription(FLOWUNIT_DESC);