Rockchip flowunit

CMake/FindROCKCHIP.cmake

@@ -3,34 +3,47 @@ if(DEFINED ENV{ROCKCHIP_PATH})
 
     message(STATUS "DEFINED ${HINTS_ROCKCHIP_PATH}")
 else()
-    set(HINTS_ROCKCHIP_PATH "/opt/rockchip")
+    set(HINTS_ROCKCHIP_PATH "/usr/local/rockchip")
 
-    message(STATUS "set default search path: /opt/rockchip")
+    message(STATUS "set default search path: /usr/local/rockchip")
 endif()
 
 find_path(ROCKCHIP_RGA_INCLUDE NAMES im2d.h rga.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/include)
+    HINTS ${HINTS_ROCKCHIP_PATH}/rga/include)
 mark_as_advanced(ROCKCHIP_RGA_INCLUDE)
 
 find_path(ROCKCHIP_MPP_INCLUDE NAMES rk_mpi.h rk_type.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include/rockchip)
+    HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/include)
 mark_as_advanced(ROCKCHIP_MPP_INCLUDE)
 
-find_path(RKNN_INCLUDE NAMES rknn_api.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rknn/include)
-mark_as_advanced(RKNN_INCLUDE)
+find_path(RKNPU_INCLUDE NAMES rknn_api.h
+    HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/include)
+mark_as_advanced(RKNPU_INCLUDE)
 
-find_path(RKNPU2_INCLUDE NAMES rknn_api.h
-    HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/include)
-mark_as_advanced(RKNPU2_INCLUDE)
+find_library(RKNPU_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknpu/rknn-api/librknn_api/Linux/lib64/)
+mark_as_advanced(RKNPU_LIBRARY)
 
-find_library(RKNN_LIBRARY NAMES rknn_api HINTS ${HINTS_ROCKCHIP_PATH}/rknn/lib)
-mark_as_advanced(RKNN_LIBRARY)
+if(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588)
+    find_path(RKNPU2_INCLUDE NAMES rknn_api.h
+        HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/include)
+    mark_as_advanced(RKNPU2_INCLUDE)
 
-find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknnrt/lib)
-mark_as_advanced(RKNPU2_LIBRARY)
+    find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64)
+    mark_as_advanced(RKNPU2_LIBRARY)
+    message(STATUS "rk3588 platform")
+elseif(EXISTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X)
+    find_path(RKNPU2_INCLUDE NAMES rknn_api.h
+        HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/include)
+    mark_as_advanced(RKNPU2_INCLUDE)
 
-find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rk-rga/lib)
+    find_library(RKNPU2_LIBRARY NAMES rknnrt HINTS ${HINTS_ROCKCHIP_PATH}/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64)
+    mark_as_advanced(RKNPU2_LIBRARY)
+    message(STATUS "rk356x platform")
+else()
+    message(STATUS "other platform")
+endif()
+
+find_library(RKRGA_LIBRARY NAMES rga HINTS ${HINTS_ROCKCHIP_PATH}/rga/libs/Linux/gcc-aarch64)
 mark_as_advanced(RKRGA_LIBRARY)
 
 find_library(RKMPP_LIBRARY NAMES rockchip_mpp HINTS ${HINTS_ROCKCHIP_PATH}/rkmpp/lib)
@@ -42,8 +55,8 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(ROCKCHIP
     VERSION_VAR ROCKCHIP_VERSION_STRING)
 
 if(ROCKCHIP_FOUND)
-    set(RKNN_LIBRARIES ${RKNN_LIBRARY})
-    set(RKNN_INCLUDE_DIR ${RKNN_INCLUDE})
+    set(RKNPU_LIBRARIES ${RKNPU_LIBRARY})
+    set(RKNPU_INCLUDE_DIR ${RKNPU_INCLUDE})
     set(RKNPU2_INCLUDE_DIR ${RKNPU2_INCLUDE})
     set(RKNPU2_LIBRARIES ${RKNPU2_LIBRARY})
     set(RKRGA_LIBRARIES ${RKRGA_LIBRARY})

docker/Dockerfile.rknnrt.build.ubuntu

@@ -59,7 +59,6 @@ RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; \
     systemctl enable ssh
 
 RUN echo "/usr/local/rockchip/rga/libs/Linux/gcc-aarch64" >> /etc/ld.so.conf.d/rockchip.conf && \
-    echo "/usr/local/rockchip/rknpu/rknn/rknn_api/librknn_api/lib64" >> /etc/ld.so.conf.d/rockchip.conf && \
     if [ -d /usr/local/rockchip/rknpu2/runtime/RK356X ]; then echo "/usr/local/rockchip/rknpu2/runtime/RK356X/Linux/librknn_api/aarch64" >> /etc/ld.so.conf.d/rockchip.conf; elif [ -d /usr/local/rockchip/rknpu2/runtime/RK3588 ]; then echo "/usr/local/rockchip/rknpu2/runtime/RK3588/Linux/librknn_api/aarch64" >> /etc/ld.so.conf.d/rockchip.conf; fi && \
     echo "/usr/local/rockchip/rkmpp/lib" >> /etc/ld.so.conf.d/rockchip.conf
 

src/drivers/devices/rockchip/common_src/video_out/ffmpeg_video_muxer.cc

@@ -25,6 +25,7 @@ modelbox::Status FfmpegVideoMuxer::Init(
     const std::shared_ptr<FfmpegWriter> &writer) {
   destination_url_ = writer->GetDestinationURL();
   format_ctx_ = writer->GetCtx();
+
   if (format_ctx_ == nullptr) {
     const auto *msg = "FfmpegVideoMuxer.Init fail format ctx is nullptr";
     MBLOG_ERROR << msg;

src/drivers/devices/rockchip/common_src/video_out/ffmpeg_writer.cc

@@ -73,4 +73,4 @@ modelbox::Status FfmpegWriter::Open(const std::string &format_name,
 
 std::string FfmpegWriter::GetFormatName() { return format_name_; }
 std::string FfmpegWriter::GetDestinationURL() { return destination_url_; }
-std::shared_ptr<AVFormatContext> FfmpegWriter::GetCtx() { return format_ctx_; }
+std::shared_ptr<AVFormatContext> FfmpegWriter::GetCtx() { return format_ctx_; }

src/drivers/devices/rockchip/core/CMakeLists.txt

@@ -29,7 +29,7 @@ include_directories(${LIBMODELBOX_INCLUDE})
 include_directories(${LIBMODELBOX_BASE_INCLUDE})
 include_directories(${LIBMODELBOX_DEVICE_ROCKCHIP_INCLUDE})
 include_directories(${ROCKCHIP_MPP_INCLUDE})
-include_directories(${RKNN_INCLUDE_DIR})
+include_directories(${RKNPU_INCLUDE_DIR})
 include_directories(${ROCKCHIP_RGA_INCLUDE})
 
 set(HEADER

src/drivers/devices/rockchip/core/rockchip_api.cc

@@ -119,6 +119,16 @@ Status CopyNVMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws,
 
 Status CopyRGBMemory(uint8_t *psrc, uint8_t *pdst, int w, int h, int ws,
                      int hs) {
+  if (psrc == nullptr) {
+    MBLOG_ERROR << "psrc is nullptr";
+    return {STATUS_FAULT, "psrc is nullptr"};
+  }
+
+  if (pdst == nullptr) {
+    MBLOG_ERROR << "pdst is nullptr";
+    return {STATUS_FAULT, "pdst is nullptr"};
+  }
+
   uint8_t *rgbsrc = psrc;
   uint8_t *rgbdst = pdst;
 

src/drivers/devices/rockchip/flowunit/crop/crop_flowunit.cc

@@ -161,7 +161,6 @@ MODELBOX_FLOWUNIT(RockchipCropFlowUnit, rk_crop_desc) {
   rk_crop_desc.AddFlowUnitInput({IN_REGION, "cpu"});
   rk_crop_desc.AddFlowUnitOutput({OUT_IMG, modelbox::DEVICE_TYPE});
   rk_crop_desc.SetFlowType(modelbox::NORMAL);
-  rk_crop_desc.SetInputContiguous(false);
   rk_crop_desc.SetDescription(FLOWUNIT_DESC);
 }
 

src/drivers/devices/rockchip/flowunit/image_decoder/image_decoder.cc

@@ -227,7 +227,6 @@ MODELBOX_FLOWUNIT(ImageDecoderFlowUnit, rk_imgdec_desc) {
       "the output pixel format"));
 
   rk_imgdec_desc.SetFlowType(modelbox::NORMAL);
-  rk_imgdec_desc.SetInputContiguous(false);
   rk_imgdec_desc.SetDescription(FLOWUNIT_DESC);
 }
 

src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference.cc

@@ -152,6 +152,9 @@ modelbox::Status modelbox::RKNPU2Inference::Init(
     return {modelbox::STATUS_FAULT, "query input_output error"};
   }
 
+  MBLOG_INFO << "model input num: " << rknpu2_io_num.n_input
+             << ", output num: " << rknpu2_io_num.n_output << std::endl;
+
   if (npu2model_input_list_.size() != rknpu2_io_num.n_input ||
       npu2model_output_list_.size() != rknpu2_io_num.n_output) {
     MBLOG_ERROR << "model input output num mismatch: input num in graph is "
@@ -285,9 +288,10 @@ size_t modelbox::RKNPU2Inference::GetInputBuffer(
     input_params->in_width_ = in_image->GetBytes();
   }
 
-  if (input_buf_list->GetDevice()->GetType() == "rknpu") {
+  if (input_buf_list->GetDevice()->GetType() == "rockchip") {
     return CopyFromAlignMemory(input_buf_list, input_buf, input_params);
   }
+
   input_buf.reset((uint8_t *)input_buf_list->ConstData(), [](uint8_t *p) {});
   return input_buf_list->GetBytes();
 }
@@ -326,6 +330,7 @@ modelbox::Status modelbox::RKNPU2Inference::Infer(
                   << " " << inputs_size_[i];
       return modelbox::STATUS_FAULT;
     }
+
     one_input.pass_through = false;
     one_input.type = (rknn_tensor_type)inputs_type_[i];
     one_input.fmt = RKNN_TENSOR_NHWC;

src/drivers/devices/rockchip/flowunit/inference_rknpu2/rknpu2_inference_flowunit.h

@@ -26,7 +26,7 @@
 
 #include "rknpu2_inference.h"
 
-constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rknpu2";
+constexpr const char *RKNPU2_FLOWUNIT_TYPE = "rockchip";
 constexpr const char *RKNPU2_INFERENCE_TYPE = "rknpu2";
 
 class RKNPU2InferenceFlowUnit : public modelbox::FlowUnit {

src/drivers/devices/rockchip/flowunit/resize/resize_flowunit.cc

@@ -135,7 +135,6 @@ MODELBOX_FLOWUNIT(ResizeFlowUnit, rk_resize_desc) {
   rk_resize_desc.AddFlowUnitInput({IN_IMG});
   rk_resize_desc.AddFlowUnitOutput({OUT_IMG});
   rk_resize_desc.SetFlowType(modelbox::NORMAL);
-  rk_resize_desc.SetInputContiguous(false);
   rk_resize_desc.SetDescription(FLOWUNIT_DESC);
   rk_resize_desc.AddFlowUnitOption(modelbox::FlowUnitOption(
       "image_width", "int", true, "0", "the resize width"));

src/drivers/devices/rockchip/flowunit/to_cpuimg/mpp_to_cpu_flowunit.h

@@ -42,6 +42,7 @@ constexpr const char *FLOWUNIT_DESC =
     "\t@Constraint: The field value range of this flowunit supports: "
     "'pix_fmt': "
     "[rgb_packed,bgr_packed], 'layout': [hwc]. ";
+
 constexpr const char *IN_IMG = "in_image";
 constexpr const char *OUT_IMG = "out_image";
 

src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.cc

@@ -174,7 +174,6 @@ MODELBOX_FLOWUNIT(VideoDecoderFlowUnit, desc) {
   desc.AddFlowUnitInput({VIDEO_PACKET_INPUT, "cpu"});
   desc.AddFlowUnitOutput({FRAME_INFO_OUTPUT, modelbox::DEVICE_TYPE});
   desc.SetFlowType(modelbox::STREAM);
-  desc.SetInputContiguous(false);
   desc.SetResourceNice(false);
   desc.SetDescription(FLOWUNIT_DESC);
   desc.AddFlowUnitOption(modelbox::FlowUnitOption(

src/drivers/devices/rockchip/flowunit/video_decoder/video_decoder_flowunit.h

@@ -34,6 +34,7 @@
 #include "rga.h"
 
 constexpr const char *FLOWUNIT_NAME = "video_decoder";
+
 constexpr const char *FLOWUNIT_DESC =
     "\n\t@Brief: A video decoder flowunit on rockchip. \n"
     "\t@Port parameter: The input port buffer type is video_packet, the output "
@@ -64,6 +65,7 @@ constexpr const char *FLOWUNIT_DESC =
     "\t@Constraint: The flowuint 'video_decoder' must be used pair "
     "with 'video_demuxer. the output buffer meta fields 'pix_fmt' is "
     "'brg_packed' or 'rgb_packed', 'layout' is 'hcw'.";
+
 constexpr const char *VIDEO_PACKET_INPUT = "in_video_packet";
 constexpr const char *FRAME_INFO_OUTPUT = "out_video_frame";
 constexpr const char *CODEC_META = "codec_meta";

src/drivers/devices/rockchip/flowunit/video_out/ffmpeg_video_encoder.cc

@@ -360,6 +360,7 @@ std::shared_ptr<AVPacket> FfmpegVideoEncoder::NewPacket(MppPacket &packet) {
   size_t len = mpp_packet_get_length(packet);
 
   auto *av_packet_ptr = av_packet_alloc();
+
   Defer { mpp_packet_deinit(&packet); };
   if (av_packet_ptr == nullptr) {
     MBLOG_ERROR << "av packet alloc failed";

src/drivers/devices/rockchip/flowunit/video_out/video_out_flowunit.cc

@@ -317,25 +317,37 @@ modelbox::Status VideoOutFlowUnit::GetDestUrl(
     const std::shared_ptr<modelbox::DataContext> &data_ctx,
     std::string &dest_url) {
   dest_url = default_dest_url_;
-
   Defer {
     std::regex url_auth_pattern("://[^ /]*?:[^ /]*?@");
     auto result = std::regex_replace(dest_url, url_auth_pattern, "://*:*@");
     MBLOG_INFO << "video_out url is " << result;
   };
 
+  if (data_ctx == nullptr) {
+    MBLOG_ERROR << "data ctx is nullptr";
+    return modelbox::STATUS_INVALID;
+  }
+
   // 3种方式获取
   auto stream_meta = data_ctx->GetInputMeta(FRAME_INFO_INPUT);
   if (stream_meta != nullptr) {
+    auto meta_dest_url = stream_meta->GetMeta(DEST_URL);
     auto dest_url_ptr =
-        std::static_pointer_cast<std::string>(stream_meta->GetMeta(DEST_URL));
+        meta_dest_url == nullptr
+            ? nullptr
+            : std::static_pointer_cast<std::string>(meta_dest_url);
     if (dest_url_ptr != nullptr && !(*dest_url_ptr).empty()) {
       dest_url = *dest_url_ptr;
       return modelbox::STATUS_SUCCESS;
     }
   }
 
   auto config = data_ctx->GetSessionConfig();
+  if (config == nullptr) {
+    MBLOG_ERROR << "data ctx session config is empty";
+    return modelbox::STATUS_INVALID;
+  }
+
   auto cfg_str = config->GetString("iva_task_output");
   if (cfg_str.empty()) {
     return modelbox::STATUS_SUCCESS;
@@ -368,7 +380,6 @@ MODELBOX_FLOWUNIT(VideoOutFlowUnit, desc) {
   desc.SetFlowUnitGroupType("Video");
   desc.AddFlowUnitInput({FRAME_INFO_INPUT, "cpu"});
   desc.SetFlowType(modelbox::STREAM);
-  desc.SetInputContiguous(false);
   // 禁止异步执行，编码必须一帧帧的编码
   desc.SetResourceNice(false);
   desc.SetDescription(FLOWUNIT_DESC);