Fix handling boxed classes returned by reference

Author: twistedfall

binding-generator/src/lib.rs

@@ -1,7 +1,6 @@
 // todo support converting pointer + size to slice of Mat and other similar objects
 // todo add support for arrays in dnn::DictValue
 // todo allow ergonomically combining of enum variants with |
-// todo cv_utils_logging_internal_getGlobalLogTag() returns LogTag**, but Rust interprets it as LogTag*, check why it doesn't crash and fix if needed
 // todo almost everything from the manual module must be connected to the binding generator, not the main crate
 // todo check that FN_FaceDetector works at all (receiving InputArray, passing as callback)
 // fixme vector<Mat*> get's interpreted as Vector<Mat> which should be wrong (e.g. Layer::forward and Layer::apply_halide_scheduler)

binding-generator/src/renderer.rs

@@ -171,7 +171,7 @@ impl<'a> TypeRefRenderer<'a> for CppExternReturnRenderer {
 		let type_ref = if kind.as_string(type_ref.type_hint()).is_some() {
 			Cow::Owned(TypeRef::new_pointer(TypeRefDesc::void()))
 		} else if kind.extern_pass_kind().is_by_void_ptr() && !kind.as_abstract_class_ptr().is_some() {
-			Cow::Owned(TypeRef::new_pointer(type_ref.clone()))
+			Cow::Owned(TypeRef::new_pointer(type_ref.source()))
 		} else {
 			Cow::Borrowed(type_ref)
 		};

binding-generator/src/type_ref.rs

@@ -172,7 +172,7 @@ impl<'tu, 'ge> TypeRef<'tu, 'ge> {
 		}
 	}
 
-	/// Like canonical(), but also removes indirection by pointer and reference
+	/// Removes indirection by pointer and reference, this will also remove typedef if it references a pointer or reference
 	pub fn source(&self) -> Self {
 		match self.kind().as_ref() {
 			TypeRefKind::Pointer(inner) | TypeRefKind::Reference(inner) | TypeRefKind::RValueReference(inner) => inner.source(),
@@ -195,8 +195,17 @@ impl<'tu, 'ge> TypeRef<'tu, 'ge> {
 			TypeRefKind::Pointer(inner) | TypeRefKind::Reference(inner) | TypeRefKind::RValueReference(inner) => {
 				inner.source_smart()
 			}
-			TypeRefKind::Typedef(tdef) => tdef.underlying_type_ref().source_smart(),
 			TypeRefKind::SmartPtr(ptr) => ptr.pointee().source_smart(),
+			TypeRefKind::Typedef(tdef) => {
+				let underlying_type = tdef.underlying_type_ref();
+				match underlying_type.kind().as_ref() {
+					TypeRefKind::Pointer(inner) | TypeRefKind::Reference(inner) | TypeRefKind::RValueReference(inner) => {
+						inner.source_smart()
+					}
+					TypeRefKind::SmartPtr(ptr) => ptr.pointee().source_smart(),
+					_ => self.clone(),
+				}
+			}
 			_ => self.clone(),
 		}
 	}
@@ -205,10 +214,19 @@ impl<'tu, 'ge> TypeRef<'tu, 'ge> {
 	pub fn base(&self) -> Self {
 		match self.kind().as_ref() {
 			TypeRefKind::Pointer(inner) | TypeRefKind::Reference(inner) | TypeRefKind::RValueReference(inner) => inner.base(),
-			TypeRefKind::Typedef(tdef) => tdef.underlying_type_ref().base(),
+			TypeRefKind::SmartPtr(ptr) => ptr.pointee().base(),
 			TypeRefKind::Array(inner, ..) => inner.base(),
 			TypeRefKind::StdVector(vec) => vec.element_type().base(),
-			TypeRefKind::SmartPtr(ptr) => ptr.pointee().base(),
+			TypeRefKind::Typedef(tdef) => {
+				let underlying_type = tdef.underlying_type_ref();
+				match underlying_type.kind().as_ref() {
+					TypeRefKind::Pointer(inner) | TypeRefKind::Reference(inner) | TypeRefKind::RValueReference(inner) => inner.base(),
+					TypeRefKind::SmartPtr(ptr) => ptr.pointee().base(),
+					TypeRefKind::Array(inner, ..) => inner.base(),
+					TypeRefKind::StdVector(vec) => vec.element_type().base(),
+					_ => self.clone(),
+				}
+			}
 			_ => self.clone(),
 		}
 	}

binding-generator/src/writer/rust_native/func.rs

@@ -769,15 +769,20 @@ pub fn cpp_return_map<'f>(return_type: &TypeRef, name: &'f str, is_constructor:
 		};
 		(str_mk, false)
 	} else if return_kind.extern_pass_kind().is_by_void_ptr() && !is_constructor {
-		let out = return_type
-			.source()
-			.kind()
-			.as_class()
-			.filter(|cls| cls.is_abstract())
-			.map_or_else(
-				|| format!("new {typ}({name})", typ = return_type.cpp_name(CppNameStyle::Reference)).into(),
-				|_| name.into(),
-			);
+		let ret_source = return_type.source();
+		let out = ret_source.kind().as_class().filter(|cls| cls.is_abstract()).map_or_else(
+			|| {
+				// todo implement higher count if it's needed
+				let deref_count = return_type.kind().as_pointer().map_or(0, |_| 1);
+				format!(
+					"new {typ}({:*<deref_count$}{name})",
+					"",
+					typ = ret_source.cpp_name(CppNameStyle::Reference)
+				)
+				.into()
+			},
+			|_| name.into(),
+		);
 		(out, false)
 	} else {
 		(name.into(), return_kind.as_fixed_array().is_some())

tests/marshalling.rs

@@ -1,7 +1,7 @@
 //! Contains all tests that cover marshalling types to and from C++
 
 use opencv::core;
-use opencv::core::{Scalar, Tuple};
+use opencv::core::{Scalar, SparseMat, Tuple};
 use opencv::prelude::*;
 use opencv::Result;
 
@@ -125,6 +125,16 @@ fn fixed_array_return() -> Result<()> {
 	Ok(())
 }
 
+/// Sometimes functions return a boxed class by pointer, so they need an additional dereference
+#[test]
+fn boxed_return_by_pointer() -> Result<()> {
+	let mut sm = SparseMat::from_mat(&Mat::new_rows_cols_with_default(5, 3, i32::opencv_type(), 1.into())?)?;
+	let hdr = sm.hdr();
+	assert_eq!(1, hdr.refcount());
+	assert_eq!(2, hdr.dims());
+	Ok(())
+}
+
 /// Return of an owned String
 #[test]
 fn string_return() -> Result<()> {