[WIP] Port hardsubx classifier (#1446)

* add tesseract-sys in dependencies of rust modules

* add appropriate feature flags and required packages to cargo toml

* expose classifier

* Redefine structs that are required for hardsubx

Note: rust-bindgen isn't being used directly for this because it will also redefine structures of leptonica, tesseract, and ffmpeg and we don't want that.
We want to use definitions of structs as in the rust interfact libraries we are importing

* write code to generate bindings for mprint

* - write a function to convert rust strings to c strings
- write a memory safe wrapper to mprint that uses above function

* - add helper function to deal with tess strings in a memory safe manner
- port get_ocr_text_simple
- port get_ocr_text_wordwise

* improve conversion of C string to Rust string by using built-in functions

* replace mprint usage with warn!

* port get_ocr_text_letterwise

* remove redundant mprint function

* improve readability _tess_string_helper by using more general variable names inside

* make get_ocr_text_simple call get_ocr_text_simple_threshold to remove redundant codefix bugs

* remove manual definition of cc_subtitle and use bindgen bindings

* style changes to rust hardsubx classifier

* add get_ocr_text_letterwise_threshold and make get_ocr_text_letterwise call it appropriately

* move hardsubx context struct to mod.rs

* add get_ocr_text_wordwise_threshold and make get_ocr_text_wordwise call it

* use the ffmpeg-sys definition of Pix

* hide ported functions under macros

* use the AVPacket from bindings and not ffmpeg to make compatibility work for now.
TODO: rewrite init_hardsubx and also deal with the ffmpeg stuff when that is done

* improce _tess_string_helper by using appropriate built-in functions

* linter recommended changes

* clang style change

* fix loop bug that didn't allow for re-evaluation of it on usage of continue statement

* start porting of decoder with the _process_frame_color_basic function and related code

* hide the C version of _process_frame_color_basic behind an #ifdef

* add _process_frame_tickertext

* hide the C version of _process_frame_tickertext behind ifdef and add #[no_mangle] to the rust version

* check if word is empty as soon as word is detected

* port _process_frame_white_basic

* hide the C version _process_frame_white_basic behind compiler macros

* stylistic changes

* safety docs for hardsubx classifier

* safety docs for decoder as of now

* safe docs for utils.rs

* style changes

* format and style changes

* modify safety docs

* formatting fix

Author: shashwat1002

src/lib_ccx/hardsubx_classifier.c

@@ -1,8 +1,8 @@
 #include "lib_ccx.h"
 #include "utility.h"
 
-#ifdef ENABLE_HARDSUBX
-//TODO: Correct FFMpeg integration
+#if defined(ENABLE_HARDSUBX) && defined(DISABLE_RUST)
+// TODO: Correct FFMpeg integration
 #include <libavcodec/avcodec.h>
 #include <libavformat/avformat.h>
 #include <libavutil/imgutils.h>

src/lib_ccx/hardsubx_decoder.c

@@ -11,6 +11,7 @@
 #include <tesseract/capi.h>
 #include "hardsubx.h"
 
+#ifdef DISABLE_RUST
 char *_process_frame_white_basic(struct lib_hardsubx_ctx *ctx, AVFrame *frame, int width, int height, int index)
 {
 	// printf("frame : %04d\n", index);
@@ -77,21 +78,21 @@ char *_process_frame_white_basic(struct lib_hardsubx_ctx *ctx, AVFrame *frame, i
 	{
 		case HARDSUBX_OCRMODE_WORD:
 			if (ctx->conf_thresh > 0)
-				subtitle_text = get_ocr_text_wordwise_threshold(ctx, lum_im, ctx->conf_thresh);
+				subtitle_text = get_ocr_text_wordwise_threshold(ctx, feat_im, ctx->conf_thresh);
 			else
-				subtitle_text = get_ocr_text_wordwise(ctx, lum_im);
+				subtitle_text = get_ocr_text_wordwise(ctx, feat_im);
 			break;
 		case HARDSUBX_OCRMODE_LETTER:
 			if (ctx->conf_thresh > 0)
-				subtitle_text = get_ocr_text_letterwise_threshold(ctx, lum_im, ctx->conf_thresh);
+				subtitle_text = get_ocr_text_letterwise_threshold(ctx, feat_im, ctx->conf_thresh);
 			else
-				subtitle_text = get_ocr_text_letterwise(ctx, lum_im);
+				subtitle_text = get_ocr_text_letterwise(ctx, feat_im);
 			break;
 		case HARDSUBX_OCRMODE_FRAME:
 			if (ctx->conf_thresh > 0)
-				subtitle_text = get_ocr_text_simple_threshold(ctx, lum_im, ctx->conf_thresh);
+				subtitle_text = get_ocr_text_simple_threshold(ctx, feat_im, ctx->conf_thresh);
 			else
-				subtitle_text = get_ocr_text_simple(ctx, lum_im);
+				subtitle_text = get_ocr_text_simple(ctx, feat_im);
 			break;
 		default:
 			fatal(EXIT_MALFORMED_PARAMETER, "Invalid OCR Mode");
@@ -214,6 +215,7 @@ char *_process_frame_color_basic(struct lib_hardsubx_ctx *ctx, AVFrame *frame, i
 
 	return subtitle_text;
 }
+#endif
 
 void _display_frame(struct lib_hardsubx_ctx *ctx, AVFrame *frame, int width, int height, int timestamp)
 {
@@ -294,6 +296,7 @@ void _display_frame(struct lib_hardsubx_ctx *ctx, AVFrame *frame, int width, int
 	pixDestroy(&feat_im);
 }
 
+#ifdef DISABLE_RUST
 char *_process_frame_tickertext(struct lib_hardsubx_ctx *ctx, AVFrame *frame, int width, int height, int index)
 {
 	PIX *im;
@@ -367,6 +370,7 @@ char *_process_frame_tickertext(struct lib_hardsubx_ctx *ctx, AVFrame *frame, in
 
 	return subtitle_text;
 }
+#endif
 
 int hardsubx_process_frames_tickertext(struct lib_hardsubx_ctx *ctx, struct encoder_ctx *enc_ctx)
 {

src/rust/Cargo.lock

@@ -15,10 +15,13 @@ log = "0.4.0"
 env_logger = "0.8.4"
 iconv = "0.1.1"
 palette = "0.6.0"
-ffmpeg-sys-next = { version = "5.0.1", optional = true, default-features = false, features = ["avcodec", "build"]}
+ffmpeg-sys-next = { version = "5.0.1", optional = true, default-features = false, features = ["avcodec", "avformat", "swscale", "build"]}
+tesseract-sys = { version = "0.5.12", optional = true, default-features = false}
+leptonica-sys = { version = "0.4.1", optional = true, default-features = false}
 
 [build-dependencies]
 bindgen = "0.58.1"
 
 [features]
-hardsubx_ocr = ["ffmpeg-sys-next"]
+hardsubx_ocr = ["ffmpeg-sys-next", "tesseract-sys", "leptonica-sys"]
+

src/rust/Cargo.toml

@@ -11,12 +11,23 @@ fn main() {
     ];
 
     #[cfg(feature = "hardsubx_ocr")]
-    allowlist_functions.extend_from_slice(&["edit_distance", "convert_pts_to_.*", "av_rescale_q"]);
+    allowlist_functions.extend_from_slice(&[
+        "edit_distance",
+        "convert_pts_to_.*",
+        "av_rescale_q",
+        "mprint",
+    ]);
 
-    let mut allowlist_types = vec![".*(?i)_?dtvcc_.*", "encoder_ctx", "lib_cc_decode"];
+    let mut allowlist_types = vec![
+        ".*(?i)_?dtvcc_.*",
+        "encoder_ctx",
+        "lib_cc_decode",
+        "cc_subtitle",
+        "ccx_output_format",
+    ];
 
     #[cfg(feature = "hardsubx_ocr")]
-    allowlist_types.extend_from_slice(&["AVRational"]);
+    allowlist_types.extend_from_slice(&["AVRational", "AVPacket", "AVFrame"]);
 
     let mut builder = bindgen::Builder::default()
         // The input header we would like to generate