FFMPEG bindings generation throws MatchError on Reference(Unnamed)

[kbeldjilali]

Hi,

I'm totally new to Scala Native (just finished reading Modern Systems Programming with Scala Native) and the sn-bindgen tool. I wanted to try something probably too ambitious for a first attempt: generating bindings for FFmpeg.

My initial goal is to generate bindings that allow me to compile and run a simple FFmpeg "hello world" example from this hello world example

built.sbt

lazy val ffmpegModule =
  project
    .in(file("modules/ffmpeg"))
    .enablePlugins(ScalaNativePlugin, BindgenPlugin, VcpkgNativePlugin)
    .settings(
      scalaVersion := "3.3.5",
      vcpkgDependencies := VcpkgDependencies("ffmpeg"),
      vcpkgNativeConfig ~= { _.withApproximate(false) },
      bindgenBindings += {
        Binding(
            header = baseDirectory.value / "src" / "main" / "resources" / "scala-native" / "ffmpeg-amalgam.h",
            packageName = "ffmpeg_wrapper"
        )
        .withCImports(List(
            "libavformat/avformat.h",
            "libavcodec/avcodec.h",
            "libavutil/avutil.h",
            "libavutil/log.h"
        ))
        .withLinkName("ffmpeg_wrapper")
        .withClangFlags(List("-I" + vcpkgConfigurator.value.includes("ffmpeg")))
        .withLogLevel(bindgen.interface.LogLevel.Trace)
      }
    )

To simplify things, in ffmpeg-amalgam.h I only exposed what I need (maybe useless? )

#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>

AVFormatContext* wrap_avformat_alloc_context() {
    return avformat_alloc_context();
}

int wrap_avformat_open_input(AVFormatContext** ps, const char* url) {
    return avformat_open_input(ps, url, NULL, NULL);
}

int wrap_avformat_find_stream_info(AVFormatContext* ps) {
    return avformat_find_stream_info(ps, NULL);
}

AVCodec* wrap_avcodec_find_decoder(enum AVCodecID id) {
    return avcodec_find_decoder(id);
}

AVCodecContext* wrap_avcodec_alloc_context3(const AVCodec* codec) {
    return avcodec_alloc_context3(codec);
}

int wrap_avcodec_parameters_to_context(AVCodecContext* ctx, const AVCodecParameters* params) {
    return avcodec_parameters_to_context(ctx, params);
}

int wrap_avcodec_open2(AVCodecContext* ctx, const AVCodec* codec) {
    return avcodec_open2(ctx, codec, NULL);
}

AVFrame* wrap_av_frame_alloc() {
    return av_frame_alloc();
}

AVPacket* wrap_av_packet_alloc() {
    return av_packet_alloc();
}

int wrap_av_read_frame(AVFormatContext* ps, AVPacket* pkt) {
    return av_read_frame(ps, pkt);
}

void wrap_av_packet_unref(AVPacket* pkt) {
    av_packet_unref(pkt);
}

void wrap_avformat_close_input(AVFormatContext** ps) {
    avformat_close_input(ps);
}

void wrap_av_packet_free(AVPacket** pkt) {
    av_packet_free(pkt);
}

void wrap_av_frame_free(AVFrame** frame) {
    av_frame_free(frame);
}

void wrap_avcodec_free_context(AVCodecContext** ctx) {
    avcodec_free_context(ctx);
}

Error output:

[ .... ] 
[bindgen] trace Rendering DefName(AVPacketSideData,Struct)
[bindgen] trace Rendering DefName(AVPanScan,Struct)
[bindgen] trace Rendering DefName(AVProbeData,Struct)
[bindgen] trace Rendering DefName(AVProducerReferenceTime,Struct)
[bindgen] trace Rendering DefName(AVProfile,Struct)
[bindgen] trace Rendering DefName(AVProgram,Struct)
[bindgen] trace Rendering DefName(AVRational,Struct)
[bindgen] trace Rendering DefName(AVRegionOfInterest,Struct)
[bindgen] trace Rendering DefName(AVSideDataDescriptor,Struct)
[bindgen] trace Rendering DefName(AVStream,Struct)
[bindgen] trace Struct 'AVStream' was detected as having cycles
[bindgen] trace Rewrite rules for struct AVStream
    av_class  Pointer(Void)
[bindgen] trace Rendering DefName(AVStreamGroup,Struct)
[bindgen] trace Struct 'AVStreamGroup' was detected as having cycles
[bindgen] trace Rewrite rules for struct AVStreamGroup
    av_class  Pointer(Void)
     streams  Pointer(Void)
[bindgen] trace Rendering DefName(AVStreamGroupLCEVC,Struct)
[bindgen] trace Struct 'AVStreamGroupLCEVC' was detected as having cycles
[bindgen] trace Rewrite rules for struct AVStreamGroupLCEVC
    av_class  Pointer(Void)
[bindgen] trace Rendering DefName(AVStreamGroupTileGrid,Struct)
[bindgen] trace Struct 'AVStreamGroupTileGrid' was detected as having cycles
[bindgen] trace Rewrite rules for struct AVStreamGroupTileGrid
    av_class  Pointer(Void)
Exception in thread "main"scala.MatchError: Reference(Unnamed) (of class bindgen.CType$Reference)
        at java.lang.Throwable.fillInStackTrace(Unknown Source)
        at bindgen.rendering.scalaType$package$.scalaType(Unknown Source)
        at bindgen.rendering.scalaType$package$.scalaType(Unknown Source)
        at scala.collection.immutable.List.map(Unknown Source)
        at bindgen.rendering.scalaType$package$.scalaType(Unknown Source)
        at bindgen.rendering.struct$package$.struct(Unknown Source)
        at bindgen.rendering.binding$package$.renderAll$$anonfun$1(Unknown Source)
        at scala.collection.immutable.List.foreach(Unknown Source)
        at bindgen.rendering.binding$package$.renderAll(Unknown Source)
        at bindgen.rendering.binding$package$.renderStructs$$anonfun$1(Unknown Source)
        at scala.runtime.function.JProcedure1.apply(Unknown Source)
        at bindgen.rendering.binding$package$.renderStructs(Unknown Source)
        at bindgen.rendering.binding$package$.renderBinding(Unknown Source)
        at scala.util.Either.map(Unknown Source)
        at bindgen.Generate$.main(Unknown Source)
        at <none>.main(Unknown Source)
(FAILED [5AA005F8]) Executing [/Users/khaled/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/indoorvivants/bindgen_native0.5_3/0.2.3/bindgen_native0.5_3-0.2.3-aarch64-osx.jar --header /Users/khaled/workspaces/oss/sn-bindgen-template/modules/ffmpeg/src/main/resources/scala-native/ffmpeg-amalgam.h --package ffmpeg_wrapper --link-name ffmpeg_wrapper --c-import libavformat/avformat.h --c-import libavcodec/avcodec.h --c-import libavutil/avutil.h --clang -I/Users/khaled/Library/Caches/sbt-vcpkg/vcpkg/packages/ffmpeg_arm64-osx/include --trace --scala --multi-file --flavour scala-native05 --clang-path /usr/bin/clang --out /Users/khaled/workspaces/oss/sn-bindgen-template/modules/ffmpeg/target/scala-3.3.5/src_managed/main/ffmpeg_wrapper --print-files]
[error] stack trace is suppressed; run last ffmpegModule / Compile / bindgenGenerateScalaSources for the full output
[error] (ffmpegModule / Compile / bindgenGenerateScalaSources) java.lang.Exception: Process [/Users/khaled/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/indoorvivants/bindgen_native0.5_3/0.2.3/bindgen_native0.5_3-0.2.3-aarch64-osx.jar --header /Users/khaled/workspaces/oss/sn-bindgen-template/modules/ffmpeg/src/main/resources/scala-native/ffmpeg-amalgam.h --package ffmpeg_wrapper --link-name ffmpeg_wrapper --c-import libavformat/avformat.h --c-import libavcodec/avcodec.h --c-import libavutil/avutil.h --clang -I/Users/khaled/Library/Caches/sbt-vcpkg/vcpkg/packages/ffmpeg_arm64-osx/include --trace --scala --multi-file --flavour scala-native05 --clang-path /usr/bin/clang --out /Users/khaled/workspaces/oss/sn-bindgen-template/modules/ffmpeg/target/scala-3.3.5/src_managed/main/ffmpeg_wrapper --print-files] failed with code 1

Could anyone help me understand:

What does Reference(Unnamed) mean in this context?
I'm not sure to understand but from the logs, it seems to be related to AVStreamGroupTileGrid. Is there a recommended workaround to help sn-bindgen resolve that ?

I noticed that jextract (via ffmpeg4java) can process these headers for Java. I was hoping to test the Scala Native path and contribute something useful back, if possible.

Thanks in advance for any insights!

[keynmol]

Hello 👋

TLDR: What you are seeing is a bug, I've minimised it to

typedef struct AVStreamGroupTileGrid {
    union {
        int vertical;
    } *offsets;

} AVStreamGroupTileGrid;

by making offsets a pointer, I can crash bindgen, it generates fine otherwise.

I'll take a look at how to fix it – but the general gist of it is that sn-bindgen tries to generate bindings with more typesafety than jextract – which erases offsets to just a generic pointer. It provides layout definition later, but it's not dealing with the complex types and nested definitions that arise if you try to build a typesafe definition of AVStreamGroupTileGrid.

This process necessarily requires some extra logic to handle anonymous structs and unions, and unfortunately this part of bindgen has always been difficult to get right.

I do have tests for anonymous structs and unions but it seems to be sensitive to pointer types, which I didn't anticipate.

[kbeldjilali]

Hi,

Thanks a lot for you detailed and insightful response (and also for the amazing work you are doing).
Following you explanation, I've decided to switch strategies a bit: I've clone the project locally and named the anonymous struct. This workaround allowed me to move forward.

I will keep an eye on future patches.

Thanks again for your time.

---

out of topic, but if someone is also interested in trying ffmpeg in scala-native. ("make it work" version for macos arm...)

ffmpeg-amalgam.h

#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>

build.sbt

lazy val ffmpegModule =
  project
    .in(file("modules/binding-ffmpeg"))
    .enablePlugins(ScalaNativePlugin, BindgenPlugin)    
    .settings(
      scalaVersion := "3.3.5",
      
      bindgenBindings += {      
        val bd = (baseDirectory.value / "ffmpeg" )
        Binding(
            header = baseDirectory.value / "src" / "main" / "resources" / "scala-native" / "ffmpeg-amalgam.h",
            packageName = "bindingffmpeg",            
          )        
          .withLinkName("bindingffmpeg")        
          .withCImports(List("ffmpeg-amalgam.h"))
          .withClangFlags(List(s"-I$bd"))
          .withLogLevel(bindgen.interface.LogLevel.Trace)

    },
    nativeConfig := {
      val base = baseDirectory.value / "ffmpeg"
      val conf = nativeConfig.value

      // heavily inspired by https://github.com/nextbreakpoint/ffmpeg4java/blob/b132cbcdc2aa9a46feeea39f580b48dd0a982e5e/ffmpeg4java-macos/native/scripts/compile.sh
      conf
        .withLinkingOptions(
          conf.linkingOptions ++ List(
            "-Wl,-force_load", s"${base / "libavutil" / "libavutil.a"}",
            "-Wl,-force_load", s"${base / "libavcodec" / "libavcodec.a"}",
            "-Wl,-force_load", s"${base / "libavformat" / "libavformat.a"}",
            "-Wl,-force_load", s"${base / "libswresample" / "libswresample.a"}",          
            "-lm", "-lz", "-lbz2", "-liconv",          
            "-framework", "CoreFoundation",
            "-framework", "VideoToolbox",
            "-framework", "AudioToolbox",
            "-framework", "AVFoundation",
            "-framework", "Foundation",
            "-framework", "CoreImage",
            "-framework", "CoreGraphics",
            "-framework", "CoreMedia",
            "-framework", "QuartzCore",
            "-framework", "OpenGL",
            "-framework", "OpenAL",
            "-framework", "VideoDecodeAcceleration",
            "-framework", "QTKit",
            "-framework", "AppKit",
            "-framework", "Security",

          )
        )
        .withCompileOptions(
           conf.compileOptions ++ List(
            s"-I${baseDirectory.value / "src" / "main" / "resources" / "scala-native"}",
            // "-Wall",            
          ))      
    }
  )

and finally, the changed file
libavformat/avformat.h

-typedef struct AVStreamGroupTileGrid {
+typedef struct AVStreamGroupTileGridOffset {
+    unsigned int idx;
+    int horizontal;
+    int vertical;
+} AVStreamGroupTileGridOffset;
+
+ typedef struct AVStreamGroupTileGrid {
     const AVClass *av_class;
 
     /**
@@ -1014,24 +1020,7 @@ typedef struct AVStreamGroupTileGrid {
      *
      * Freed by libavformat in avformat_free_context().
      */
-    struct {
-        /**
-         * Index of the stream in the group this tile references.
-         *
-         * Must be < @ref AVStreamGroup.nb_streams "nb_streams".
-         */
-        unsigned int idx;
-        /**
-         * Offset in pixels from the left edge of the canvas where the tile
-         * should be placed.
-         */
-        int horizontal;
-        /**
-         * Offset in pixels from the top edge of the canvas where the tile
-         * should be placed.
-         */
-        int vertical;
-    } *offsets;
+    AVStreamGroupTileGridOffset *offsets;

[keynmol]

Actually I will keep this open – both as a reference to the particular issue I need to fix, but also to remind myself to port you build (thanks for that btw!) to https://github.com/indoorvivants/sn-bindgen-examples