Added audio resampler first pass

Author: djthorpe

decoder.go

@@ -97,7 +97,12 @@ func (d *demuxer) newDecoder(stream *ff.AVStream, dest Parameters) (*decoder, er
 	} else if !equals {
 		switch src.CodecType() {
 		case ff.AVMEDIA_TYPE_AUDIO:
-			fmt.Println("TODO: set up resampler", dest)
+			if resampler, frame, err := newResampler(dest, src); err != nil {
+				return nil, err
+			} else {
+				decoder.resampler = resampler
+				decoder.reframe = frame
+			}
 		case ff.AVMEDIA_TYPE_VIDEO:
 			if rescaler, frame, err := newResizer(dest, src); err != nil {
 				return nil, err
@@ -155,10 +160,61 @@ func newResizer(dest Parameters, src *ff.AVCodecParameters) (*ff.SWSContext, *ff
 		ff.SWScale_free_context(ctx)
 		return nil, nil, errors.New("failed to allocate frame")
 	} else {
+		// Set frame parameters
+		frame.SetPixFmt(dest_pixel_format)
+		frame.SetWidth(dest.Width())
+		frame.SetHeight(dest.Height())
+
+		// Return success
 		return ctx, frame, nil
 	}
 }
 
+func newResampler(dest Parameters, src *ff.AVCodecParameters) (*ff.SWRContext, *ff.AVFrame, error) {
+	// Get native sample format and channel layout
+	var dest_channel_layout ff.AVChannelLayout
+	dest_sample_format := ff.AVUtil_get_sample_fmt(dest.SampleFormat())
+	if err := ff.AVUtil_channel_layout_from_string(&dest_channel_layout, dest.ChannelLayout()); err != nil {
+		return nil, nil, fmt.Errorf("failed to get channel layout: %w", err)
+	}
+
+	// Create a new resampler
+	ctx := ff.SWResample_alloc()
+	if ctx == nil {
+		return nil, nil, errors.New("failed to allocate resampler")
+	}
+
+	// Set options to covert from the codec frame to the decoder frame
+	if err := ff.SWResample_set_opts(ctx,
+		dest_channel_layout, dest_sample_format, dest.Samplerate(), // destination
+		src.ChannelLayout(), src.SampleFormat(), src.Samplerate(), // source
+	); err != nil {
+		ff.SWResample_free(ctx)
+		return nil, nil, fmt.Errorf("SWResample_set_opts: %w", err)
+	}
+
+	// Initialize the resampling context
+	if err := ff.SWResample_init(ctx); err != nil {
+		ff.SWResample_free(ctx)
+		return nil, nil, fmt.Errorf("SWResample_init: %w", err)
+	}
+
+	// Create a new frame for the resampled audio
+	frame := ff.AVUtil_frame_alloc()
+	if frame == nil {
+		ff.SWResample_free(ctx)
+		return nil, nil, errors.New("failed to allocate frame")
+	}
+
+	// Set frame parameters
+	frame.SetSampleRate(dest.Samplerate())
+	frame.SetSampleFormat(dest_sample_format)
+	frame.SetChannelLayout(dest_channel_layout)
+
+	// Return success
+	return ctx, frame, nil
+}
+
 func (d *demuxer) close() error {
 	var result error
 
@@ -327,7 +383,14 @@ func (d *decoder) decode(packet *ff.AVPacket, demuxfn DecoderFunc, framefn Frame
 func (d *decoder) re(src *ff.AVFrame) (*ff.AVFrame, error) {
 	switch d.codec.Codec().Type() {
 	case ff.AVMEDIA_TYPE_AUDIO:
-		fmt.Println("TODO: resample audio", src)
+		if d.resampler != nil && src != nil {
+			// Resample the audio
+			if err := resample(d.resampler, d.reframe, src); err != nil {
+				return nil, err
+			} else {
+				return d.reframe, nil
+			}
+		}
 	case ff.AVMEDIA_TYPE_VIDEO:
 		if d.rescaler != nil && src != nil {
 			// Rescale the video
@@ -339,10 +402,6 @@ func (d *decoder) re(src *ff.AVFrame) (*ff.AVFrame, error) {
 		}
 	}
 
-	//	if err := decoder.rescale(decoder.frame, src); err != nil {
-	//		return nil, err
-	//	}
-
 	// NO-OP - just return the source frame
 	return src, nil
 }
@@ -359,6 +418,27 @@ func rescale(ctx *ff.SWSContext, dest, src *ff.AVFrame) error {
 	return nil
 }
 
+func resample(ctx *ff.SWRContext, dest, src *ff.AVFrame) error {
+	// Copy properties from source
+	//if err := ff.AVUtil_frame_copy_props(dest, src); err != nil {
+	//	return fmt.Errorf("failed to copy props: %w", err)
+	//}
+
+	dest_samples, err := ff.SWResample_get_out_samples(ctx, src.NumSamples())
+	if err != nil {
+		return fmt.Errorf("SWResample_get_out_samples: %w", err)
+	}
+	dest.SetNumSamples(dest_samples)
+	fmt.Println("dest frame=", dest)
+
+	// Perform resampling
+	if err := ff.SWResample_convert_frame(ctx, src, dest); err != nil {
+		return fmt.Errorf("SWResample_convert_frame: %w", err)
+	}
+
+	return nil
+}
+
 // Return an error if the parameters don't match the stream type (AUDIO, VIDEO)
 // Return true if the codec parameters are compatible with the stream
 func equalsStream(dest Parameters, src *ff.AVCodecParameters) (bool, error) {

decoder_test.go

@@ -168,7 +168,7 @@ func Test_decoder_004(t *testing.T) {
 
 	// Open the file
 	manager := NewManager()
-	media, err := manager.Open("./etc/test/sample.mp4", nil)
+	media, err := manager.Open("./etc/test/audio_22050_1ch_5m35.s16le.sw", nil)
 	if !assert.NoError(err) {
 		t.SkipNow()
 	}
@@ -188,8 +188,81 @@ func Test_decoder_004(t *testing.T) {
 	}
 
 	// This is the function which processes the audio frames
+	tmp, err := os.MkdirTemp("", "media_test_")
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	filename := filepath.Join(tmp, "audio.sw")
+	f, err := os.Create(filename)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	defer f.Close()
+
+	bytes_written := 0
+	framefn := func(frame Frame) error {
+		n, err := f.Write(frame.Bytes(0))
+		if err != nil {
+			return err
+		} else {
+			bytes_written += n
+			t.Logf("Written %d bytes to %v", bytes_written, filename)
+		}
+		return nil
+	}
+
+	// Finally, this is where we actually decode frames from the stream
+	assert.NoError(decoder.Decode(context.Background(), framefn))
+}
+
+func Test_decoder_005(t *testing.T) {
+	// Decode audio frames
+	assert := assert.New(t)
+
+	// Open the file
+	manager := NewManager()
+	media, err := manager.Open("./test.mp3", nil)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	defer media.Close()
+
+	// Create a decoder to decompress the audio
+	decoder, err := media.Decoder(func(stream Stream) (Parameters, error) {
+		// Audio - downsample to stereo, s16
+		if stream.Type() == AUDIO {
+			return manager.AudioParameters("mono", "s16", 44100)
+		}
+		// Ignore other streams
+		return nil, nil
+	})
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+
+	// This is the function which processes the audio frames
+	tmp, err := os.MkdirTemp("", "media_test_")
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+
+	// TODO: Endian might be le or be depending on the native endianness
+	filename := filepath.Join(tmp, "audio.s16le.sw")
+	f, err := os.Create(filename)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	defer f.Close()
+
+	bytes_written := 0
 	framefn := func(frame Frame) error {
-		t.Log(frame)
+		n, err := f.Write(frame.Bytes(0))
+		if err != nil {
+			return err
+		} else {
+			bytes_written += n
+			t.Logf("Written %d bytes to %v", bytes_written, filename)
+		}
 		return nil
 	}
 

etc/test/audio_22050_1ch_5m35.s16le renamed to etc/test/audio_22050_1ch_5m35.s16le.sw

@@ -249,7 +249,7 @@ type Frame interface {
 
 	// Return a frame as an image, which supports the following
 	// pixel formats: AV_PIX_FMT_GRAY8, AV_PIX_FMT_RGBA,
-	// AV_PIX_FMT_RGB24, AV_PIX_FMT_YUV420P (4:2:0)
+	// AV_PIX_FMT_RGB24, AV_PIX_FMT_YUV420P
 	Image() (image.Image, error)
 }
 

interfaces.go

@@ -7,9 +7,6 @@ import (
 	// Package imports
 	version "github.com/mutablelogic/go-media/pkg/version"
 	ff "github.com/mutablelogic/go-media/sys/ffmpeg61"
-
-	// Namespace imports
-	. "github.com/djthorpe/go-errors"
 )
 
 ////////////////////////////////////////////////////////////////////////////

manager.go

@@ -87,6 +87,26 @@ func newAudioParametersEx(channels string, samplefmt string, samplerate int) (*p
 	return par, nil
 }
 
+func newCodecAudioParameters(codec *ff.AVCodecParameters) *par {
+	par := new(par)
+	par.t = AUDIO
+	par.audiopar.Ch = codec.ChannelLayout()
+	par.audiopar.SampleFormat = codec.SampleFormat()
+	par.audiopar.Samplerate = codec.Samplerate()
+	par.planepar.NumPlanes = par.NumPlanes()
+	return par
+}
+
+func newCodecVideoParameters(codec *ff.AVCodecParameters) *par {
+	par := new(par)
+	par.t = VIDEO
+	par.videopar.Width = codec.Width()
+	par.videopar.Height = codec.Height()
+	par.videopar.PixelFormat = codec.PixelFormat()
+	par.planepar.NumPlanes = par.NumPlanes()
+	return par
+}
+
 // Create new parameters for video from a width, height and pixel format
 func newVideoParametersEx(width int, height int, pixelfmt string) (*par, error) {
 	par := new(par)

parameters.go

@@ -17,7 +17,8 @@ type writerstream struct {
 }
 
 var _ Stream = (*stream)(nil)
-var _ Stream = (*writerstream)(nil)
+
+//var _ Stream = (*writerstream)(nil)
 
 ////////////////////////////////////////////////////////////////////////////////
 // LIFECYCLE
@@ -27,6 +28,7 @@ func newStream(ctx *ff.AVStream) *stream {
 	return &stream{ctx}
 }
 
+/*
 // Stream wrapper for encoding
 func newWriterStream(ctx *ff.AVFormatContext, param Parameters) (*writerstream, error) {
 	// Parameters - Codec
@@ -44,7 +46,7 @@ func newWriterStream(ctx *ff.AVFormatContext, param Parameters) (*writerstream,
 
 	return nil, ErrNotImplemented
 }
-
+*/
 ////////////////////////////////////////////////////////////////////////////////
 // PUBLIC METHODS
 
@@ -61,6 +63,13 @@ func (stream *stream) Type() MediaType {
 }
 
 func (stream *stream) Parameters() Parameters {
-	// TODO
-	return new(par)
+	switch stream.Type() {
+	case AUDIO:
+		return newCodecAudioParameters(stream.CodecPar())
+	case VIDEO:
+		return newCodecVideoParameters(stream.CodecPar())
+	}
+
+	// Other types not yet supported
+	return nil
 }

stream.go

@@ -2,7 +2,6 @@ package media
 
 import (
 	"errors"
-	"fmt"
 	"io"
 
 	// Packages
@@ -55,15 +54,16 @@ func createMedia(url string, format Format, metadata []Metadata, params ...Param
 	}
 
 	// Add streams
-	for _, param := range params {
-		stream, err := newWriterStream(ctx, param)
-		if err != nil {
-			return nil, errors.Join(err, writer.Close())
-		} else {
-			fmt.Println("TODO: STREAM", stream)
+	/*
+		for _, param := range params {
+			stream, err := newWriterStream(ctx, param)
+			if err != nil {
+				return nil, errors.Join(err, writer.Close())
+			} else {
+				fmt.Println("TODO: STREAM", stream)
+			}
 		}
-	}
-
+	*/
 	// Open the output file, if needed
 	if !ctx.Flags().Is(ff.AVFMT_NOFILE) {
 		w, err := ff.AVFormat_avio_open(url, ff.AVIO_FLAG_WRITE)