Updated

Author: djthorpe

cmd/decode/main.go renamed to cmd/examples/decode/main.go

@@ -23,9 +23,12 @@ type demuxer struct {
 
 // decoder context - decodes packets into frames
 type decoder struct {
-	stream int
-	codec  *ff.AVCodecContext
-	frame  *ff.AVFrame // Destination frame
+	stream    int
+	codec     *ff.AVCodecContext
+	frame     *ff.AVFrame    // Destination frame
+	reframe   *ff.AVFrame    // Destination frame after resample or resize
+	resampler *ff.SWRContext // Resampler for audio
+	rescaler  *ff.SWSContext // Rescaler for video
 }
 
 var _ Decoder = (*demuxer)(nil)
@@ -53,16 +56,13 @@ func newDemuxer(input *ff.AVFormatContext, mapfn DecoderMapFunc) (*demuxer, erro
 	// destination frame. If it's nil then it's mostly a copy.
 	var result error
 	for _, stream := range streams {
-		// Get decoder parameters
+		// Get decoder parameters and map to a decoder
 		parameters, err := mapfn(newStream(stream))
 		if err != nil {
 			result = errors.Join(result, err)
 		} else if parameters == nil {
 			continue
-		}
-
-		// Create the decoder with the parameters
-		if decoder, err := demuxer.newDecoder(stream, parameters); err != nil {
+		} else if decoder, err := demuxer.newDecoder(stream, parameters); err != nil {
 			result = errors.Join(result, err)
 		} else {
 			streamNum := stream.Index()
@@ -86,11 +86,29 @@ func newDemuxer(input *ff.AVFormatContext, mapfn DecoderMapFunc) (*demuxer, erro
 	return demuxer, nil
 }
 
-func (d *demuxer) newDecoder(stream *ff.AVStream, parameters Parameters) (*decoder, error) {
+func (d *demuxer) newDecoder(stream *ff.AVStream, dest Parameters) (*decoder, error) {
 	decoder := new(decoder)
 	decoder.stream = stream.Id()
 
-	// TODO: Use parameters to create the decoder
+	// Use parameters to create the decoder resampler or resizer
+	src := stream.CodecPar()
+	if equals, err := equalsStream(dest, src); err != nil {
+		return nil, err
+	} else if !equals {
+		switch src.CodecType() {
+		case ff.AVMEDIA_TYPE_AUDIO:
+			fmt.Println("TODO: set up resampler", dest)
+		case ff.AVMEDIA_TYPE_VIDEO:
+			if rescaler, frame, err := newResizer(dest, src); err != nil {
+				return nil, err
+			} else {
+				decoder.rescaler = rescaler
+				decoder.reframe = frame
+			}
+		default:
+			return nil, fmt.Errorf("new decoder: unsupported stream type %v", src.CodecType())
+		}
+	}
 
 	// Create a codec context for the decoder
 	codec := ff.AVCodec_find_decoder(stream.CodecPar().CodecID())
@@ -123,6 +141,24 @@ func (d *demuxer) newDecoder(stream *ff.AVStream, parameters Parameters) (*decod
 	return decoder, nil
 }
 
+func newResizer(dest Parameters, src *ff.AVCodecParameters) (*ff.SWSContext, *ff.AVFrame, error) {
+	// Get native pixel format
+	dest_pixel_format := ff.AVUtil_get_pix_fmt(dest.PixelFormat())
+
+	// Create scaling context and destination frame
+	if ctx := ff.SWScale_get_context(
+		src.Width(), src.Height(), src.PixelFormat(), // source
+		dest.Width(), dest.Height(), dest_pixel_format, // destination
+		ff.SWS_BILINEAR, nil, nil, nil); ctx == nil {
+		return nil, nil, errors.New("failed to allocate swscale context")
+	} else if frame := ff.AVUtil_frame_alloc(); frame == nil {
+		ff.SWScale_free_context(ctx)
+		return nil, nil, errors.New("failed to allocate frame")
+	} else {
+		return ctx, frame, nil
+	}
+}
+
 func (d *demuxer) close() error {
 	var result error
 
@@ -149,6 +185,21 @@ func (d *decoder) close() error {
 		ff.AVCodec_free_context(d.codec)
 	}
 
+	// Free the resampler
+	if d.resampler != nil {
+		ff.SWResample_free(d.resampler)
+	}
+
+	// Free the rescaler
+	if d.rescaler != nil {
+		ff.SWScale_free_context(d.rescaler)
+	}
+
+	// Free rescaler frame
+	if d.reframe != nil {
+		ff.AVUtil_frame_free(d.reframe)
+	}
+
 	// Free destination frame
 	if d.frame != nil {
 		ff.AVUtil_frame_free(d.frame)
@@ -235,6 +286,7 @@ func (d *decoder) decode(packet *ff.AVPacket, demuxfn DecoderFunc, framefn Frame
 	}
 
 	// get all the available frames from the decoder
+	var result error
 	for {
 		if err := ff.AVCodec_receive_frame(d.codec, d.frame); errors.Is(err, syscall.EAGAIN) || errors.Is(err, io.EOF) {
 			// Finished decoding packet or EOF
@@ -243,39 +295,121 @@ func (d *decoder) decode(packet *ff.AVPacket, demuxfn DecoderFunc, framefn Frame
 			return err
 		}
 
-		// Pass the frame
-		if err := framefn(newFrame(d.frame)); errors.Is(err, io.EOF) {
-			// End early
+		// Resample or resize the frame, then pass to the frame function
+		if frame, err := d.re(d.frame); err != nil {
+			return err
+		} else if err := framefn(newFrame(frame)); errors.Is(err, io.EOF) {
+			// End early, return EOF
+			result = io.EOF
 			break
 		} else if err != nil {
 			return err
 		}
-
-		// Resample or resize the frame, then pass back
-		/*
-			if frame, err := codec.(*decoder).re(r.frame); err != nil {
-				return err
-			} else if err := fn(frame); errors.Is(err, io.EOF) {
-				// End early
-				break
-			} else if err != nil {
-				return err
-			}
-		*/
 	}
 
-	// TODO: Flush
-	/*
-		if frame, err := codec.(*decoder).re(nil); err != nil {
+	// Flush the resizer or resampler if we haven't received an EOF
+	if result == nil {
+		if frame, err := d.re(nil); err != nil {
 			return err
 		} else if frame == nil {
 			// NOOP
-		} else if err := fn(frame); errors.Is(err, io.EOF) {
+		} else if err := framefn(newFrame(d.frame)); errors.Is(err, io.EOF) {
 			// NOOP
 		} else if err != nil {
 			return err
-		}*/
+		}
+	}
 
-	// Return success
+	// Return success or EOF
+	return result
+}
+
+func (d *decoder) re(src *ff.AVFrame) (*ff.AVFrame, error) {
+	switch d.codec.Codec().Type() {
+	case ff.AVMEDIA_TYPE_AUDIO:
+		fmt.Println("TODO: resample audio", src)
+	case ff.AVMEDIA_TYPE_VIDEO:
+		if d.rescaler != nil && src != nil {
+			// Rescale the video
+			if err := rescale(d.rescaler, d.reframe, src); err != nil {
+				return nil, err
+			} else {
+				return d.reframe, nil
+			}
+		}
+	}
+
+	//	if err := decoder.rescale(decoder.frame, src); err != nil {
+	//		return nil, err
+	//	}
+
+	// NO-OP - just return the source frame
+	return src, nil
+}
+
+func rescale(ctx *ff.SWSContext, dest, src *ff.AVFrame) error {
+	// Copy properties from source
+	if err := ff.AVUtil_frame_copy_props(dest, src); err != nil {
+		return fmt.Errorf("failed to copy props: %w", err)
+	}
+	// Perform rescale
+	if err := ff.SWScale_scale_frame(ctx, dest, src, false); err != nil {
+		return fmt.Errorf("SWScale_scale_frame: %w", err)
+	}
 	return nil
 }
+
+// Return an error if the parameters don't match the stream type (AUDIO, VIDEO)
+// Return true if the codec parameters are compatible with the stream
+func equalsStream(dest Parameters, src *ff.AVCodecParameters) (bool, error) {
+	switch src.CodecType() {
+	case ff.AVMEDIA_TYPE_AUDIO:
+		if !dest.Type().Is(AUDIO) {
+			return false, fmt.Errorf("source is audio, but destination is %v", dest.Type())
+		} else {
+			return equalsAudioPar(dest, src), nil
+		}
+	case ff.AVMEDIA_TYPE_VIDEO:
+		if !dest.Type().Is(VIDEO) {
+			return false, fmt.Errorf("source is video, but destination are %v", dest.Type())
+		} else {
+			return equalsVideoPar(dest, src), nil
+		}
+	default:
+		return false, fmt.Errorf("unsupported source %v", src.CodecType())
+	}
+}
+
+// Return true if the audio parameters are compatible with the stream
+func equalsAudioPar(parameters Parameters, codec *ff.AVCodecParameters) bool {
+	samplefmt := ff.AVUtil_get_sample_fmt_name(codec.SampleFormat())
+	if samplefmt != parameters.SampleFormat() {
+		return false
+	}
+	ch_layout := ff.AVChannelLayout(codec.ChannelLayout())
+	channellayout, err := ff.AVUtil_channel_layout_describe(&ch_layout)
+	if err != nil || channellayout != parameters.ChannelLayout() {
+		return false
+	}
+	if codec.Samplerate() != parameters.Samplerate() {
+		return false
+	}
+	// Matches
+	return true
+}
+
+// Return true if the video parameters are compatible with the stream
+func equalsVideoPar(parameters Parameters, codec *ff.AVCodecParameters) bool {
+	pixelfmt := ff.AVUtil_get_pix_fmt_name(codec.PixelFormat())
+	if pixelfmt != parameters.PixelFormat() {
+		return false
+	}
+	if codec.Width() != parameters.Width() {
+		return false
+	}
+	if codec.Height() != parameters.Height() {
+		return false
+	}
+	// Matches
+	return true
+}

cmd/server/handle_metadata.go renamed to cmd/examples/server/handle_metadata.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"image/jpeg"
+	"io"
 	"os"
 	"path/filepath"
 	"testing"
@@ -17,6 +18,7 @@ import (
 )
 
 func Test_decoder_001(t *testing.T) {
+	// Decode packets
 	assert := assert.New(t)
 
 	manager := NewManager()
@@ -46,6 +48,7 @@ func Test_decoder_001(t *testing.T) {
 }
 
 func Test_decoder_002(t *testing.T) {
+	// Decode video frames
 	assert := assert.New(t)
 
 	manager := NewManager()
@@ -93,3 +96,63 @@ func Test_decoder_002(t *testing.T) {
 	// decode frames from the stream
 	assert.NoError(decoder.Decode(context.Background(), framefn))
 }
+
+func Test_decoder_003(t *testing.T) {
+	// Decode video frames and resize them
+	assert := assert.New(t)
+
+	manager := NewManager()
+	media, err := manager.Open("./etc/test/sample.mp4", nil)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	defer media.Close()
+
+	decoder, err := media.Decoder(func(stream Stream) (Parameters, error) {
+		// Make greyscale images
+		if stream.Type() == VIDEO {
+			return manager.VideoParameters(640, 480, "yuv420p")
+		}
+		// Ignore other streams
+		return nil, nil
+	})
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+
+	// Frame function
+	n := 0
+	// tmpdir := t.TempDir()
+	tmpdir, err := os.MkdirTemp("", "media_test")
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	framefn := func(frame Frame) error {
+		if frame.Type() != VIDEO {
+			return nil
+		}
+		filename := filepath.Join(tmpdir, fmt.Sprintf("frame%03d.jpg", n))
+		w, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+		defer w.Close()
+		if image, err := frame.Image(); err != nil {
+			return err
+		} else if err := jpeg.Encode(w, image, nil); err != nil {
+			return err
+		} else {
+			t.Logf("Frame %d: %dx%d (%q) => %s", n, frame.Width(), frame.Height(), frame.PixelFormat(), filename)
+			n++
+		}
+		// Stop after 10 frames
+		if n >= 10 {
+			return io.EOF
+		} else {
+			return nil
+		}
+	}
+
+	// decode frames from the stream
+	assert.NoError(decoder.Decode(context.Background(), framefn))
+}

cmd/server/main.go renamed to cmd/examples/server/main.go

@@ -67,11 +67,11 @@ type Manager interface {
 
 	// Return audio parameters for encoding
 	// ChannelLayout, SampleFormat, Samplerate
-	AudioParameters(string, string, int) (AudioParameters, error)
+	AudioParameters(string, string, int) (Parameters, error)
 
 	// Return video parameters for encoding
 	// Width, Height, PixelFormat
-	VideoParameters(int, int, string) (VideoParameters, error)
+	VideoParameters(int, int, string) (Parameters, error)
 
 	// Return codec parameters for audio encoding
 	// Codec name and AudioParameters

decoder.go

@@ -247,13 +247,13 @@ func (manager *manager) Codecs() []Metadata {
 
 // Return audio parameters for encoding
 // ChannelLayout, SampleFormat, Samplerate
-func (manager *manager) AudioParameters(channels string, samplefmt string, samplerate int) (AudioParameters, error) {
+func (manager *manager) AudioParameters(channels string, samplefmt string, samplerate int) (Parameters, error) {
 	return newAudioParametersEx(channels, samplefmt, samplerate)
 }
 
 // Return video parameters for encoding
 // Width, Height, PixelFormat, Framerate
-func (manager *manager) VideoParameters(width int, height int, pixelfmt string) (VideoParameters, error) {
+func (manager *manager) VideoParameters(width int, height int, pixelfmt string) (Parameters, error) {
 	return newVideoParametersEx(width, height, pixelfmt)
 }