Example code for SDL

Author: djthorpe

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/llgcode/draw2d v0.0.0-20240627062922-0ed1ff131195
 	github.com/mutablelogic/go-client v1.0.8
 	github.com/stretchr/testify v1.9.0
+	github.com/veandco/go-sdl2 v0.4.40
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 )
 

pkg/ffmpeg/decoder.go

@@ -7,6 +7,7 @@ import (
 	"syscall"
 
 	// Packages
+
 	ff "github.com/mutablelogic/go-media/sys/ffmpeg61"
 )
 

pkg/ffmpeg/frame.go

@@ -19,6 +19,7 @@ type Frame ff.AVFrame
 
 const (
 	PTS_UNDEFINED = ff.AV_NOPTS_VALUE
+	TS_UNDEFINED  = -1.0
 )
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -92,6 +93,42 @@ func (frame *Frame) MakeWritable() error {
 	return ff.AVUtil_frame_make_writable((*ff.AVFrame)(frame))
 }
 
+// Make a copy of the frame, which should be released by the caller
+func (frame *Frame) Copy() (*Frame, error) {
+	copy := ff.AVUtil_frame_alloc()
+	if copy == nil {
+		return nil, errors.New("failed to allocate frame")
+	}
+	switch frame.Type() {
+	case media.AUDIO:
+		copy.SetSampleFormat(frame.SampleFormat())
+		copy.SetChannelLayout(frame.ChannelLayout())
+		copy.SetSampleRate(frame.SampleRate())
+		copy.SetNumSamples(frame.NumSamples())
+	case media.VIDEO:
+		copy.SetPixFmt(frame.PixelFormat())
+		copy.SetWidth(frame.Width())
+		copy.SetHeight(frame.Height())
+		copy.SetSampleAspectRatio(frame.SampleAspectRatio())
+	default:
+		ff.AVUtil_frame_free(copy)
+		return nil, errors.New("invalid codec type")
+	}
+	if err := ff.AVUtil_frame_get_buffer(copy, false); err != nil {
+		ff.AVUtil_frame_free(copy)
+		return nil, err
+	}
+	if err := ff.AVUtil_frame_copy(copy, (*ff.AVFrame)(frame)); err != nil {
+		ff.AVUtil_frame_free(copy)
+		return nil, err
+	}
+	if err := ff.AVUtil_frame_copy_props(copy, (*ff.AVFrame)(frame)); err != nil {
+		ff.AVUtil_frame_free(copy)
+		return nil, err
+	}
+	return (*Frame)(copy), nil
+}
+
 // Unreference frame buffers
 func (frame *Frame) Unref() {
 	ff.AVUtil_frame_unref((*ff.AVFrame)(frame))
@@ -199,17 +236,17 @@ func (frame *Frame) IncPts(v int64) {
 	(*ff.AVFrame)(frame).SetPts((*ff.AVFrame)(frame).Pts() + v)
 }
 
-// Return the timestamp in seconds, or PTS_UNDEFINED if the timestamp
+// Return the timestamp in seconds, or TS_UNDEFINED if the timestamp
 // is undefined or timebase is not set
 func (frame *Frame) Ts() float64 {
 	ctx := (*ff.AVFrame)(frame)
 	pts := ctx.Pts()
 	if pts == ff.AV_NOPTS_VALUE {
-		return PTS_UNDEFINED
+		return TS_UNDEFINED
 	}
 	tb := ctx.TimeBase()
 	if tb.Num() == 0 || tb.Den() == 0 {
-		return PTS_UNDEFINED
+		return TS_UNDEFINED
 	}
 	return ff.AVUtil_rational_q2d(tb) * float64(pts)
 }

pkg/ffmpeg/packet.go

@@ -23,16 +23,16 @@ func (packet *Packet) String() string {
 ///////////////////////////////////////////////////////////////////////////////
 // PUBLIC METHODS
 
-// Return the timestamp in seconds, or PTS_UNDEFINED if the timestamp
+// Return the timestamp in seconds, or TS_UNDEFINED if the timestamp
 // is undefined or timebase is not set
 func (packet *Packet) Ts() float64 {
 	if packet == nil {
-		return PTS_UNDEFINED
+		return TS_UNDEFINED
 	}
 	if pts := (*ff.AVPacket)(packet).Pts(); pts == ff.AV_NOPTS_VALUE {
-		return PTS_UNDEFINED
+		return TS_UNDEFINED
 	} else if tb := (*ff.AVPacket)(packet).TimeBase(); tb.Num() == 0 || tb.Den() == 0 {
-		return PTS_UNDEFINED
+		return TS_UNDEFINED
 	} else {
 		return ff.AVUtil_rational_q2d(tb) * float64(pts)
 	}

pkg/sdl/sdl.go

@@ -3,17 +3,17 @@ package main
 import (
 	"context"
 	"errors"
-	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"runtime"
 	"sync"
 	"time"
+	"unsafe"
 
 	// Packages
-	"github.com/mutablelogic/go-media"
-	"github.com/mutablelogic/go-media/pkg/ffmpeg"
+	media "github.com/mutablelogic/go-media"
+	ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg"
 	sdl "github.com/veandco/go-sdl2/sdl"
 )
 
@@ -41,12 +41,6 @@ func (s *Context) Close() error {
 	return nil
 }
 
-func (s *Context) PushQuitEvent() {
-	sdl.PushEvent(&sdl.QuitEvent{
-		Type: sdl.QUIT,
-	})
-}
-
 func (s *Context) NewWindow(title string, width, height int32) (*Window, error) {
 	window, err := sdl.CreateWindow(
 		title,
@@ -67,7 +61,6 @@ func (s *Context) NewWindow(title string, width, height int32) (*Window, error)
 		window.Destroy()
 		return nil, err
 	}
-
 	return &Window{window, renderer, texture}, nil
 }
 
@@ -82,6 +75,7 @@ func (w *Window) Close() error {
 	if err := (*sdl.Window)(w.Window).Destroy(); err != nil {
 		result = errors.Join(result, err)
 	}
+	w.Texture = nil
 	w.Renderer = nil
 	w.Window = nil
 
@@ -109,26 +103,63 @@ func (w *Window) RenderFrame(frame *ffmpeg.Frame) error {
 	)
 }
 
-func (s *Context) RunLoop() {
+func (s *Context) RunLoop(w *Window, evt uint32) {
 	runtime.LockOSThread()
 	running := true
+
+	pts := ffmpeg.TS_UNDEFINED
 	for running {
 		for event := sdl.PollEvent(); event != nil; event = sdl.PollEvent() {
 			switch event.(type) {
 			case *sdl.QuitEvent:
 				running = false
 				break
+			case *sdl.UserEvent:
+				if event.(*sdl.UserEvent).Type != evt {
+					break
+				}
+
+				// Get the video frame - if nil, then end of stream
+				frame := (*ffmpeg.Frame)(event.(*sdl.UserEvent).Data1)
+				if frame == nil {
+					running = false
+					break
+				}
+
+				// Pause to present the frame at the correct PTS
+				if pts != ffmpeg.TS_UNDEFINED && pts < frame.Ts() {
+					pause := frame.Ts() - pts
+					if pause > 0 {
+						sdl.Delay(uint32(pause * 1000))
+					}
+				}
+
+				// Set current timestamp
+				pts = frame.Ts()
+
+				// Render the frame, release the frame resources
+				if err := w.RenderFrame(frame); err != nil {
+					log.Print(err)
+				} else if err := w.Flush(); err != nil {
+					log.Print(err)
+				} else if err := frame.Close(); err != nil {
+					log.Print(err)
+				}
+
 			}
 		}
 	}
 }
 
 func main() {
-	sdl, err := NewSDL()
+	ctx, err := NewSDL()
 	if err != nil {
 		log.Fatal(err)
 	}
-	defer sdl.Close()
+	defer ctx.Close()
+
+	// Register an event for a new frame
+	evt := sdl.RegisterEvents(1)
 
 	// Open video
 	input, err := ffmpeg.Open(os.Args[1])
@@ -151,24 +182,28 @@ func main() {
 		return nil, nil
 	}
 
-	ch := make(chan *ffmpeg.Frame)
-
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		err := input.Decode(context.Background(), mapfn, func(stream int, frame *ffmpeg.Frame) error {
-			ch <- frame
+			copy, err := frame.Copy()
+			if err != nil {
+				copy.Close()
+				return err
+			}
+			sdl.PushEvent(&sdl.UserEvent{
+				Type:  evt,
+				Data1: unsafe.Pointer(copy),
+			})
 			return nil
 		})
 		if err != nil {
 			result = errors.Join(result, err)
 		}
-
-		// Close channel
-		close(ch)
-
 		// Quit event
-		sdl.PushQuitEvent()
+		sdl.PushEvent(&sdl.QuitEvent{
+			Type: sdl.QUIT,
+		})
 	}()
 
 	// HACK
@@ -183,27 +218,15 @@ func main() {
 		title = meta[0].Value()
 	}
 
-	window, err := sdl.NewWindow(title, w, h)
+	// Create a new window
+	window, err := ctx.NewWindow(title, w, h)
 	if err != nil {
 		log.Fatal(err)
 	}
 	defer window.Close()
 
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		for frame := range ch {
-			if err := window.RenderFrame(frame); err != nil {
-				fmt.Println("Error rendering frame:", err)
-			}
-			if err := window.Flush(); err != nil {
-				fmt.Println("Error flushing frame:", err)
-			}
-		}
-	}()
-
-	// Run the SDL loop
-	sdl.RunLoop()
+	// Run the SDL loop until quit
+	ctx.RunLoop(window, evt)
 
 	// Wait until all goroutines have finished
 	wg.Wait()

sys/ffmpeg61/avutil_frame.go

@@ -150,6 +150,14 @@ func AVUtil_frame_get_num_planes(frame *AVFrame) int {
 	return 0
 }
 
+// Copy frame data
+func AVUtil_frame_copy(dst, src *AVFrame) error {
+	if ret := AVError(C.av_frame_copy((*C.struct_AVFrame)(dst), (*C.struct_AVFrame)(src))); ret < 0 {
+		return ret
+	}
+	return nil
+}
+
 // Copy only "metadata" fields from src to dst, those fields that do not affect the data layout in the buffers.
 // E.g. pts, sample rate (for audio) or sample aspect ratio (for video), but not width/height or channel layout.
 // Side data is also copied.