Merge pull request #37 from mutablelogic/main

update from main

Author: djthorpe

Makefile

@@ -22,7 +22,7 @@ ifeq ($(GGML_CUDA),1)
 endif
 
 # Targets
-all: build server cli
+all: server cli
 
 # Generate the pkg-config files
 generate: mkdir go-tidy

cmd/server/main.go

@@ -56,7 +56,7 @@ func main() {
 
 	// Create a whisper service
 	log.Println("Storing models at", dir)
-	whisper, err := whisper.New(dir)
+	whisper, err := whisper.New(dir, whisper.OptMaxConcurrent(1))
 	if err != nil {
 		log.Println(err)
 		os.Exit(-2)

doc/notes.md

@@ -0,0 +1,7 @@
+        if (!whisper_is_multilingual(ctx)) {
+            if (params.language != "en" || params.translate) {
+                params.language = "en";
+                params.translate = false;
+                fprintf(stderr, "%s: WARNING: model is not multilingual, ignoring language and translation options\n", __func__);
+            }
+        }

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/djthorpe/go-tablewriter v0.0.8
 	github.com/go-audio/wav v1.1.0
 	github.com/mutablelogic/go-client v1.0.9
+	github.com/mutablelogic/go-media v1.6.8
 	github.com/mutablelogic/go-server v1.4.13
 	github.com/stretchr/testify v1.9.0
 )
@@ -19,6 +20,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/sys v0.22.0 // indirect
 	golang.org/x/term v0.22.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -8,8 +8,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/djthorpe/go-errors v1.0.3 h1:GZeMPkC1mx2vteXLI/gvxZS0Ee9zxzwD1mcYyKU5jD0=
 github.com/djthorpe/go-errors v1.0.3/go.mod h1:HtfrZnMd6HsX75Mtbv9Qcnn0BqOrrFArvCaj3RMnZhY=
-github.com/djthorpe/go-tablewriter v0.0.7 h1:jnNsJDjjLLCt0OAqB5DzGZN7V3beT1IpNMQ8GcOwZDU=
-github.com/djthorpe/go-tablewriter v0.0.7/go.mod h1:NVBvytpL+6fHfCKn0+3lSi15/G3A1HWf2cLNeHg6YBg=
 github.com/djthorpe/go-tablewriter v0.0.8 h1:uRhB9XVgK1n9tvVS7KMyxhxxGGtDvqC80toDTpW4DB4=
 github.com/djthorpe/go-tablewriter v0.0.8/go.mod h1:NVBvytpL+6fHfCKn0+3lSi15/G3A1HWf2cLNeHg6YBg=
 github.com/go-audio/audio v1.0.0 h1:zS9vebldgbQqktK4H0lUqWrG8P0NxCJVqcj7ZpNnwd4=
@@ -18,18 +16,18 @@ github.com/go-audio/riff v1.0.0 h1:d8iCGbDvox9BfLagY94fBynxSPHO80LmZCaOsmKxokA=
 github.com/go-audio/riff v1.0.0/go.mod h1:l3cQwc85y79NQFCRB7TiPoNiaijp6q8Z0Uv38rVG498=
 github.com/go-audio/wav v1.1.0 h1:jQgLtbqBzY7G+BM8fXF7AHUk1uHUviWS4X39d5rsL2g=
 github.com/go-audio/wav v1.1.0/go.mod h1:mpe9qfwbScEbkd8uybLuIpTgHyrISw/OTuvjUW2iGtE=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
-github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/llgcode/draw2d v0.0.0-20240627062922-0ed1ff131195 h1:Vdz2cBh5Fw2MYHWi3ED2PraDQaWEUhNCr1XFHrP4N5A=
+github.com/llgcode/draw2d v0.0.0-20240627062922-0ed1ff131195/go.mod h1:1Vk0LDW6jG5cGc2D9RQUxHaE0vYhTvIwSo9mOL6K4/U=
 github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
 github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mutablelogic/go-client v1.0.8 h1:A3QtP0wdf+W3dE5k7dobwGYqqn4ZpIqRFu+h9vPoy7Y=
-github.com/mutablelogic/go-client v1.0.8/go.mod h1:aP9ecBd4R/acJEJSyp81U3mey9W3AHQV/G1XzfcrLx0=
 github.com/mutablelogic/go-client v1.0.9 h1:Eh4sjQOFDldP/L3IizqkcOD3WigZR+u1VaHTUM4ujYw=
 github.com/mutablelogic/go-client v1.0.9/go.mod h1:VLyB8j8IBJSK/FXvvqhmq93PRWDKkyLu8R7V2Vudb6A=
-github.com/mutablelogic/go-server v1.4.11 h1:feI9IyuK6pv7Gi7fbExfU51uDRMfQo0U9wo0vWN2wf8=
-github.com/mutablelogic/go-server v1.4.11/go.mod h1:9nenPAohKu8bFoRgwHJh+3s8h0kLFjUAb8KZvT1TQNU=
+github.com/mutablelogic/go-media v1.6.8 h1:3v4povSQlOnvg9mHx6Bp9NVdCCjrNdDCjMHBGFHnVE8=
+github.com/mutablelogic/go-media v1.6.8/go.mod h1:HulNT0yyH63a3FRlbuzNDakhOypYrmtFVkHEXZjDgAY=
 github.com/mutablelogic/go-server v1.4.13 h1:k5LJJ/pCvyiw34UX341vRhliBOS6i7V65U/UICcOJOA=
 github.com/mutablelogic/go-server v1.4.13/go.mod h1:9nenPAohKu8bFoRgwHJh+3s8h0kLFjUAb8KZvT1TQNU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -39,14 +37,12 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
-golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8=
+golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
+golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ=
+golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
-golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk=
 golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/whisper/api/transcribe.go

@@ -8,6 +8,7 @@ import (
 
 	// Packages
 
+	"github.com/go-audio/wav"
 	"github.com/mutablelogic/go-server/pkg/httprequest"
 	"github.com/mutablelogic/go-server/pkg/httpresponse"
 	"github.com/mutablelogic/go-whisper/pkg/whisper"
@@ -67,28 +68,32 @@ func TranscribeFile(ctx context.Context, service *whisper.Whisper, w http.Respon
 	defer f.Close()
 
 	// Read samples
-	//buf, err := wav.NewDecoder(f).FullPCMBuffer()
-	//if err != nil {
-	//	httpresponse.Error(w, http.StatusInternalServerError, err.Error())
-	//	return
-	//}
+	buf, err := wav.NewDecoder(f).FullPCMBuffer()
+	if err != nil {
+		httpresponse.Error(w, http.StatusInternalServerError, err.Error())
+		return
+	}
 
 	// Get context for the model, perform transcription
 	var result *whisper.Transcription
-	if err := service.WithModel(model, func(ctx *task.Context) error {
+	if err := service.WithModel(model, func(task *task.Context) error {
+		// Check model
+		if translate && !task.CanTranslate() {
+			return ErrBadParameter.With("model is not multilingual, cannot translate")
+		}
+
 		// Set parameters for transcription & translation, default to english
-		ctx.SetTranslate(translate)
+		task.SetTranslate(translate)
 		if req.Language != nil {
-			if err := ctx.SetLanguage(*req.Language); err != nil {
+			if err := task.SetLanguage(*req.Language); err != nil {
 				return err
 			}
 		} else if translate {
-			if err := ctx.SetLanguage("en"); err != nil {
+			if err := task.SetLanguage("en"); err != nil {
 				return err
 			}
 		}
-
-		// Set prompt and temperature
+		// TODO Set prompt and temperature
 		/*
 			if req.Prompt != nil {
 				ctx.SetPrompt(*req.Prompt)
@@ -98,8 +103,7 @@ func TranscribeFile(ctx context.Context, service *whisper.Whisper, w http.Respon
 			}
 		*/
 		// Perform the transcription, return any errors
-		//result, err = service.Transcribe(ctx, buf.AsFloat32Buffer().Data)
-		return ErrNotImplemented
+		return task.Transcribe(ctx, buf.AsFloat32Buffer().Data)
 	}); err != nil {
 		httpresponse.Error(w, http.StatusBadRequest, err.Error())
 		return

pkg/whisper/segmenter/segmenter.go

@@ -0,0 +1,128 @@
+package segmenter
+
+import (
+	"context"
+	"errors"
+	"io"
+	"time"
+
+	// Packages
+	media "github.com/mutablelogic/go-media"
+	ffmpeg "github.com/mutablelogic/go-media/pkg/ffmpeg"
+
+	// Namespace imports
+	. "github.com/djthorpe/go-errors"
+)
+
+type Segmenter struct {
+	ts          time.Duration
+	sample_rate int
+	n           int
+	buf         []float32
+	reader      *ffmpeg.Reader
+}
+
+// SegmentFunc is a callback function which is called when a segment is ready
+// to be processed. The first argument is the timestamp of the segment.
+type SegmentFunc func(time.Duration, []float32)
+
+//////////////////////////////////////////////////////////////////////////////
+// LIFECYCLE
+
+// Create a new segmenter for "NumSamples" with a reader r
+// If NumSamples is zero then no segmenting is performed
+func NewSegmenter(r io.Reader, dur time.Duration, sample_rate int) (*Segmenter, error) {
+	segmenter := new(Segmenter)
+
+	// Check arguments
+	if dur < 0 || sample_rate <= 0 {
+		return nil, ErrBadParameter.With("invalid duration or sample rate arguments")
+	} else {
+		segmenter.sample_rate = sample_rate
+	}
+
+	// Sample buffer is duration * sample rate
+	if dur > 0 {
+		segmenter.n = int(dur.Seconds()) * sample_rate
+		segmenter.buf = make([]float32, 0, int(dur.Seconds())*sample_rate)
+	}
+
+	// Open the file
+	media, err := ffmpeg.NewReader(r)
+	if err != nil {
+		return nil, err
+	} else {
+		segmenter.reader = media
+	}
+
+	return segmenter, nil
+}
+
+// Close the segmenter
+func (s *Segmenter) Close() error {
+	var result error
+
+	if s.reader != nil {
+		result = errors.Join(result, s.reader.Close())
+	}
+	s.reader = nil
+	s.buf = nil
+
+	// Return any errors
+	return result
+}
+
+//////////////////////////////////////////////////////////////////////////////
+// PUBLIC METHODS
+
+// TODO: segments are output through a callback, with the samples and a timestamp
+// TODO: we could do some basic silence and voice detection to segment to ensure
+// we don't overtax the CPU/GPU with silence and non-speech
+func (s *Segmenter) Decode(ctx context.Context, fn SegmentFunc) error {
+	// Check input parameters
+	if fn == nil {
+		return ErrBadParameter.With("SegmentFunc is nil")
+	}
+
+	// Map function chooses the best audio stream
+	mapFunc := func(stream int, params *ffmpeg.Par) (*ffmpeg.Par, error) {
+		if stream == s.reader.BestStream(media.AUDIO) {
+			return ffmpeg.NewAudioPar("flt", "mono", s.sample_rate)
+		}
+		// Ignore no-audio streams
+		return nil, nil
+	}
+
+	// Decode samples and segment
+	if err := s.reader.Decode(ctx, mapFunc, func(stream int, frame *ffmpeg.Frame) error {
+		// We get null frames sometimes, ignore them
+		if frame == nil {
+			return nil
+		}
+
+		// Append float32 samples from plane 0 to buffer
+		s.buf = append(s.buf, frame.Float32(0)...)
+
+		// n != 0 and len(buf) >= n we have a segment to process
+		if s.n != 0 && len(s.buf) >= s.n {
+			fn(s.ts, s.buf)
+			// Clear the buffer
+			s.buf = s.buf[:0]
+			// Increment the timestamp
+			s.ts += time.Duration(float64(s.n)/float64(s.sample_rate)) * time.Second
+		}
+
+		// Continue processing
+		return nil
+	}); err != nil {
+		return err
+	}
+
+	// Output any remaining samples
+	if len(s.buf) > 0 {
+		fn(s.ts, s.buf)
+	}
+
+	// Return success
+	return nil
+}

pkg/whisper/segmenter/segmenter_test.go

@@ -0,0 +1,52 @@
+package segmenter_test
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	// Packages
+	segmenter "github.com/mutablelogic/go-whisper/pkg/whisper/segmenter"
+	assert "github.com/stretchr/testify/assert"
+)
+
+const SAMPLE = "../../../samples/OlivierL.wav"
+
+func Test_segmenter_001(t *testing.T) {
+	assert := assert.New(t)
+
+	f, err := os.Open(SAMPLE)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	segmenter, err := segmenter.NewSegmenter(f, time.Second, 16000)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	defer segmenter.Close()
+
+	assert.NoError(segmenter.Decode(context.Background(), func(ts time.Duration, buf []float32) {
+		t.Log(ts, len(buf))
+	}))
+}
+
+func Test_segmenter_002(t *testing.T) {
+	assert := assert.New(t)
+
+	f, err := os.Open(SAMPLE)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+
+	// No segmentation, just output the audio
+	segmenter, err := segmenter.NewSegmenter(f, 0, 16000)
+	if !assert.NoError(err) {
+		t.SkipNow()
+	}
+	defer segmenter.Close()
+
+	assert.NoError(segmenter.Decode(context.Background(), func(ts time.Duration, buf []float32) {
+		t.Log(ts, len(buf))
+	}))
+}