Merge pull request #54 from mutablelogic/stream

Stream

Author: djthorpe

Makefile

@@ -39,8 +39,8 @@ generate: mkdir go-tidy
 
 # Make server
 server: mkdir generate go-tidy libwhisper libggml
-	@echo "Building whisper-server"
-	@PKG_CONFIG_PATH=${ROOT_PATH}/${BUILD_DIR} ${GO} build ${BUILD_FLAGS} -o ${BUILD_DIR}/whisper-server ./cmd/server
+	@echo "Building whisper"
+	@PKG_CONFIG_PATH=${ROOT_PATH}/${BUILD_DIR} ${GO} build ${BUILD_FLAGS} -o ${BUILD_DIR}/whisper ./cmd/whisper
 
 # Make cli
 cli: mkdir generate go-tidy

cmd/server/main.go

@@ -13,7 +13,7 @@ import (
 	// Packages
 	context "github.com/mutablelogic/go-server/pkg/context"
 	httpserver "github.com/mutablelogic/go-server/pkg/httpserver"
-	whisper "github.com/mutablelogic/go-whisper/pkg/whisper"
+	whisper "github.com/mutablelogic/go-whisper"
 	api "github.com/mutablelogic/go-whisper/pkg/whisper/api"
 	version "github.com/mutablelogic/go-whisper/pkg/whisper/version"
 )

cmd/whisper/download.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"log"
+
+	// Packages
+	"github.com/djthorpe/go-tablewriter"
+)
+
+type DownloadCmd struct {
+	Model string `arg:"" help:"Model to download"`
+}
+
+func (cmd *DownloadCmd) Run(ctx *Globals) error {
+	model, err := ctx.service.DownloadModel(ctx.ctx, cmd.Model, func(curBytes, totalBytes uint64) {
+		log.Printf("Downloaded %d of %d bytes", curBytes, totalBytes)
+	})
+	if err != nil {
+		return err
+	}
+	return ctx.writer.Write(model, tablewriter.OptHeader())
+}

cmd/whisper/main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	// Packages
+	kong "github.com/alecthomas/kong"
+	tablewriter "github.com/djthorpe/go-tablewriter"
+	ctx "github.com/mutablelogic/go-server/pkg/context"
+	whisper "github.com/mutablelogic/go-whisper"
+)
+
+type Globals struct {
+	NoGPU bool   `name:"nogpu" help:"Disable GPU acceleration"`
+	Debug bool   `name:"debug" help:"Enable debug output"`
+	Dir   string `name:"dir" help:"Path to model store, uses ${WHISPER_DIR} " default:"${WHISPER_DIR}"`
+
+	// Writer, service and context
+	writer  *tablewriter.Writer
+	service *whisper.Whisper
+	ctx     context.Context
+}
+
+type CLI struct {
+	Globals
+	Models   ModelsCmd   `cmd:"models" help:"List models"`
+	Download DownloadCmd `cmd:"download" help:"Download a model"`
+	Server   ServerCmd   `cmd:"models" help:"Run the whisper server"`
+}
+
+func main() {
+	// The name of the executable
+	name, err := os.Executable()
+	if err != nil {
+		panic(err)
+	} else {
+		name = filepath.Base(name)
+	}
+
+	// Create a cli parser
+	cli := CLI{}
+	cmd := kong.Parse(&cli,
+		kong.Name(name),
+		kong.Description("speech transcription and translation service"),
+		kong.UsageOnError(),
+		kong.ConfigureHelp(kong.HelpOptions{Compact: true}),
+		kong.Vars{
+			"WHISPER_DIR": dirEnvOrDefault(name),
+		},
+	)
+
+	// Create a whisper server - set options
+	opts := []whisper.Opt{}
+	if cli.Globals.Debug {
+		opts = append(opts, whisper.OptDebug())
+	}
+	if cli.Globals.NoGPU {
+		opts = append(opts, whisper.OptNoGPU())
+	}
+
+	// Create directory if it doesn't exist
+	if err := os.MkdirAll(cli.Globals.Dir, 0755); err != nil {
+		cmd.FatalIfErrorf(err)
+		return
+	}
+
+	// Create a whisper server - create
+	service, err := whisper.New(cli.Globals.Dir, opts...)
+	if err != nil {
+		cmd.FatalIfErrorf(err)
+		return
+	} else {
+		cli.Globals.service = service
+	}
+	defer service.Close()
+
+	// Create a tablewriter object with text output
+	writer := tablewriter.New(os.Stdout, tablewriter.OptOutputText())
+	cli.Globals.writer = writer
+
+	// Create a context
+	cli.Globals.ctx = ctx.ContextForSignal(os.Interrupt, syscall.SIGQUIT)
+
+	// Run the command
+	if err := cmd.Run(&cli.Globals); err != nil {
+		cmd.FatalIfErrorf(err)
+	}
+}
+
+func dirEnvOrDefault(name string) string {
+	if dir := os.Getenv("WHISPER_DIR"); dir != "" {
+		return dir
+	}
+	if dir, err := os.UserCacheDir(); err == nil {
+		return filepath.Join(dir, name)
+	}
+	return os.TempDir()
+}

cmd/whisper/models.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	// Packages
+	"github.com/djthorpe/go-tablewriter"
+)
+
+type ModelsCmd struct{}
+
+func (*ModelsCmd) Run(ctx *Globals) error {
+	return ctx.writer.Write(ctx.service.ListModels(), tablewriter.OptHeader())
+}

cmd/whisper/server.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"log"
+	"net/http"
+
+	// Packages
+	"github.com/mutablelogic/go-server/pkg/httpserver"
+	"github.com/mutablelogic/go-whisper/pkg/whisper/api"
+)
+
+type ServerCmd struct {
+	Endpoint string `name:"endpoint" help:"Endpoint for the server" default:"/api/v1"`
+	Listen   string `name:"listen" help:"Listen address for the server" default:"localhost:8080"`
+}
+
+func (cmd *ServerCmd) Run(ctx *Globals) error {
+	// Create a mux for serving requests, then register the endpoints with the mux
+	mux := http.NewServeMux()
+
+	// Register the endpoints
+	api.RegisterEndpoints(cmd.Endpoint, mux, ctx.service)
+
+	// Create a new HTTP server
+	log.Println("List address", cmd.Listen)
+	server, err := httpserver.Config{
+		Listen: cmd.Listen,
+		Router: mux,
+	}.New()
+	if err != nil {
+		return err
+	}
+
+	// Run the server until CTRL+C
+	log.Println("Press CTRL+C to exit")
+	return server.Run(ctx.ctx)
+}

pkg/whisper/opt.go renamed to opt.go

@@ -0,0 +1,26 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	req int32
+)
+
+func wrapLogging(fn http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		req := nextReq()
+		delta := time.Now()
+		log.Printf("R%d %s %s", req, r.Method, r.URL)
+		fn(w, r)
+		log.Printf("R%d Took %v", req, time.Since(delta).Truncate(time.Millisecond))
+	}
+}
+
+func nextReq() int32 {
+	return atomic.AddInt32(&req, 1)
+}

pkg/whisper/api/logging.go

@@ -11,7 +11,7 @@ import (
 	// Packages
 	"github.com/mutablelogic/go-server/pkg/httprequest"
 	"github.com/mutablelogic/go-server/pkg/httpresponse"
-	"github.com/mutablelogic/go-whisper/pkg/whisper"
+	"github.com/mutablelogic/go-whisper"
 	"github.com/mutablelogic/go-whisper/pkg/whisper/schema"
 )
 

pkg/whisper/api/models.go

@@ -6,7 +6,7 @@ import (
 
 	// Packages
 	"github.com/mutablelogic/go-server/pkg/httpresponse"
-	"github.com/mutablelogic/go-whisper/pkg/whisper"
+	"github.com/mutablelogic/go-whisper"
 )
 
 /////////////////////////////////////////////////////////////////////////////
@@ -15,7 +15,7 @@ import (
 func RegisterEndpoints(base string, mux *http.ServeMux, whisper *whisper.Whisper) {
 	// Health: GET /v1/health
 	//   returns an empty OK response
-	mux.HandleFunc(joinPath(base, "health"), func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc(joinPath(base, "health"), wrapLogging(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
 		switch r.Method {
@@ -24,14 +24,14 @@ func RegisterEndpoints(base string, mux *http.ServeMux, whisper *whisper.Whisper
 		default:
 			httpresponse.Error(w, http.StatusMethodNotAllowed)
 		}
-	})
+	}))
 
 	// List Models: GET /v1/models
 	//   returns available models
 	// Download Model: POST /v1/models?stream={bool}
 	//   downloads a model from the server
 	//   if stream is true then progress is streamed back to the client
-	mux.HandleFunc(joinPath(base, "models"), func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc(joinPath(base, "models"), wrapLogging(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
 		switch r.Method {
@@ -42,13 +42,13 @@ func RegisterEndpoints(base string, mux *http.ServeMux, whisper *whisper.Whisper
 		default:
 			httpresponse.Error(w, http.StatusMethodNotAllowed)
 		}
-	})
+	}))
 
 	// Get: GET /v1/models/{id}
 	//   returns an existing model
 	// Delete: DELETE /v1/models/{id}
 	//   deletes an existing model
-	mux.HandleFunc(joinPath(base, "models/{id}"), func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc(joinPath(base, "models/{id}"), wrapLogging(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
 		id := r.PathValue("id")
@@ -60,12 +60,12 @@ func RegisterEndpoints(base string, mux *http.ServeMux, whisper *whisper.Whisper
 		default:
 			httpresponse.Error(w, http.StatusMethodNotAllowed)
 		}
-	})
+	}))
 
 	// Translate: POST /v1/audio/translations
 	//   Translates audio into english or another language  - language parameter should be set to the
 	//   destination language of the audio. Will default to english if not set.
-	mux.HandleFunc(joinPath(base, "audio/translations"), func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc(joinPath(base, "audio/translations"), wrapLogging(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
 		switch r.Method {
@@ -74,12 +74,12 @@ func RegisterEndpoints(base string, mux *http.ServeMux, whisper *whisper.Whisper
 		default:
 			httpresponse.Error(w, http.StatusMethodNotAllowed)
 		}
-	})
+	}))
 
 	// Transcribe: POST /v1/audio/transcriptions
 	//   Transcribes audio into the input language - language parameter should be set to the source
 	//   language of the audio
-	mux.HandleFunc(joinPath(base, "audio/transcriptions"), func(w http.ResponseWriter, r *http.Request) {
+	mux.HandleFunc(joinPath(base, "audio/transcriptions"), wrapLogging(func(w http.ResponseWriter, r *http.Request) {
 		defer r.Body.Close()
 
 		switch r.Method {
@@ -102,7 +102,21 @@ func RegisterEndpoints(base string, mux *http.ServeMux, whisper *whisper.Whisper
 		default:
 			httpresponse.Error(w, http.StatusMethodNotAllowed)
 		}
-	})
+	}))
+
+	// Transcribe: POST /v1/audio/transcriptions/{model-id}
+	//   Transcribes streamed media into the input language
+	mux.HandleFunc(joinPath(base, "audio/transcriptions/{model}"), wrapLogging(func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+
+		model := r.PathValue("model")
+		switch r.Method {
+		case http.MethodPost:
+			TranscribeStream(r.Context(), whisper, w, r, model)
+		default:
+			httpresponse.Error(w, http.StatusMethodNotAllowed)
+		}
+	}))
 }
 
 /////////////////////////////////////////////////////////////////////////////