Merge pull request #55 from mutablelogic/stream

Stream

Author: djthorpe

Makefile

@@ -30,22 +30,17 @@ ifeq ($(GGML_CUDA),1)
 endif
 
 # Targets
-all: server cli
+all: whisper
 
 # Generate the pkg-config files
 generate: mkdir go-tidy
 	@echo "Generating pkg-config"
 	@PKG_CONFIG_PATH=${ROOT_PATH}/${BUILD_DIR} go generate ./sys/whisper
 
-# Make server
-server: mkdir generate go-tidy libwhisper libggml
-	@echo "Building whisper-server"
-	@PKG_CONFIG_PATH=${ROOT_PATH}/${BUILD_DIR} ${GO} build ${BUILD_FLAGS} -o ${BUILD_DIR}/whisper-server ./cmd/server
-
-# Make cli
-cli: mkdir generate go-tidy
-	@echo "Building whisper-cli"
-	@PKG_CONFIG_PATH=${ROOT_PATH}/${BUILD_DIR} ${GO} build ${BUILD_FLAGS} -o ${BUILD_DIR}/whisper-cli ./cmd/cli
+# Make whisper
+whisper: mkdir generate go-tidy libwhisper libggml
+	@echo "Building whisper"
+	@PKG_CONFIG_PATH=${ROOT_PATH}/${BUILD_DIR} ${GO} build ${BUILD_FLAGS} -o ${BUILD_DIR}/whisper ./cmd/whisper
 
 # Build docker container
 docker: docker-dep submodule
@@ -75,11 +70,6 @@ libggml: submodule
 	@echo "Building libggml.a"
 	@cd third_party/whisper.cpp && make -j$(nproc) libggml.a
 
-# Build whisper-server
-whisper-server: submodule
-	@echo "Building whisper-server"
-	@cd third_party/whisper.cpp && make -j$(nproc) server
-	
 # Push docker container
 docker-push: docker-dep 
 	@echo push docker image: ${BUILD_TAG}

cmd/server/main.go

@@ -13,7 +13,7 @@ import (
 	// Packages
 	context "github.com/mutablelogic/go-server/pkg/context"
 	httpserver "github.com/mutablelogic/go-server/pkg/httpserver"
-	whisper "github.com/mutablelogic/go-whisper/pkg/whisper"
+	whisper "github.com/mutablelogic/go-whisper"
 	api "github.com/mutablelogic/go-whisper/pkg/whisper/api"
 	version "github.com/mutablelogic/go-whisper/pkg/whisper/version"
 )

cmd/whisper/download.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"log"
+
+	// Packages
+	"github.com/djthorpe/go-tablewriter"
+)
+
+type DownloadCmd struct {
+	Model string `arg:"" help:"Model to download"`
+}
+
+func (cmd *DownloadCmd) Run(ctx *Globals) error {
+	model, err := ctx.service.DownloadModel(ctx.ctx, cmd.Model, func(curBytes, totalBytes uint64) {
+		log.Printf("Downloaded %d of %d bytes", curBytes, totalBytes)
+	})
+	if err != nil {
+		return err
+	}
+	return ctx.writer.Write(model, tablewriter.OptHeader())
+}

cmd/whisper/main.go

@@ -0,0 +1,101 @@
+package main
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	// Packages
+	kong "github.com/alecthomas/kong"
+	tablewriter "github.com/djthorpe/go-tablewriter"
+	ctx "github.com/mutablelogic/go-server/pkg/context"
+	whisper "github.com/mutablelogic/go-whisper"
+)
+
+type Globals struct {
+	NoGPU bool   `name:"nogpu" help:"Disable GPU acceleration"`
+	Debug bool   `name:"debug" help:"Enable debug output"`
+	Dir   string `name:"dir" help:"Path to model store, uses ${WHISPER_DIR} " default:"${WHISPER_DIR}"`
+
+	// Writer, service and context
+	writer  *tablewriter.Writer
+	service *whisper.Whisper
+	ctx     context.Context
+}
+
+type CLI struct {
+	Globals
+	Models   ModelsCmd   `cmd:"models" help:"List models"`
+	Download DownloadCmd `cmd:"download" help:"Download a model"`
+	Server   ServerCmd   `cmd:"models" help:"Run the whisper server"`
+}
+
+func main() {
+	// The name of the executable
+	name, err := os.Executable()
+	if err != nil {
+		panic(err)
+	} else {
+		name = filepath.Base(name)
+	}
+
+	// Create a cli parser
+	cli := CLI{}
+	cmd := kong.Parse(&cli,
+		kong.Name(name),
+		kong.Description("speech transcription and translation service"),
+		kong.UsageOnError(),
+		kong.ConfigureHelp(kong.HelpOptions{Compact: true}),
+		kong.Vars{
+			"WHISPER_DIR": dirEnvOrDefault(name),
+		},
+	)
+
+	// Create a whisper server - set options
+	opts := []whisper.Opt{}
+	if cli.Globals.Debug {
+		opts = append(opts, whisper.OptDebug())
+	}
+	if cli.Globals.NoGPU {
+		opts = append(opts, whisper.OptNoGPU())
+	}
+
+	// Create directory if it doesn't exist
+	if err := os.MkdirAll(cli.Globals.Dir, 0755); err != nil {
+		cmd.FatalIfErrorf(err)
+		return
+	}
+
+	// Create a whisper server - create
+	service, err := whisper.New(cli.Globals.Dir, opts...)
+	if err != nil {
+		cmd.FatalIfErrorf(err)
+		return
+	} else {
+		cli.Globals.service = service
+	}
+	defer service.Close()
+
+	// Create a tablewriter object with text output
+	writer := tablewriter.New(os.Stdout, tablewriter.OptOutputText())
+	cli.Globals.writer = writer
+
+	// Create a context
+	cli.Globals.ctx = ctx.ContextForSignal(os.Interrupt, syscall.SIGQUIT)
+
+	// Run the command
+	if err := cmd.Run(&cli.Globals); err != nil {
+		cmd.FatalIfErrorf(err)
+	}
+}
+
+func dirEnvOrDefault(name string) string {
+	if dir := os.Getenv("WHISPER_DIR"); dir != "" {
+		return dir
+	}
+	if dir, err := os.UserCacheDir(); err == nil {
+		return filepath.Join(dir, name)
+	}
+	return os.TempDir()
+}

cmd/whisper/models.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	// Packages
+	"github.com/djthorpe/go-tablewriter"
+)
+
+type ModelsCmd struct{}
+
+func (*ModelsCmd) Run(ctx *Globals) error {
+	return ctx.writer.Write(ctx.service.ListModels(), tablewriter.OptHeader())
+}

cmd/whisper/server.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"log"
+	"net/http"
+
+	// Packages
+	"github.com/mutablelogic/go-server/pkg/httpserver"
+	"github.com/mutablelogic/go-whisper/pkg/whisper/api"
+)
+
+type ServerCmd struct {
+	Endpoint string `name:"endpoint" help:"Endpoint for the server" default:"/api/v1"`
+	Listen   string `name:"listen" help:"Listen address for the server" default:"localhost:8080"`
+}
+
+func (cmd *ServerCmd) Run(ctx *Globals) error {
+	// Create a mux for serving requests, then register the endpoints with the mux
+	mux := http.NewServeMux()
+
+	// Register the endpoints
+	api.RegisterEndpoints(cmd.Endpoint, mux, ctx.service)
+
+	// Create a new HTTP server
+	log.Println("List address", cmd.Listen)
+	server, err := httpserver.Config{
+		Listen: cmd.Listen,
+		Router: mux,
+	}.New()
+	if err != nil {
+		return err
+	}
+
+	// Run the server until CTRL+C
+	log.Println("Press CTRL+C to exit")
+	return server.Run(ctx.ctx)
+}

etc/Dockerfile.linux-amd64

@@ -32,8 +32,9 @@ RUN make -j$(nproc) server
 FROM ${BASE_CUDA_RUN_CONTAINER} AS runtime
 ARG CUDA_MAIN_VERSION=12.5
 RUN apt-get -y update && apt-get -y upgrade && apt-get -y install libgomp1
-COPY --from=build /app/build/whisper-server /usr/local/bin/whisper-server
+COPY --from=build /app/build/whisper /usr/local/bin/whisper
 ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 
 # Expose
-ENTRYPOINT [ "/usr/local/bin/whisper-server" ]
+ENTRYPOINT [ "/usr/local/bin/whisper" ]
+CMD [ "server", "--dir=/data" ]

etc/Dockerfile.linux-arm64

@@ -32,8 +32,9 @@ RUN make -j$(nproc) server
 FROM ${BASE_CUDA_RUN_CONTAINER} AS runtime
 ARG CUDA_MAIN_VERSION=12.2
 RUN apt-get -y update && apt-get -y upgrade && apt-get -y install libgomp1
-COPY --from=build /app/build/whisper-server /usr/local/bin/whisper-server
+COPY --from=build /app/build/whisper /usr/local/bin/whisper
 ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_MAIN_VERSION}/compat:$LD_LIBRARY_PATH
 
 # Expose
-ENTRYPOINT [ "/usr/local/bin/whisper-server" ]
+ENTRYPOINT [ "/usr/local/bin/whisper" ]
+CMD [ "server", "--dir=/data" ]

pkg/whisper/opt.go renamed to opt.go

@@ -0,0 +1,26 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"sync/atomic"
+	"time"
+)
+
+var (
+	req int32
+)
+
+func wrapLogging(fn http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		req := nextReq()
+		delta := time.Now()
+		log.Printf("R%d %s %s", req, r.Method, r.URL)
+		fn(w, r)
+		log.Printf("R%d Took %v", req, time.Since(delta).Truncate(time.Millisecond))
+	}
+}
+
+func nextReq() int32 {
+	return atomic.AddInt32(&req, 1)
+}