Fixed streaming responses

Author: djthorpe

cmd/llm/chat.go

@@ -38,15 +38,16 @@ func (cmd *ChatCmd) Run(globals *Globals) error {
 			return err
 		}
 
+		// Current buffer
+		var buf string
+
 		// Set the options
 		opts := []llm.Opt{}
 		if !cmd.NoStream {
 			opts = append(opts, llm.WithStream(func(cc llm.Completion) {
-				if text := cc.Text(0); text != "" {
-					count := strings.Count(text, "\n")
-					fmt.Print(strings.Repeat("\033[F", count) + strings.Repeat(" ", count) + "\r")
-					fmt.Print(text)
-				}
+				text := cc.Text(0)
+				fmt.Print(strings.TrimPrefix(text, buf))
+				buf = text
 			}))
 		}
 		if cmd.System != "" {
@@ -91,6 +92,7 @@ func (cmd *ChatCmd) Run(globals *Globals) error {
 				if len(calls) == 0 {
 					break
 				}
+
 				if session.Text(0) != "" {
 					globals.term.Println(session.Text(0))
 				} else {
@@ -100,15 +102,20 @@ func (cmd *ChatCmd) Run(globals *Globals) error {
 					}
 					globals.term.Println("Calling ", strings.Join(names, ", "))
 				}
+
 				if results, err := globals.toolkit.Run(ctx, calls...); err != nil {
 					return err
 				} else if err := session.FromTool(ctx, results...); err != nil {
 					return err
 				}
 			}
 
-			// Print the response
-			globals.term.Println("\n" + session.Text(0) + "\n")
+			// Print the response, if not streaming
+			if cmd.NoStream {
+				globals.term.Println("\n" + session.Text(0) + "\n")
+			} else {
+				globals.term.Println()
+			}
 		}
 	})
 }

cmd/llm/main.go

@@ -109,15 +109,17 @@ func main() {
 
 	// Create an agent
 	opts := []llm.Opt{}
-	if cli.OllamaEndpoint != "" {
-		opts = append(opts, agent.WithOllama(cli.OllamaEndpoint, clientopts...))
-	}
-	if cli.AnthropicKey != "" {
-		opts = append(opts, agent.WithAnthropic(cli.AnthropicKey, clientopts...))
-	}
-	if cli.MistralKey != "" {
-		opts = append(opts, agent.WithMistral(cli.MistralKey, clientopts...))
-	}
+	/*
+		if cli.OllamaEndpoint != "" {
+			opts = append(opts, agent.WithOllama(cli.OllamaEndpoint, clientopts...))
+		}
+		if cli.AnthropicKey != "" {
+			opts = append(opts, agent.WithAnthropic(cli.AnthropicKey, clientopts...))
+		}
+		if cli.MistralKey != "" {
+			opts = append(opts, agent.WithMistral(cli.MistralKey, clientopts...))
+		}
+	*/
 	if cli.OpenAIKey != "" {
 		opts = append(opts, agent.WithOpenAI(cli.OpenAIKey, clientopts...))
 	}

cmd/llm/models.go

@@ -8,7 +8,6 @@ import (
 	// Packages
 	llm "github.com/mutablelogic/go-llm"
 	agent "github.com/mutablelogic/go-llm/pkg/agent"
-	ollama "github.com/mutablelogic/go-llm/pkg/ollama"
 )
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -83,12 +82,14 @@ func (cmd *DownloadModelCmd) Run(globals *Globals) error {
 		}
 		// Download the model
 		switch agent.Name() {
-		case "ollama":
-			model, err := agent.(*ollama.Client).PullModel(ctx, cmd.Model)
-			if err != nil {
-				return err
-			}
-			fmt.Println(model)
+		/*
+			case "ollama":
+				model, err := agent.(*ollama.Client).PullModel(ctx, cmd.Model)
+				if err != nil {
+					return err
+				}
+				fmt.Println(model)
+		*/
 		default:
 			return fmt.Errorf("Agent %q does not support model download", agent.Name())
 		}

pkg/agent/opt.go

@@ -4,15 +4,12 @@ import (
 	// Packages
 	client "github.com/mutablelogic/go-client"
 	llm "github.com/mutablelogic/go-llm"
-	anthropic "github.com/mutablelogic/go-llm/pkg/anthropic"
-	mistral "github.com/mutablelogic/go-llm/pkg/mistral"
-	ollama "github.com/mutablelogic/go-llm/pkg/ollama"
 	openai "github.com/mutablelogic/go-llm/pkg/openai"
 )
 
 ////////////////////////////////////////////////////////////////////////////////
 // PUBLIC METHODS
-
+/*
 func WithOllama(endpoint string, opts ...client.ClientOpt) llm.Opt {
 	return func(o *llm.Opts) error {
 		client, err := ollama.New(endpoint, opts...)
@@ -45,6 +42,7 @@ func WithMistral(key string, opts ...client.ClientOpt) llm.Opt {
 		}
 	}
 }
+*/
 
 func WithOpenAI(key string, opts ...client.ClientOpt) llm.Opt {
 	return func(o *llm.Opts) error {

pkg/openai/completion.go

@@ -3,7 +3,6 @@ package openai
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"strings"
 
 	// Packages
@@ -249,7 +248,6 @@ func streamEvent(response *Response, evt client.TextStreamEvent) error {
 }
 
 func appendCompletion(response *Response, c *Completion) error {
-	fmt.Println(c)
 	// Append a new completion
 	for {
 		if c.Index < uint64(len(response.Completions)) {
@@ -303,6 +301,31 @@ func appendCompletion(response *Response, c *Completion) error {
 		message.Media.Append(c.Delta.Media)
 	}
 
+	// Append tool calls
+	for i := range c.Delta.Calls {
+		if i >= len(message.Calls) {
+			message.Calls = append(message.Calls, toolcall{})
+		}
+	}
+
+	for i, call := range c.Delta.Calls {
+		if call.meta.Id != "" {
+			message.Calls[i].meta.Id = call.meta.Id
+		}
+		if call.meta.Index != 0 {
+			message.Calls[i].meta.Index = call.meta.Index
+		}
+		if call.meta.Type != "" {
+			message.Calls[i].meta.Type = call.meta.Type
+		}
+		if call.meta.Function.Name != "" {
+			message.Calls[i].meta.Function.Name = call.meta.Function.Name
+		}
+		if call.meta.Function.Arguments != "" {
+			message.Calls[i].meta.Function.Arguments += call.meta.Function.Arguments
+		}
+	}
+
 	// Return success
 	return nil
 }