Add ask, add proper logger

Author: chand1012

.gitignore

@@ -1,3 +1,4 @@
 ottodocs
 bin/**
 dist/**
+*.bleve

Justfile

@@ -8,7 +8,7 @@ build:
   go build -v -o bin/otto
 
 clean:
-  rm -f otto
+  rm -rf bin dist
 
 run *commands:
   go run main.go {{commands}}

document/markdown.go renamed to ai/markdown.go

@@ -1,4 +1,4 @@
-package document
+package ai
 
 import (
 	"errors"

ai/question.go

@@ -0,0 +1,54 @@
+package ai
+
+import (
+	"fmt"
+	"strings"
+
+	gopenai "github.com/CasualCodersProjects/gopenai"
+	ai_types "github.com/CasualCodersProjects/gopenai/types"
+	"github.com/chand1012/ottodocs/constants"
+	"github.com/pandodao/tokenizer-go"
+)
+
+func Question(filePath, fileContent, chatPrompt, APIKey string) (string, error) {
+	openai := gopenai.NewOpenAI(&gopenai.OpenAIOpts{
+		APIKey: APIKey,
+	})
+
+	question := "File Name: " + filePath + "\nQuestion: " + chatPrompt + "\n\n" + strings.TrimRight(string(fileContent), " \n") + "\nAnswer:"
+
+	messages := []ai_types.ChatMessage{
+		{
+			Content: constants.QUESTION_PROMPT,
+			Role:    "system",
+		},
+		{
+			Content: question,
+			Role:    "user",
+		},
+	}
+
+	tokens := tokenizer.MustCalToken(messages[0].Content) + tokenizer.MustCalToken(messages[1].Content)
+
+	maxTokens := constants.OPENAI_MAX_TOKENS - tokens
+
+	if maxTokens < 0 {
+		return "", fmt.Errorf("the prompt is too long. max length is %d. Got %d", constants.OPENAI_MAX_TOKENS, tokens)
+	}
+
+	req := ai_types.NewDefaultChatRequest("")
+	req.Messages = messages
+	req.MaxTokens = maxTokens
+	// lower the temperature to make the model more deterministic
+	// req.Temperature = 0.3
+
+	resp, err := openai.CreateChat(req)
+	if err != nil {
+		fmt.Printf("Error: %s", err)
+		return "", err
+	}
+
+	message := resp.Choices[0].Message.Content
+
+	return message, nil
+}

document/single_file.go renamed to ai/single_file.go

@@ -1,4 +1,4 @@
-package document
+package ai
 
 import (
 	"errors"
@@ -92,7 +92,7 @@ func SingleFile(filePath, chatPrompt, APIKey string) (string, error) {
 	resp, err := openai.CreateChat(req)
 	if err != nil {
 		fmt.Printf("Error: %s", err)
-		os.Exit(1)
+		return "", err
 	}
 
 	message := resp.Choices[0].Message.Content

cmd/ask.go

@@ -0,0 +1,165 @@
+/*
+Copyright © 2023 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/chand1012/git2gpt/prompt"
+	"github.com/spf13/cobra"
+
+	"github.com/chand1012/ottodocs/ai"
+	"github.com/chand1012/ottodocs/config"
+	"github.com/chand1012/ottodocs/utils"
+)
+
+// askCmd represents the ask command
+var askCmd = &cobra.Command{
+	Use:   "ask",
+	Short: "Ask a question about a file or repo",
+	Long:  `Uses a vector search to find the most similar questions and answers`,
+	Args: cobra.PositionalArgs(func(cmd *cobra.Command, args []string) error {
+		if len(args) < 1 {
+			return fmt.Errorf("requires a path to a repository or file")
+		}
+		return nil
+	}),
+	Run: func(cmd *cobra.Command, args []string) {
+		repoPath := args[0]
+		var fileName string
+		conf, err := config.Load()
+		if err != nil || conf.APIKey == "" {
+			// if the API key is not set, prompt the user to login
+			log.Error("Please login first.")
+			log.Error("Run `ottodocs login` to login.")
+			os.Exit(1)
+		}
+
+		if chatPrompt == "" {
+			fmt.Println("Please enter a question: ")
+			fmt.Scanln(&chatPrompt)
+			// strip the newline character
+			chatPrompt = strings.TrimRight(chatPrompt, " \n")
+		}
+
+		// if .index.bleve exists, delete it
+		if _, err := os.Stat(filepath.Join(args[0], ".index.bleve")); err == nil {
+			err = os.RemoveAll(filepath.Join(args[0], ".index.bleve"))
+			if err != nil {
+				log.Errorf("Error deleting index: %s", err)
+				os.Exit(1)
+			}
+		}
+
+		info, err := os.Stat(repoPath)
+		if err != nil {
+			log.Errorf("Error getting file info: %s", err)
+			os.Exit(1)
+		}
+		// check if the first arg is a file or a directory
+		// if it's a file, ask a question about that file directly
+		if info.IsDir() {
+			var index bleve.Index
+			var err error
+			mapping := bleve.NewIndexMapping()
+
+			// check if .index.bleve exists
+			// if it does, load it
+			// if it doesn't, create it
+			// fmt.Println("Indexing repo...")
+			index, err = bleve.New(filepath.Join(args[0], ".index.bleve"), mapping)
+			if err != nil {
+				log.Errorf("Error creating index: %s", err)
+				os.Exit(1)
+			}
+			// index the repo
+			ignoreList := prompt.GenerateIgnoreList(repoPath, ignoreFilePath, !ignoreGitignore)
+			ignoreList = append(ignoreList, filepath.Join(repoPath, ".gptignore"))
+			repo, err := prompt.ProcessGitRepo(repoPath, ignoreList)
+			if err != nil {
+				log.Errorf("Error processing repo: %s", err)
+				os.Exit(1)
+			}
+			// index the files
+			for _, file := range repo.Files {
+				err = index.Index(file.Path, file)
+				if err != nil {
+					log.Errorf("Error indexing file: %s", err)
+					os.Exit(1)
+				}
+			}
+
+			// ask a question about the repo
+			query := bleve.NewQueryStringQuery(chatPrompt)
+			search := bleve.NewSearchRequest(query)
+			searchResults, err := index.Search(search)
+			if err != nil {
+				log.Errorf("Error searching index: %s", err)
+				os.Exit(1)
+			}
+
+			// tokenize the question
+			tokens := utils.SimpleTokenize(chatPrompt)
+			for _, token := range tokens {
+				query := bleve.NewQueryStringQuery(token)
+				search := bleve.NewSearchRequest(query)
+				r, err := index.Search(search)
+				if err != nil {
+					log.Errorf("Error searching index: %s", err)
+					os.Exit(1)
+				}
+				// combines the searches, but doesn't weight by ID
+				searchResults.Merge(r)
+			}
+			hits := make(map[string]float64)
+
+			for _, results := range searchResults.Hits {
+				currentScore := hits[results.ID]
+				hits[results.ID] = currentScore + results.Score
+			}
+
+			var bestHit string
+			var bestScore float64
+			for hit, score := range hits {
+				if score > bestScore {
+					bestScore = score
+					bestHit = hit
+				}
+			}
+
+			fileName = bestHit
+		} else {
+			fileName = repoPath
+		}
+
+		fmt.Println("Asking question about " + fileName + "...")
+
+		// load the file
+		content, err := os.ReadFile(fileName)
+		if err != nil {
+			log.Errorf("Error reading file: %s", err)
+			os.Exit(1)
+		}
+
+		resp, err := ai.Question(fileName, string(content), chatPrompt, conf.APIKey)
+
+		if err != nil {
+			log.Errorf("Error asking question: %s", err)
+			os.Exit(1)
+		}
+
+		fmt.Println(resp)
+	},
+}
+
+func init() {
+	RootCmd.AddCommand(askCmd)
+	askCmd.Flags().StringVarP(&chatPrompt, "question", "q", "", "The question to ask")
+	askCmd.Flags().BoolVarP(&ignoreGitignore, "ignore-gitignore", "g", false, "ignore .gitignore file")
+	askCmd.Flags().StringVarP(&ignoreFilePath, "ignore", "n", "", "path to .gptignore file")
+}

cmd/chat.go

@@ -26,8 +26,8 @@ If '-q' is not specified, the user will be prompted to enter a question.
 		conf, err := config.Load()
 		if err != nil || conf.APIKey == "" {
 			// if the API key is not set, prompt the user to login
-			fmt.Println("Please login first.")
-			fmt.Println("Run `ottodocs login` to login.")
+			log.Error("Please login first.")
+			log.Error("Run `ottodocs login` to login.")
 			os.Exit(1)
 		}
 
@@ -44,7 +44,7 @@ If '-q' is not specified, the user will be prompted to enter a question.
 		// chat ChatGPT the question
 		resp, err := openai.CreateChatSimple(question, 0) // 0 sets the max tokens to 1024
 		if err != nil {
-			fmt.Printf("Error: %s", err)
+			log.Errorf("Error: %s", err)
 			os.Exit(1)
 		}
 

cmd/doc.go

@@ -9,8 +9,8 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/chand1012/ottodocs/ai"
 	"github.com/chand1012/ottodocs/config"
-	"github.com/chand1012/ottodocs/document"
 )
 
 // docCmd represents the doc command
@@ -20,8 +20,8 @@ var docCmd = &cobra.Command{
 	Long:  `Document a file using the OpenAI ChatGPT API.`,
 	Run: func(cmd *cobra.Command, args []string) {
 		if filePath == "" {
-			fmt.Println("Please provide a path to a file to document.")
-			fmt.Println("Run `ottodocs doc -h` for more information.")
+			log.Error("Please provide a path to a file to document.")
+			log.Error("Run `ottodocs doc -h` for more information.")
 			os.Exit(1)
 		}
 
@@ -32,44 +32,44 @@ var docCmd = &cobra.Command{
 		conf, err := config.Load()
 		if err != nil || conf.APIKey == "" {
 			// if the API key is not set, prompt the user to login
-			fmt.Println("Please login first.")
-			fmt.Println("Run `ottodocs login` to login.")
+			log.Error("Please login first.")
+			log.Error("Run `ottodocs login` to login.")
 			os.Exit(1)
 		}
 
 		var contents string
 
 		if inlineMode || !markdownMode {
-			contents, err = document.SingleFile(filePath, chatPrompt, conf.APIKey)
+			contents, err = ai.SingleFile(filePath, chatPrompt, conf.APIKey)
 		} else {
-			contents, err = document.Markdown(filePath, chatPrompt, conf.APIKey)
+			contents, err = ai.Markdown(filePath, chatPrompt, conf.APIKey)
 		}
 
 		if err != nil {
-			fmt.Printf("Error: %s", err)
+			log.Errorf("Error: %s", err)
 			os.Exit(1)
 		}
 
 		if outputFile != "" {
 			// write the string to the output file
 			err = os.WriteFile(outputFile, []byte(contents), 0644)
 			if err != nil {
-				fmt.Printf("Error: %s", err)
+				log.Errorf("Error: %s", err)
 				os.Exit(1)
 			}
 		} else if overwriteOriginal {
 			// overwrite the original file
 			// clear the contents of the file
 			file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 			if err != nil {
-				fmt.Printf("Error: %s", err)
+				log.Errorf("Error: %s", err)
 				os.Exit(1)
 			}
 
 			// write the new contents to the file
 			_, err = file.WriteString(contents)
 			if err != nil {
-				fmt.Printf("Error: %s", err)
+				log.Errorf("Error: %s", err)
 				os.Exit(1)
 			}
 		} else {